diff --git a/photon-api/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/AtlasTuner.scala b/photon-api/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/AtlasTuner.scala
deleted file mode 100644
index e9f95b49..00000000
--- a/photon-api/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/AtlasTuner.scala
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright 2018 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.tuner
-
-import breeze.linalg.DenseVector
-
-import com.linkedin.photon.ml.HyperparameterTuningMode
-import com.linkedin.photon.ml.HyperparameterTuningMode.HyperparameterTuningMode
-import com.linkedin.photon.ml.hyperparameter.EvaluationFunction
-import com.linkedin.photon.ml.hyperparameter.search.{GaussianProcessSearch, RandomSearch}
-
-/**
- * A hyper-parameter tuner which depends on an internal LinkedIn library.
- */
-class AtlasTuner[T] extends HyperparameterTuner[T] {
-
-  /**
-   * Search hyper-parameters to optimize the model
-   *
-   * @param n The number of points to find
-   * @param dimension Numbers of hyper-parameters to be tuned
-   * @param mode Hyper-parameter tuning mode (random or Bayesian)
-   * @param evaluationFunction Function that evaluates points in the space to real values
-   * @param observations Observations made prior to searching, from this data set (not mean-centered)
-   * @param priorObservations Observations made prior to searching, from past data sets (mean-centered)
-   * @param discreteParams Map that specifies the indices of discrete parameters and their numbers of discrete values
-   * @return A Seq of the found results
-   */
-  def search(
-      n: Int,
-      dimension: Int,
-      mode: HyperparameterTuningMode,
-      evaluationFunction: EvaluationFunction[T],
-      observations: Seq[(DenseVector[Double], Double)],
-      priorObservations: Seq[(DenseVector[Double], Double)] = Seq(),
-      discreteParams: Map[Int, Int] = Map()): Seq[T] = {
-
-    val searcher = mode match {
-      case HyperparameterTuningMode.BAYESIAN =>
-        new GaussianProcessSearch[T](dimension, evaluationFunction)
-
-      case HyperparameterTuningMode.RANDOM =>
-        new RandomSearch[T](dimension, evaluationFunction)
-    }
-
-    searcher.findWithPriors(n, observations, priorObservations)
-  }
-}
diff --git a/photon-api/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/DummyTuner.scala b/photon-api/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/DummyTuner.scala
deleted file mode 100644
index 0f089792..00000000
--- a/photon-api/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/DummyTuner.scala
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright 2018 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.tuner
-
-import breeze.linalg.DenseVector
-
-import com.linkedin.photon.ml.HyperparameterTuningMode.HyperparameterTuningMode
-import com.linkedin.photon.ml.hyperparameter.EvaluationFunction
-
-/**
- * A dummy hyper-parameter tuner which runs an empty operation.
- */
-class DummyTuner[T] extends HyperparameterTuner[T] {
-
-  /**
-   * Search hyper-parameters to optimize the model
-   *
-   * @param n The number of points to find
-   * @param dimension Numbers of hyper-parameters to be tuned
-   * @param mode Hyper-parameter tuning mode (random or Bayesian)
-   * @param evaluationFunction Function that evaluates points in the space to real values
-   * @param observations Observations made prior to searching, from this data set (not mean-centered)
-   * @param priorObservations Observations made prior to searching, from past data sets (mean-centered)
-   * @param discreteParams Map that specifies the indices of discrete parameters and their numbers of discrete values
-   * @return A Seq of the found results
-   */
-  def search(
-      n: Int,
-      dimension: Int,
-      mode: HyperparameterTuningMode,
-      evaluationFunction: EvaluationFunction[T],
-      observations: Seq[(DenseVector[Double], Double)],
-      priorObservations: Seq[(DenseVector[Double], Double)] = Seq(),
-      discreteParams: Map[Int, Int] = Map()): Seq[T] = Seq()
-}
diff --git a/photon-api/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/HyperparameterTunerFactory.scala b/photon-api/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/HyperparameterTunerFactory.scala
deleted file mode 100644
index 0d345b27..00000000
--- a/photon-api/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/HyperparameterTunerFactory.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2018 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.tuner
-
-import com.linkedin.photon.ml.HyperparameterTunerName.{ATLAS, DUMMY, HyperparameterTunerName}
-
-object HyperparameterTunerFactory {
-
-  // Use DUMMY_TUNER for photon-ml, which does an empty operation for hyper-parameter tuning
-  val DUMMY_TUNER = "com.linkedin.photon.ml.hyperparameter.tuner.DummyTuner"
-  val ATLAS_TUNER = "com.linkedin.atlas.tuner.AtlasTuner"
-
-  /**
-   * Factory for different packages of [[HyperparameterTuner]].
-   *
-   * @param tunerName The name of the auto-tuning package
-   * @return The hyper-parameter tuner
-   */
-  def apply[T](tunerName: HyperparameterTunerName): HyperparameterTuner[T] = {
-
-    val className = tunerName match {
-      case DUMMY => DUMMY_TUNER
-      case ATLAS => ATLAS_TUNER
-      case other => throw new IllegalArgumentException(s"Invalid HyperparameterTuner name: ${other.toString}")
-    }
-
-    try {
-      Class.forName(className)
-        .newInstance
-        .asInstanceOf[HyperparameterTuner[T]]
-    } catch {
-      case ex: Exception =>
-        throw new IllegalArgumentException(s"Invalid HyperparameterTuner class: $className", ex)
-    }
-  }
-}
diff --git a/photon-client/src/main/scala/com/linkedin/photon/ml/cli/game/training/GameTrainingDriver.scala b/photon-client/src/main/scala/com/linkedin/photon/ml/cli/game/training/GameTrainingDriver.scala
index 2e7a3a74..4c73b9e7 100644
--- a/photon-client/src/main/scala/com/linkedin/photon/ml/cli/game/training/GameTrainingDriver.scala
+++ b/photon-client/src/main/scala/com/linkedin/photon/ml/cli/game/training/GameTrainingDriver.scala
@@ -23,15 +23,16 @@ import org.apache.spark.sql.{DataFrame, SparkSession}
 import org.apache.spark.storage.StorageLevel
 
 import com.linkedin.photon.ml._
-import com.linkedin.photon.ml.HyperparameterTunerName.HyperparameterTunerName
-import com.linkedin.photon.ml.HyperparameterTuningMode.HyperparameterTuningMode
+import com.linkedin.photon.ml.hyperparameter.HyperparameterTuningMode.HyperparameterTuningMode
 import com.linkedin.photon.ml.TaskType.TaskType
 import com.linkedin.photon.ml.Types._
 import com.linkedin.photon.ml.cli.game.GameDriver
 import com.linkedin.photon.ml.data.{DataValidators, FixedEffectDataConfiguration, InputColumnsNames, RandomEffectDataConfiguration}
 import com.linkedin.photon.ml.data.avro.{AvroDataReader, ModelProcessingUtils}
 import com.linkedin.photon.ml.estimators.GameEstimator.GameOptimizationConfiguration
-import com.linkedin.photon.ml.estimators.{GameEstimator, GameEstimatorEvaluationFunction}
+import com.linkedin.photon.ml.estimators.GameEstimator
+import com.linkedin.photon.ml.hyperparameter.HyperparameterTuningMode
+import com.linkedin.photon.ml.hyperparameter.evaluation.GameEstimatorEvaluationFunction
 import com.linkedin.photon.ml.hyperparameter.tuner.HyperparameterTunerFactory
 import com.linkedin.photon.ml.index.{IndexMap, IndexMapLoader}
 import com.linkedin.photon.ml.io.{CoordinateConfiguration, ModelOutputMode, RandomEffectCoordinateConfiguration}
@@ -145,7 +146,7 @@ object GameTrainingDriver extends GameDriver {
     "Suggested depth for tree aggregation.",
     ParamValidators.gt[Int](0.0))
 
-  val hyperParameterTunerName: Param[HyperparameterTunerName] = ParamUtils.createParam[HyperparameterTunerName](
+  val hyperParameterTunerName: Param[String] = ParamUtils.createParam[String](
     "hyper parameter tuner",
     "Package name of hyperparameter tuner."
   )
@@ -217,7 +218,6 @@ object GameTrainingDriver extends GameDriver {
     setDefault(outputMode, ModelOutputMode.BEST)
     setDefault(overrideOutputDirectory, false)
     setDefault(normalization, NormalizationType.NONE)
-    setDefault(hyperParameterTunerName, HyperparameterTunerName.DUMMY)
     setDefault(hyperParameterTuning, HyperparameterTuningMode.NONE)
     setDefault(varianceComputationType, VarianceComputationType.NONE)
     setDefault(dataValidation, DataValidationType.VALIDATE_DISABLED)
@@ -329,13 +329,25 @@ object GameTrainingDriver extends GameDriver {
       case _ =>
     }
 
-    // If hyperparameter tuning is enabled, need to specify the number of tuning iterations
+    // If hyperparameter tuning is enabled...
     hyperParameterTuningMode match {
       case HyperparameterTuningMode.BAYESIAN | HyperparameterTuningMode.RANDOM =>
-        require(
+
+      // ... need to specify the hyperparameter tuner
+      require(
+          paramMap.get(hyperParameterTunerName).isDefined,
+          "Hyperparameter tuning enabled, but tuner not specified.")
+
+      // ... need to specify the number of tuning iterations
+      require(
           paramMap.get(hyperParameterTuningIter).isDefined,
           "Hyperparameter tuning enabled, but number of iterations unspecified.")
 
+      // ... validation data must be provided
+      require(
+        paramMap.get(validationDataDirectories).isDefined,
+        "Hyperparameter tuning enabled, but no validation data provided")
+
       case _ =>
     }
 
@@ -493,10 +505,16 @@ object GameTrainingDriver extends GameDriver {
       gameEstimator.fit(trainingData, validationData, gameOptimizationConfigs)
     }
 
-    val tunedModels = Timed("Tune hyperparameters") {
-      // Disable warm start for autotuning
-      gameEstimator.setUseWarmStart(false)
-      runHyperparameterTuning(gameEstimator, trainingData, validationData, explicitModels)
+    val tunedModels = getOrDefault(hyperParameterTuning) match {
+      case HyperparameterTuningMode.NONE =>
+        Seq()
+
+      case _ =>
+        Timed("Tune hyperparameters") {
+          // Disable warm start for autotuning
+          gameEstimator.setUseWarmStart(false)
+          runHyperparameterTuning(gameEstimator, trainingData, validationData, explicitModels)
+        }
     }
 
     trainingData.unpersist()
@@ -678,45 +696,42 @@ object GameTrainingDriver extends GameDriver {
       estimator: GameEstimator,
       trainingData: DataFrame,
       validationData: Option[DataFrame],
-      models: Seq[GameEstimator.GameResult]): Seq[GameEstimator.GameResult] =
-
-    validationData match {
-      case Some(testData) if getOrDefault(hyperParameterTuning) != HyperparameterTuningMode.NONE =>
-
-        val (_, baseConfig, evaluationResults) = models.head
-
-        val iteration = getOrDefault(hyperParameterTuningIter)
-
-        val dimension = baseConfig.toSeq.map {
-          case (_, config: GLMOptimizationConfiguration) =>
-            config.regularizationContext.regularizationType match {
-              case RegularizationType.ELASTIC_NET => 2
-              case RegularizationType.L2 => 1
-              case RegularizationType.L1 => 1
-              case RegularizationType.NONE => 0
-            }
-          case _ => throw new IllegalArgumentException(s"Unknown optimization config!")
-        }.sum
-
-        val mode = getOrDefault(hyperParameterTuning)
-
-        val evaluator = evaluationResults.get.primaryEvaluator
-        val isOptMax = evaluator.betterThan(1.0, 0.0)
-        val evaluationFunction = new GameEstimatorEvaluationFunction(
-          estimator,
-          baseConfig,
-          trainingData,
-          testData,
-          isOptMax)
+      models: Seq[GameEstimator.GameResult]): Seq[GameEstimator.GameResult] = {
+
+    val (_, baseConfig, evaluationResults) = models.head
+
+    val iteration = getOrDefault(hyperParameterTuningIter)
+    val dimension = baseConfig
+      .toSeq
+      .map {
+        case (_, config: GLMOptimizationConfiguration) =>
+          config.regularizationContext.regularizationType match {
+            case RegularizationType.ELASTIC_NET => 2
+            case RegularizationType.L2 => 1
+            case RegularizationType.L1 => 1
+            case RegularizationType.NONE => 0
+          }
+        case _ =>
+            throw new IllegalArgumentException(s"Unknown optimization config!")
+      }
+      .sum
+    val mode = getOrDefault(hyperParameterTuning)
 
-        val observations = evaluationFunction.convertObservations(models)
+    val evaluator = evaluationResults.get.primaryEvaluator
+    val isOptMax = evaluator.betterThan(1.0, 0.0)
+    val evaluationFunction = new GameEstimatorEvaluationFunction(
+      estimator,
+      baseConfig,
+      trainingData,
+      validationData.get,
+      isOptMax)
 
-        val hyperparameterTuner = HyperparameterTunerFactory[GameEstimator.GameResult](getOrDefault(hyperParameterTunerName))
+    val observations = evaluationFunction.convertObservations(models)
 
-        hyperparameterTuner.search(iteration, dimension, mode, evaluationFunction, observations)
+    val hyperparameterTuner = HyperparameterTunerFactory(getOrDefault(hyperParameterTunerName))
 
-      case _ => Seq()
-    }
+    hyperparameterTuner.search(iteration, dimension, mode, evaluationFunction, observations)
+  }
 
   /**
    * Select which models will be output to HDFS.
diff --git a/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/GameHyperparameterDefaults.scala b/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/GameHyperparameterDefaults.scala
deleted file mode 100644
index 98e69397..00000000
--- a/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/GameHyperparameterDefaults.scala
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2018 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter
-
-/**
- * Hyper-parameter default values
- */
-object GameHyperparameterDefaults {
-
-  val priorDefault: Map[String, String] = Map(
-    "global_regularizer" -> "0.0",
-    "member_regularizer" -> "0.0",
-    "item_regularizer" -> "0.0")
-
-  val configDefault: String =
-    """
-      |{ "tuning_mode" : "BAYESIAN",
-      |  "variables" : {
-      |    "global_regularizer" : {
-      |      "type" : "FLOAT",
-      |      "transform" : "LOG",
-      |      "min" : -3,
-      |      "max" : 3
-      |    },
-      |    "member_regularizer" : {
-      |      "type" : "FLOAT",
-      |      "transform" : "LOG",
-      |      "min" : -3,
-      |      "max" : 3
-      |    },
-      |    "item_regularizer" : {
-      |      "type" : "FLOAT",
-      |      "transform" : "LOG",
-      |      "min" : -3,
-      |      "max" : 3
-      |    }
-      |  }
-      |}
-    """.stripMargin
-}
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/HyperparameterTuningMode.scala b/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/HyperparameterTuningMode.scala
similarity index 88%
rename from photon-lib/src/main/scala/com/linkedin/photon/ml/HyperparameterTuningMode.scala
rename to photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/HyperparameterTuningMode.scala
index adc08dd1..85cfb191 100644
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/HyperparameterTuningMode.scala
+++ b/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/HyperparameterTuningMode.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
+ * Copyright 2020 LinkedIn Corp. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License"); you may
  * not use this file except in compliance with the License. You may obtain a
  * copy of the License at
@@ -12,12 +12,14 @@
  * License for the specific language governing permissions and limitations
  * under the License.
  */
-package com.linkedin.photon.ml
+package com.linkedin.photon.ml.hyperparameter
 
 /**
  * Supported options for hyperparameter tuning mode
  */
 object HyperparameterTuningMode extends Enumeration {
+
   type HyperparameterTuningMode = Value
+
   val BAYESIAN, RANDOM, NONE = Value
 }
diff --git a/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/ShrinkSearchRange.scala b/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/ShrinkSearchRange.scala
deleted file mode 100644
index 07d044fd..00000000
--- a/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/ShrinkSearchRange.scala
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter
-
-import scala.math.{floor, max, min}
-
-import breeze.linalg.{DenseMatrix, DenseVector}
-import org.apache.commons.math3.random.SobolSequenceGenerator
-
-import com.linkedin.photon.ml.hyperparameter.estimators.GaussianProcessEstimator
-import com.linkedin.photon.ml.hyperparameter.estimators.kernels.Matern52
-
-/**
- * An object to shrink search range given prior data.
- */
-object ShrinkSearchRange {
-
-  /**
-   * Compute the lower bound and upper bound of the new prior config
-   *
-   * @param hyperParams Configurations of hyper-parameters
-   * @param priorJsonString JSON string containing prior observations
-   * @param priorDefault Default values for missing hyper-parameters
-   * @param radius The radius of search range after transformation to [0, 1]
-   * @return A tuple of lower bounds and upper bounds for each hyperparameter. Lower bounds and upper bounds are
-   *         discrete for discrete parameters. Lower bounds and upper bounds are dense vectors.
-   */
-  def getBounds(
-      hyperParams: HyperparameterConfig,
-      priorJsonString: String,
-      priorDefault: Map[String, String],
-      radius: Double,
-      candidatePoolSize: Int = 1000,
-      seed: Long = System.currentTimeMillis): (DenseVector[Double], DenseVector[Double]) = {
-
-    val hyperparameterList = hyperParams.names
-    val ranges = hyperParams.ranges
-    val discreteParams = hyperParams.discreteParams
-    val numParams = ranges.length
-
-    // Get a [[Seq]] of (vectorized hyper-parameters, evaluationValue) tuples
-    val hyperparameterPairs = HyperparameterSerialization.priorFromJson(priorJsonString, priorDefault, hyperparameterList)
-
-    // Rescale the hyperparameters to [0,1]
-    val hyperparameterRescaled = VectorRescaling.rescalePriors(hyperparameterPairs, hyperParams)
-
-    // Combine hyperparameters as a dense matrix and evaluation value as a dense vector
-    val (overallPoints, overallEvals) = hyperparameterRescaled.map(x => (x._1.asDenseMatrix, DenseVector(x._2))).reduce(
-      (a, b) => (DenseMatrix.vertcat(a._1, b._1), DenseVector.vertcat(a._2, b._2))
-    )
-
-    // Fit Gaussian process regression model
-    val estimator = new GaussianProcessEstimator(kernel = new Matern52)
-    val model = estimator.fit(overallPoints, overallEvals)
-
-    // Sobol generator
-    val paramDistributions = {
-      val sobol = new SobolSequenceGenerator(numParams)
-      sobol.skipTo((seed % (Int.MaxValue.toLong + 1)).toInt)
-      sobol
-    }
-
-    // Draw candidates from a Sobol generator
-    val candidates = (1 until candidatePoolSize).foldLeft(DenseMatrix(paramDistributions.nextVector)) { case (acc, _) =>
-      DenseMatrix.vertcat(acc, DenseMatrix(paramDistributions.nextVector))
-    }
-
-    // Select the best candidate
-    val predictions = model.predict(candidates)
-    val bestCandidate = selectBestCandidate(candidates, predictions._1)
-
-    // compute lower bound and upper bound
-    val upperBound = VectorRescaling.scaleBackward(
-      discretizeCandidate(bestCandidate.map(x => x + radius), discreteParams),
-      ranges,
-      discreteParams.keySet)
-    val lowerBound = VectorRescaling.scaleBackward(
-      discretizeCandidate(bestCandidate.map(x => x - radius), discreteParams),
-      ranges,
-      discreteParams.keySet)
-
-    (0 until numParams).foreach{
-      index =>
-        upperBound(index) = min(upperBound(index), ranges(index).end)
-        lowerBound(index) = max(lowerBound(index), ranges(index).start)
-    }
-
-    (lowerBound, upperBound)
-  }
-
-  /**
-   * Selects the best candidate according to the predicted values, where "best" is defined as the largest
-   *
-   * @param candidates matrix of candidates
-   * @param predictions predicted values for each candidate
-   * @return the candidate with the best value
-   */
-  private def selectBestCandidate(
-      candidates: DenseMatrix[Double],
-      predictions: DenseVector[Double]): DenseVector[Double] = {
-
-    val init = (candidates(0,::).t, predictions(0))
-
-    val (selectedCandidate, _) = (1 until candidates.rows).foldLeft(init) {
-      case ((bestCandidate, bestPrediction), i) =>
-        if (predictions(i) > bestPrediction) {
-          (candidates(i,::).t, predictions(i))
-        } else {
-          (bestCandidate, bestPrediction)
-        }
-    }
-
-    selectedCandidate
-  }
-
-  /**
-   * Discretize candidates with specified indices.
-   *
-   * @param candidate candidate with values in [0, 1]
-   * @param discreteParams Map that specifies the indices of discrete parameters and their numbers of discrete values
-   * @return candidate with the specified discrete values
-   */
-  private def discretizeCandidate(
-      candidate: DenseVector[Double],
-      discreteParams: Map[Int, Int]): DenseVector[Double] = {
-
-    val candidateWithDiscrete = candidate.copy
-
-    discreteParams.foreach { case (index, numDiscreteValues) =>
-      candidateWithDiscrete(index) = floor(candidate(index) * numDiscreteValues) / numDiscreteValues
-    }
-
-    candidateWithDiscrete
-  }
-}
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/VectorRescaling.scala b/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/VectorRescaling.scala
similarity index 52%
rename from photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/VectorRescaling.scala
rename to photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/VectorRescaling.scala
index 21ccfcfc..3af95dce 100644
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/VectorRescaling.scala
+++ b/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/VectorRescaling.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright 2018 LinkedIn Corp. All rights reserved.
+ * Copyright 2020 LinkedIn Corp. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License"); you may
  * not use this file except in compliance with the License. You may obtain a
  * copy of the License at
@@ -26,54 +26,6 @@ object VectorRescaling {
   val LOG_TRANSFORM = "LOG"
   val SQRT_TRANSFORM = "SQRT"
 
-  /**
-   * Apply forward transformation to a subset of elements in a vector.
-   *
-   * @param vector A DenseVector.
-   * @param transformMap A Map with key-value pairs of indices and names of transform functions.
-   * @return The transformed vector.
-   */
-  def transformForward(
-      vector: DenseVector[Double],
-      transformMap: Map[Int, String]): DenseVector[Double] = {
-
-    val vectorTransformed = vector.copy
-
-    transformMap.foreach { case (index, transform) =>
-      transform match {
-        case LOG_TRANSFORM => vectorTransformed(index) = Math.log10(vectorTransformed(index))
-        case SQRT_TRANSFORM => vectorTransformed(index) = Math.sqrt(vectorTransformed(index))
-        case other => throw new IllegalArgumentException(s"Unknown transformation: $other")
-      }
-    }
-
-    vectorTransformed
-  }
-
-  /**
-   * Apply backward transformation to a subset of elements in a vector.
-   *
-   * @param vector A DenseVector.
-   * @param transformMap A Map with key-value pairs of indices and names of transform functions.
-   * @return The transformed vector.
-   */
-  def transformBackward(
-      vector: DenseVector[Double],
-      transformMap: Map[Int, String]): DenseVector[Double] = {
-
-    val vectorTransformed = vector.copy
-
-    transformMap.foreach { case (index, transform) =>
-      transform match {
-        case LOG_TRANSFORM => vectorTransformed(index) = Math.pow(10, vectorTransformed(index))
-        case SQRT_TRANSFORM => vectorTransformed(index) = Math.pow(vectorTransformed(index), 2)
-        case other => throw new IllegalArgumentException(s"Unknown transformation: $other")
-      }
-    }
-
-    vectorTransformed
-  }
-
   /**
    * Apply forward scaling to a vector. Given a range [a, b] and an element x of the vector,
    * y = (x - a) / (b - a), if x is continuous;
@@ -131,20 +83,4 @@ object VectorRescaling {
 
     vectorScaled
   }
-
-  /**
-   * This function applies forward transformation and scaling on prior data.
-   *
-   * @param priors A sequence of observations (vector, eval) from previous iterations or past dataset.
-   * @param hyperParams Hyper-parameter configuration.
-   * @return Obervations with vectors transformed and scaled forward.
-   */
-  def rescalePriors(priors: Seq[(DenseVector[Double], Double)], hyperParams: HyperparameterConfig): Seq[(DenseVector[Double], Double)] =
-
-    priors.map { case (candidate, eval) =>
-      val candidateTransformed = VectorRescaling.transformForward(candidate, hyperParams.transformMap)
-      val candidateScaled = VectorRescaling.scaleForward(candidateTransformed, hyperParams.ranges, hyperParams.discreteParams.keySet)
-
-      (candidateScaled, eval)
-    }
 }
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/EvaluationFunction.scala b/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/evaluation/EvaluationFunction.scala
similarity index 94%
rename from photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/EvaluationFunction.scala
rename to photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/evaluation/EvaluationFunction.scala
index af687389..09bb8ea1 100644
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/EvaluationFunction.scala
+++ b/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/evaluation/EvaluationFunction.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
+ * Copyright 2020 LinkedIn Corp. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License"); you may
  * not use this file except in compliance with the License. You may obtain a
  * copy of the License at
@@ -12,7 +12,7 @@
  * License for the specific language governing permissions and limitations
  * under the License.
  */
-package com.linkedin.photon.ml.hyperparameter
+package com.linkedin.photon.ml.hyperparameter.evaluation
 
 import breeze.linalg.DenseVector
 
diff --git a/photon-client/src/main/scala/com/linkedin/photon/ml/estimators/GameEstimatorEvaluationFunction.scala b/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/evaluation/GameEstimatorEvaluationFunction.scala
similarity index 97%
rename from photon-client/src/main/scala/com/linkedin/photon/ml/estimators/GameEstimatorEvaluationFunction.scala
rename to photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/evaluation/GameEstimatorEvaluationFunction.scala
index c74c0735..e9a616dc 100644
--- a/photon-client/src/main/scala/com/linkedin/photon/ml/estimators/GameEstimatorEvaluationFunction.scala
+++ b/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/evaluation/GameEstimatorEvaluationFunction.scala
@@ -12,7 +12,7 @@
  * License for the specific language governing permissions and limitations
  * under the License.
  */
-package com.linkedin.photon.ml.estimators
+package com.linkedin.photon.ml.hyperparameter.evaluation
 
 import scala.collection.mutable
 import scala.math.{exp, log}
@@ -20,10 +20,11 @@ import scala.math.{exp, log}
 import breeze.linalg.DenseVector
 import org.apache.spark.sql.DataFrame
 
+import com.linkedin.photon.ml.estimators.GameEstimator
 import com.linkedin.photon.ml.estimators.GameEstimator.{GameOptimizationConfiguration, GameResult}
-import com.linkedin.photon.ml.hyperparameter.{EvaluationFunction, VectorRescaling}
-import com.linkedin.photon.ml.optimization.{ElasticNetRegularizationContext, RegularizationContext, RegularizationType}
+import com.linkedin.photon.ml.hyperparameter.VectorRescaling
 import com.linkedin.photon.ml.optimization.game._
+import com.linkedin.photon.ml.optimization.{ElasticNetRegularizationContext, RegularizationContext, RegularizationType}
 import com.linkedin.photon.ml.util.DoubleRange
 
 /**
@@ -51,7 +52,7 @@ class GameEstimatorEvaluationFunction(
   private val baseConfigSeq = baseConfig.toSeq.sortBy(_._1)
 
   // Pull the hyperparameter ranges from the optimization configuration
-  protected[estimators] val ranges: Seq[DoubleRange] = baseConfigSeq
+  protected[evaluation] val ranges: Seq[DoubleRange] = baseConfigSeq
     .flatMap {
       case (_, config: GLMOptimizationConfiguration) =>
         val regularizationWeightRange = config
diff --git a/photon-api/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/HyperparameterTuner.scala b/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/HyperparameterTuner.scala
similarity index 88%
rename from photon-api/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/HyperparameterTuner.scala
rename to photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/HyperparameterTuner.scala
index aec25063..86895781 100644
--- a/photon-api/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/HyperparameterTuner.scala
+++ b/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/HyperparameterTuner.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright 2018 LinkedIn Corp. All rights reserved.
+ * Copyright 2020 LinkedIn Corp. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License"); you may
  * not use this file except in compliance with the License. You may obtain a
  * copy of the License at
@@ -16,8 +16,8 @@ package com.linkedin.photon.ml.hyperparameter.tuner
 
 import breeze.linalg.DenseVector
 
-import com.linkedin.photon.ml.HyperparameterTuningMode.HyperparameterTuningMode
-import com.linkedin.photon.ml.hyperparameter.EvaluationFunction
+import com.linkedin.photon.ml.hyperparameter.HyperparameterTuningMode.HyperparameterTuningMode
+import com.linkedin.photon.ml.hyperparameter.evaluation.EvaluationFunction
 
 /**
  * Interface for hyper-parameter tuner.
diff --git a/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/HyperparameterTunerFactory.scala b/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/HyperparameterTunerFactory.scala
new file mode 100644
index 00000000..11934e5e
--- /dev/null
+++ b/photon-client/src/main/scala/com/linkedin/photon/ml/hyperparameter/tuner/HyperparameterTunerFactory.scala
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2020 LinkedIn Corp. All rights reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package com.linkedin.photon.ml.hyperparameter.tuner
+
+import com.linkedin.photon.ml.estimators.GameEstimator
+
+object HyperparameterTunerFactory {
+
+  /**
+   * Factory for different [[HyperparameterTuner]] objects.
+   *
+   * @param className Name of [[HyperparameterTuner]] class to instantiate
+   * @return The hyper-parameter tuner
+   * @throws ClassNotFoundException
+   * @throws InstantiationException
+   * @throws IllegalAccessException
+   */
+  def apply(className: String): HyperparameterTuner[GameEstimator.GameResult] =
+    Class.forName(className)
+      .newInstance
+      .asInstanceOf[HyperparameterTuner[GameEstimator.GameResult]]
+}
diff --git a/photon-client/src/main/scala/com/linkedin/photon/ml/io/scopt/ScoptParserReads.scala b/photon-client/src/main/scala/com/linkedin/photon/ml/io/scopt/ScoptParserReads.scala
index b9c773f2..9bbaef92 100644
--- a/photon-client/src/main/scala/com/linkedin/photon/ml/io/scopt/ScoptParserReads.scala
+++ b/photon-client/src/main/scala/com/linkedin/photon/ml/io/scopt/ScoptParserReads.scala
@@ -17,12 +17,12 @@ package com.linkedin.photon.ml.io.scopt
 import org.apache.hadoop.fs.Path
 import org.joda.time.DateTimeZone
 
-import com.linkedin.photon.ml.{DataValidationType, HyperparameterTunerName, HyperparameterTuningMode, TaskType}
+import com.linkedin.photon.ml.{DataValidationType, TaskType}
 import com.linkedin.photon.ml.DataValidationType.DataValidationType
-import com.linkedin.photon.ml.HyperparameterTunerName.HyperparameterTunerName
-import com.linkedin.photon.ml.HyperparameterTuningMode.HyperparameterTuningMode
+import com.linkedin.photon.ml.hyperparameter.HyperparameterTuningMode.HyperparameterTuningMode
 import com.linkedin.photon.ml.TaskType.TaskType
 import com.linkedin.photon.ml.evaluation.EvaluatorType
+import com.linkedin.photon.ml.hyperparameter.HyperparameterTuningMode
 import com.linkedin.photon.ml.io.ModelOutputMode
 import com.linkedin.photon.ml.io.ModelOutputMode.ModelOutputMode
 import com.linkedin.photon.ml.normalization.NormalizationType
@@ -40,8 +40,6 @@ object ScoptParserReads {
   implicit val dateRangeRead: scopt.Read[DateRange] = scopt.Read.reads(DateRange.fromDateString)
   implicit val daysRangeRead: scopt.Read[DaysRange] = scopt.Read.reads(DaysRange.fromDaysString)
   implicit val evaluatorTypeRead: scopt.Read[EvaluatorType] = scopt.Read.reads(Utils.evaluatorWithName)
-  implicit val hyperParameterTunerNameRead: scopt.Read[HyperparameterTunerName] =
-    scopt.Read.reads(HyperparameterTunerName.withName)
   implicit val hyperParameterTuningModeRead: scopt.Read[HyperparameterTuningMode] =
     scopt.Read.reads(HyperparameterTuningMode.withName)
   implicit val modelOutputModeRead: scopt.Read[ModelOutputMode] = scopt.Read.reads(ModelOutputMode.withName)
diff --git a/photon-client/src/main/scala/com/linkedin/photon/ml/io/scopt/game/ScoptGameTrainingParametersParser.scala b/photon-client/src/main/scala/com/linkedin/photon/ml/io/scopt/game/ScoptGameTrainingParametersParser.scala
index 54606cb6..e2531c63 100644
--- a/photon-client/src/main/scala/com/linkedin/photon/ml/io/scopt/game/ScoptGameTrainingParametersParser.scala
+++ b/photon-client/src/main/scala/com/linkedin/photon/ml/io/scopt/game/ScoptGameTrainingParametersParser.scala
@@ -20,8 +20,7 @@ import org.apache.hadoop.fs.Path
 import org.apache.spark.ml.param.ParamMap
 import scopt.{OptionDef, OptionParser, Read}
 
-import com.linkedin.photon.ml.HyperparameterTunerName.HyperparameterTunerName
-import com.linkedin.photon.ml.HyperparameterTuningMode.HyperparameterTuningMode
+import com.linkedin.photon.ml.hyperparameter.HyperparameterTuningMode.HyperparameterTuningMode
 import com.linkedin.photon.ml.TaskType.TaskType
 import com.linkedin.photon.ml.Types.CoordinateId
 import com.linkedin.photon.ml.cli.game.training.GameTrainingDriver
@@ -33,7 +32,8 @@ import com.linkedin.photon.ml.normalization.NormalizationType.NormalizationType
 import com.linkedin.photon.ml.optimization.VarianceComputationType
 import com.linkedin.photon.ml.optimization.VarianceComputationType.VarianceComputationType
 import com.linkedin.photon.ml.util.{DateRange, DaysRange}
-import com.linkedin.photon.ml.{HyperparameterTuningMode, TaskType}
+import com.linkedin.photon.ml.TaskType
+import com.linkedin.photon.ml.hyperparameter.HyperparameterTuningMode
 
 /**
  * Scopt command line argument parser for GAME training parameters.
@@ -138,9 +138,9 @@ object ScoptGameTrainingParametersParser extends ScoptGameParametersParser {
         usageText = "<value>"),
 
       // Hyper Parameter Tuner Name
-      ScoptParameter[HyperparameterTunerName, HyperparameterTunerName](
+      ScoptParameter[String, String](
         GameTrainingDriver.hyperParameterTunerName,
-        usageText = "<type>"),
+        usageText = "<class path>"),
 
       // Hyper Parameter Tuning
       ScoptParameter[HyperparameterTuningMode, HyperparameterTuningMode](
diff --git a/photon-client/src/test/scala/com/linkedin/photon/ml/cli/game/training/GameTrainingDriverTest.scala b/photon-client/src/test/scala/com/linkedin/photon/ml/cli/game/training/GameTrainingDriverTest.scala
index 13fe4ed6..cbef48d6 100644
--- a/photon-client/src/test/scala/com/linkedin/photon/ml/cli/game/training/GameTrainingDriverTest.scala
+++ b/photon-client/src/test/scala/com/linkedin/photon/ml/cli/game/training/GameTrainingDriverTest.scala
@@ -21,10 +21,11 @@ import org.mockito.Mockito._
 import org.testng.Assert._
 import org.testng.annotations.{DataProvider, Test}
 
-import com.linkedin.photon.ml.{DataValidationType, HyperparameterTunerName, HyperparameterTuningMode, TaskType}
+import com.linkedin.photon.ml.{DataValidationType, TaskType}
 import com.linkedin.photon.ml.data.{CoordinateDataConfiguration, InputColumnsNames, RandomEffectDataConfiguration}
 import com.linkedin.photon.ml.estimators.GameEstimator
 import com.linkedin.photon.ml.evaluation.{EvaluationResults, EvaluatorType}
+import com.linkedin.photon.ml.hyperparameter.HyperparameterTuningMode
 import com.linkedin.photon.ml.io.{CoordinateConfiguration, FeatureShardConfiguration, ModelOutputMode, RandomEffectCoordinateConfiguration}
 import com.linkedin.photon.ml.io.ModelOutputMode.ModelOutputMode
 import com.linkedin.photon.ml.model.GameModel
@@ -123,7 +124,7 @@ class GameTrainingDriverTest {
       .put(GameTrainingDriver.normalization, NormalizationType.STANDARDIZATION)
       .put(GameTrainingDriver.dataSummaryDirectory, mockPath)
       .put(GameTrainingDriver.treeAggregateDepth, mockInt)
-      .put(GameTrainingDriver.hyperParameterTunerName, HyperparameterTunerName.DUMMY)
+      .put(GameTrainingDriver.hyperParameterTunerName, mockString)
       .put(GameTrainingDriver.hyperParameterTuning, HyperparameterTuningMode.BAYESIAN)
       .put(GameTrainingDriver.hyperParameterTuningIter, mockInt)
       .put(GameTrainingDriver.varianceComputationType, mockVarianceComputationType)
@@ -238,8 +239,21 @@ class GameTrainingDriverTest {
           .put(GameTrainingDriver.coordinateConfigurations, Map((coordinateId1, mockRECoordinateConfig)))),
       // No intercepts for standardization
       Array(validParamMap.copy.put(GameTrainingDriver.normalization, NormalizationType.STANDARDIZATION)),
+      // No tuner for hyperparameter tuning
+      Array(validParamMap.copy.put(GameTrainingDriver.hyperParameterTuning, HyperparameterTuningMode.BAYESIAN)),
       // No iterations for hyperparameter tuning
-      Array(validParamMap.copy.put(GameTrainingDriver.hyperParameterTuning, HyperparameterTuningMode.BAYESIAN)))
+      Array(
+        validParamMap
+          .copy
+          .put(GameTrainingDriver.hyperParameterTuning, HyperparameterTuningMode.BAYESIAN)
+          .put(GameTrainingDriver.hyperParameterTunerName, "some.fake.tuner")),
+      // No validation data for hyperparameter tuning
+      Array(
+        validParamMap
+          .copy
+          .put(GameTrainingDriver.hyperParameterTuning, HyperparameterTuningMode.BAYESIAN)
+          .put(GameTrainingDriver.hyperParameterTunerName, "some.fake.tuner")
+          .put(GameTrainingDriver.hyperParameterTuningIter, 1)))
   }
 
   /**
@@ -263,7 +277,6 @@ class GameTrainingDriverTest {
     GameTrainingDriver.getOrDefault(GameTrainingDriver.outputMode)
     GameTrainingDriver.getOrDefault(GameTrainingDriver.overrideOutputDirectory)
     GameTrainingDriver.getOrDefault(GameTrainingDriver.normalization)
-    GameTrainingDriver.getOrDefault(GameTrainingDriver.hyperParameterTunerName)
     GameTrainingDriver.getOrDefault(GameTrainingDriver.hyperParameterTuning)
     GameTrainingDriver.getOrDefault(GameTrainingDriver.varianceComputationType)
     GameTrainingDriver.getOrDefault(GameTrainingDriver.dataValidation)
diff --git a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/VectorRescalingTest.scala b/photon-client/src/test/scala/com/linkedin/photon/ml/hyperparameter/VectorRescalingTest.scala
similarity index 52%
rename from photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/VectorRescalingTest.scala
rename to photon-client/src/test/scala/com/linkedin/photon/ml/hyperparameter/VectorRescalingTest.scala
index 6db48669..ee603444 100644
--- a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/VectorRescalingTest.scala
+++ b/photon-client/src/test/scala/com/linkedin/photon/ml/hyperparameter/VectorRescalingTest.scala
@@ -18,7 +18,6 @@ import breeze.linalg.DenseVector
 import org.testng.Assert.assertEquals
 import org.testng.annotations.Test
 
-import com.linkedin.photon.ml.HyperparameterTuningMode
 import com.linkedin.photon.ml.util.DoubleRange
 
 /**
@@ -26,30 +25,6 @@ import com.linkedin.photon.ml.util.DoubleRange
  */
 class VectorRescalingTest {
 
-  /**
-   * Unit test for VectorRescaling.transformForward
-   */
-  @Test
-  def testTransformForward(): Unit = {
-    val vector = DenseVector(1000, 0.001, 8, 4)
-    val transformMap = Map(0 -> "LOG", 1 -> "LOG", 3-> "SQRT")
-    val vectorTransformed = VectorRescaling.transformForward(vector, transformMap)
-    val expectedData = DenseVector(3.0, -3.0, 8.0, 2.0)
-    assertEquals(vectorTransformed, expectedData)
-  }
-
-  /**
-   * Unit test for VectorRescaling.transformBackward
-   */
-  @Test
-  def testTransformBackward(): Unit = {
-    val vector = DenseVector(3.0, -3.0, 8.0, 2.0)
-    val transformMap = Map(0 -> "LOG", 1 -> "LOG", 3-> "SQRT")
-    val vectorTransformed = VectorRescaling.transformBackward(vector, transformMap)
-    val expectedData = DenseVector(1000, 0.001, 8, 4)
-    assertEquals(vectorTransformed, expectedData)
-  }
-
   /**
    * Unit test for VectorRescaling.scaleForward
    */
@@ -77,26 +52,4 @@ class VectorRescalingTest {
     val expectedData = DenseVector(5, 0.5, -1.0, 10.23)
     assertEquals(vectorScaled, expectedData)
   }
-
-  /**
-   * Unit test for VectorRescaling.rescalePriors
-   */
-  @Test
-  def testRescalePriors(): Unit = {
-
-    val tuningMode = HyperparameterTuningMode.BAYESIAN
-    val hyperparameters = Seq("alpha", "beta", "gamma", "lambda")
-    val ranges = Seq(DoubleRange(0, 4), DoubleRange(0, 4), DoubleRange(-2, 2), DoubleRange(-2, 2))
-    val discreteParam = Map(0 -> 8)
-    val transformMap = Map(0 -> "LOG", 1 -> "LOG", 3-> "SQRT")
-
-    val hyperParams = HyperparameterConfig(tuningMode, hyperparameters, ranges, discreteParam, transformMap)
-
-    val priors = Seq((DenseVector(1000.0, 1000.0, 8.0, 4.0), 0.1))
-
-    val priorsRescaled = VectorRescaling.rescalePriors(priors, hyperParams)
-    val expectedData = Seq((DenseVector(0.6, 0.75, 2.5, 1), 0.1))
-
-    assertEquals(priorsRescaled, expectedData)
-  }
 }
diff --git a/photon-client/src/test/scala/com/linkedin/photon/ml/estimators/GameEstimatorEvaluationFunctionTest.scala b/photon-client/src/test/scala/com/linkedin/photon/ml/hyperparameter/evaluation/GameEstimatorEvaluationFunctionTest.scala
similarity index 98%
rename from photon-client/src/test/scala/com/linkedin/photon/ml/estimators/GameEstimatorEvaluationFunctionTest.scala
rename to photon-client/src/test/scala/com/linkedin/photon/ml/hyperparameter/evaluation/GameEstimatorEvaluationFunctionTest.scala
index da3638b9..39ca0fc0 100644
--- a/photon-client/src/test/scala/com/linkedin/photon/ml/estimators/GameEstimatorEvaluationFunctionTest.scala
+++ b/photon-client/src/test/scala/com/linkedin/photon/ml/hyperparameter/evaluation/GameEstimatorEvaluationFunctionTest.scala
@@ -12,12 +12,12 @@
  * License for the specific language governing permissions and limitations
  * under the License.
  */
-package com.linkedin.photon.ml.estimators
-
-import scala.math.log
+package com.linkedin.photon.ml.hyperparameter.evaluation
 
 import java.util.Random
 
+import scala.math.log
+
 import breeze.linalg.DenseVector
 import org.apache.spark.sql.DataFrame
 import org.mockito.Mockito._
@@ -25,6 +25,7 @@ import org.testng.Assert._
 import org.testng.annotations.{DataProvider, Test}
 
 import com.linkedin.photon.ml.constants.MathConst
+import com.linkedin.photon.ml.estimators.GameEstimator
 import com.linkedin.photon.ml.estimators.GameEstimator.GameOptimizationConfiguration
 import com.linkedin.photon.ml.optimization._
 import com.linkedin.photon.ml.optimization.game._
diff --git a/photon-client/src/test/scala/com/linkedin/photon/ml/io/scopt/game/ScoptGameTrainingParametersParserTest.scala b/photon-client/src/test/scala/com/linkedin/photon/ml/io/scopt/game/ScoptGameTrainingParametersParserTest.scala
index 95be9acb..c0e3f886 100644
--- a/photon-client/src/test/scala/com/linkedin/photon/ml/io/scopt/game/ScoptGameTrainingParametersParserTest.scala
+++ b/photon-client/src/test/scala/com/linkedin/photon/ml/io/scopt/game/ScoptGameTrainingParametersParserTest.scala
@@ -22,12 +22,13 @@ import org.testng.annotations.Test
 import com.linkedin.photon.ml.cli.game.training.GameTrainingDriver
 import com.linkedin.photon.ml.data.{FixedEffectDataConfiguration, InputColumnsNames, RandomEffectDataConfiguration}
 import com.linkedin.photon.ml.evaluation.EvaluatorType.{AUC, RMSE}
+import com.linkedin.photon.ml.hyperparameter.HyperparameterTuningMode
 import com.linkedin.photon.ml.io.{FeatureShardConfiguration, FixedEffectCoordinateConfiguration, ModelOutputMode, RandomEffectCoordinateConfiguration}
 import com.linkedin.photon.ml.normalization.NormalizationType
 import com.linkedin.photon.ml.optimization.game.{FixedEffectOptimizationConfiguration, RandomEffectOptimizationConfiguration}
 import com.linkedin.photon.ml.optimization._
 import com.linkedin.photon.ml.util.{DateRange, DoubleRange, PhotonLogger}
-import com.linkedin.photon.ml.{DataValidationType, HyperparameterTunerName, HyperparameterTuningMode, TaskType}
+import com.linkedin.photon.ml.{DataValidationType, TaskType}
 
 /**
  * Unit tests for the [[ScoptGameTrainingParametersParser]].
@@ -62,7 +63,7 @@ class ScoptGameTrainingParametersParserTest {
     val normalization = NormalizationType.SCALE_WITH_MAX_MAGNITUDE
     val dataSummaryPath = new Path("/some/summary/path")
     val treeAggregateDepth = 5
-    val hyperparameterTunerName = HyperparameterTunerName.DUMMY
+    val hyperparameterTunerName = "some.fake.class"
     val hyperparameterTuningMode = HyperparameterTuningMode.BAYESIAN
     val hyperparameterTuningIter = 6
     val varianceComputation = VarianceComputationType.SIMPLE
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/HyperparameterTunerName.scala b/photon-lib/src/main/scala/com/linkedin/photon/ml/HyperparameterTunerName.scala
deleted file mode 100644
index 4ae9519c..00000000
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/HyperparameterTunerName.scala
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2018 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml
-
-/**
- * Supported options for hyperparameter tuner name
- */
-object HyperparameterTunerName extends Enumeration {
-  type HyperparameterTunerName = Value
-  val DUMMY, ATLAS = Value
-}
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/HyperparameterConfig.scala b/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/HyperparameterConfig.scala
deleted file mode 100644
index 5047d0d1..00000000
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/HyperparameterConfig.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 2018 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter
-
-import com.linkedin.photon.ml.HyperparameterTuningMode.HyperparameterTuningMode
-import com.linkedin.photon.ml.util.DoubleRange
-
-/**
- * Configurations of hyper-parameters.
- *
- * @param tuningMode Hyper-parameter auto-tuning mode.
- * @param names A Seq of hyper-parameter names.
- * @param ranges A Seq of searching ranges for hyper-parameters.
- * @param discreteParams A Map that specifies the indices of discrete parameters and their numbers of discrete values.
- * @param transformMap A Map that specifies the indices of parameters and their names of transform functions.
- */
-case class HyperparameterConfig(
-    tuningMode: HyperparameterTuningMode,
-    names: Seq[String],
-    ranges: Seq[DoubleRange],
-    discreteParams: Map[Int, Int],
-    transformMap: Map[Int, String])
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/HyperparameterSerialization.scala b/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/HyperparameterSerialization.scala
deleted file mode 100644
index ab4d1269..00000000
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/HyperparameterSerialization.scala
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Copyright 2018 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter
-
-import breeze.linalg.DenseVector
-import com.linkedin.photon.ml.HyperparameterTuningMode
-import com.linkedin.photon.ml.util.DoubleRange
-
-import scala.util.parsing.json.JSON
-
-
-/**
- * An object to deserialize configuration and prior observations of hyper-parameters.
- */
-object HyperparameterSerialization {
-
-  val BAYESIAN_MODEL = "BAYESIAN"
-  val RANDOM_MODEL = "RANDOM"
-  val LOG_TRANSFORM = "LOG"
-  val SQRT_TRANSFORM = "SQRT"
-
-  /**
-   * Parse a [[Seq]] of prior observations for hyper-parameter tuning from JSON format.
-   *
-   * @param priorDataJson The JSON containing prior observations
-   * @param priorDefault Default values for missing hyper-parameters
-   * @param hyperParameterList The list of hyper-parameters to tune
-   * @return A [[Seq]] of (vectorized hyper-parameter settings, evaluationValue) tuples
-   */
-  def priorFromJson(
-      priorDataJson: String,
-      priorDefault: Map[String, String],
-      hyperParameterList: Seq[String]): Seq[(DenseVector[Double], Double)] = {
-
-    val priorData = JSON.parseFull(priorDataJson) match {
-      case Some(priorDataMap: Map[String, Any]) =>
-
-        priorDataMap("records") match {
-          case optionsList: Seq[Map[String, String]] =>
-
-            optionsList.map { paramMap =>
-              val evaluationValue = paramMap("evaluationValue").toDouble
-              val sortedValues = hyperParameterList.map { paramName =>
-                paramMap.getOrElse(paramName, priorDefault(paramName)).toDouble
-              }
-
-              (sortedValues, evaluationValue)
-            }
-
-          case _ =>
-            throw new IllegalArgumentException("Each record is not a list of Map[String, String]")
-        }
-
-      case _ =>
-        throw new IllegalArgumentException("The JSON file is not a Map")
-    }
-
-    priorData.map { case (params, evalValue) =>
-      (DenseVector(params.toArray), evalValue)
-    }
-  }
-
-  /**
-   * Read in the JSON config file and returns the parsed hyper-parameter configurations.
-   *
-   * @param jsonConfig The config file in json format.
-   * @return A tuple containing the hyperparameter tuning mode, list of parameters, ranges of min and max for each
-   *         parameter.
-   */
-  def configFromJson(jsonConfig: String): HyperparameterConfig = {
-
-    val (tuningMode, paramDetails) = JSON.parseFull(jsonConfig) match {
-      case Some(inputConfig: Map[String, Any]) =>
-
-        val mode = inputConfig("tuning_mode") match {
-          case BAYESIAN_MODEL => HyperparameterTuningMode.BAYESIAN
-          case RANDOM_MODEL => HyperparameterTuningMode.RANDOM
-          case _ => HyperparameterTuningMode.NONE
-        }
-
-        val hyperparameterDetails = inputConfig("variables") match {
-
-          case variables: Map[String, Any] =>
-
-            variables.map {
-              case (key: String, value: Map[String, Any]) =>
-
-                (value("type"), value("min"), value("max"), value.get("transform")) match {
-                  case (varType: String, min: Double, max: Double, transform: Option[_]) =>
-                    (key, varType, min, max, transform)
-
-                  case _ => throw new IllegalArgumentException("The minimum and maximum values must be numeric")
-                }
-
-              case _ =>
-                throw new IllegalArgumentException("Each hyper-parameter configuration must be a map")
-            }
-
-          case _ =>
-            throw new IllegalArgumentException("The hyper-parameter configurations must be a map")
-        }
-
-        (mode, hyperparameterDetails)
-
-      case _ =>
-        throw new IllegalArgumentException("JSON config is not a Map[String, Any]")
-    }
-
-    val hyperparameters = paramDetails.map(_._1).toSeq
-    val discreteParams = paramDetails.zipWithIndex.filter(_._1._2 == "INT").map {
-      case ((_, _, min: Double, max: Double, _), index: Int) => index -> ((max - min).toInt + 1)
-    }.toMap
-    val ranges = paramDetails.map { case (_, _, min, max, _) => DoubleRange(min, max) }.toSeq
-
-    val transformMap = paramDetails.map(_._5).zipWithIndex.flatMap {
-      case (transform, index) => transform.map(trans => trans.toString match {
-        case LOG_TRANSFORM | SQRT_TRANSFORM => index -> trans.toString
-        case _ => throw new IllegalArgumentException("The transformation is not valid")
-      })
-    }.toMap
-
-    HyperparameterConfig(tuningMode, hyperparameters, ranges, discreteParams, transformMap)
-  }
-}
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/SliceSampler.scala b/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/SliceSampler.scala
deleted file mode 100644
index 93482011..00000000
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/SliceSampler.scala
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter
-
-import scala.annotation.tailrec
-import scala.util.Random
-
-import breeze.linalg.{DenseVector, norm}
-import breeze.numerics.log
-import breeze.stats.distributions.Gaussian
-
-/**
- * This class implements the slice sampling algorithm. Slice sampling is an Markov chain Monte Carlo algorithm for
- * sampling from an arbitrary continuous distribution function.
- *
- * Consider the ASCII-normal distribution function shown below.
- *
- *
- *                  /-\                /-\ f(x)           /-\                /-\                /-\
- *                 /   \              /  |\              /  |\              /   \              /   \
- *                /     \            /   | \            /---|-\            /-x'--\            /     \
- *             --/    x  \--      --/    x  \--      --/    x  \--      --/       \--      --/  x'   \--
- *
- *                   1                  2                  3                  4                  5
- *
- * The procedure for drawing a sample from the function is as follows:
- *
- *   1) Start with an initial point x in the domain (e.g. the sample from a previous iteration).
- *   2) Sample a vertical value uniformly between zero and f(x).
- *   3) Step out horizontally in both directions from the value obtained in step 2 until reaching the boundary of the
-          function. This is the "slice".
- *   4) Sample uniformly from the slice to obtain the new point x'.
- *   5) Back to step 1 with the new point.
- *
- * @param stepSize the size to expand a slice while taking one step in a single direction from the
- *   starting point
- * @param maxStepsOut the maximum number of steps in either direction while expanding a slice
- * @param seed the random seed
- */
-class SliceSampler(
-    stepSize: Double = 1.0,
-    maxStepsOut: Int = 1000,
-    seed: Long = System.currentTimeMillis) {
-
-  val random = new Random(seed)
-
-  /**
-   * Draws a new point from the target distribution
-   *
-   * @param x the original point
-   * @param logp the log-transformed function from which to sample. Note that this need not sum to 1, and need only be
-   *   proportional to the target PDF.
-   * @return the new point
-   */
-  def draw(x: DenseVector[Double], logp: (DenseVector[Double]) => Double): DenseVector[Double] = {
-    val sampledVector = DenseVector(Gaussian(0, 1).sample(x.length):_*)
-    val direction = sampledVector / norm(sampledVector)
-
-    draw(x, logp, direction)
-  }
-
-  /**
-   * Draws a new point from the target distribution, sampling from each dimension one-by-one
-   *
-   * @param x the original point
-   * @param logp the log-transformed function from which to sample. Note that this need not sum to 1, and need only be
-   *   proportional to the target PDF.
-   * @return the new point
-   */
-  def drawDimensionWise(x: DenseVector[Double], logp: (DenseVector[Double]) => Double): DenseVector[Double] = {
-    val directions = random.shuffle((0 until x.length).toList)
-    directions.foldLeft(x) { (currX, i) =>
-      draw(currX, logp, getDirection(i, x.length))
-    }
-  }
-
-  /**
-   * Draws a new point from the target distribution along the given direction
-   *
-   * @param x the original point
-   * @param logp the log-transformed function from which to sample. Note that this need not sum to 1, and need only be
-   *   proportional to the target PDF.
-   * @param direction the direction along which to draw a new point
-   * @return the new point
-   */
-  protected def draw(
-      x: DenseVector[Double],
-      logp: (DenseVector[Double]) => Double,
-      direction: DenseVector[Double]): DenseVector[Double] = {
-
-    val y = log(random.nextDouble()) + logp(x)
-    val slice = stepOut(x, y, logp, direction)
-
-    draw(x, y, logp, direction, slice)
-  }
-
-  /**
-   * Draws a new point uniformly from the given horizontal slice through the function, along the
-   * given direction
-   *
-   * @param x the original point
-   * @param y the value of the function slice
-   * @param logp the log-transformed function from which to sample. Note that this need not sum to 1, and need only be
-   *   proportional to the target PDF.
-   * @param direction the direction along which to draw a new point
-   * @param slice the slice bounds
-   * @return the new point
-   */
-  @tailrec
-  protected final def draw(
-      x: DenseVector[Double],
-      y: Double,
-      logp: (DenseVector[Double]) => Double,
-      direction: DenseVector[Double],
-      slice: (DenseVector[Double], DenseVector[Double])): DenseVector[Double] = {
-
-    // Sample uniformly from the slice
-    val (lower, upper) = slice
-    val newX = lower + random.nextDouble() * (upper - lower)
-
-    if (logp(newX) > y) {
-      // If we've landed in at point of the PDF that's above the slice, return the point
-      newX
-
-    } else {
-      // Otherwise, reject the sample and shrink the slice
-      val newSlice = shrink(slice, x, newX, direction)
-      draw(x, y, logp, direction, newSlice)
-    }
-  }
-
-  /**
-   * Builds a direction basis vector from the index
-   *
-   * @param i the active index
-   * @param dim size of the vector
-   * @return the direction basis vector
-   */
-  protected def getDirection(i: Int, dim: Int) =
-    DenseVector.tabulate(dim) { j => if (i == j) 1.0 else 0.0 }
-
-  /**
-   * Performs the step out procedure. Start at the given x position, and expand outwards along the
-   * given direction until the slice extends beyond where the PDF lies above the y value.
-   *
-   * @param x the starting point
-   * @param y the value at which the function will be sliced
-   * @param logp the log-transformed function from which to sample. Note that this need not sum to 1, and need only be
-   *   proportional to the target PDF.
-   * @param direction the direction along which to slice
-   * @return the slice lower and upper bounds
-   */
-  protected def stepOut(
-      x: DenseVector[Double],
-      y: Double,
-      logp: (DenseVector[Double]) => Double,
-      direction: DenseVector[Double]): (DenseVector[Double], DenseVector[Double]) = {
-
-    var lower = x - direction * random.nextDouble() * stepSize
-    var upper = lower + direction * stepSize
-    var lowerStepsOut = 0
-    var upperStepsOut = 0
-
-    while ((logp(lower) > y) && lowerStepsOut < maxStepsOut) {
-      lower -= direction * stepSize
-      lowerStepsOut += 1
-    }
-
-    while ((logp(upper) > y) && upperStepsOut < maxStepsOut) {
-      upper += direction * stepSize
-      upperStepsOut += 1
-    }
-
-    (lower, upper)
-  }
-
-  /**
-   * Shrinks the slice by clamping to the new point x.
-   *
-   * @param slice the original slice
-   * @param x the original point x
-   * @param newX the new point x
-   * @param direction the direction along which to shrink the slice
-   * @return the shrunken slice
-   */
-  protected def shrink(
-      slice: (DenseVector[Double], DenseVector[Double]),
-      x: DenseVector[Double],
-      newX: DenseVector[Double],
-      direction: DenseVector[Double]): (DenseVector[Double], DenseVector[Double]) = {
-
-    val (lower, upper) = slice
-
-    if ((newX.t * direction) < (x.t * direction)) {
-      (newX, upper)
-
-    } else if ((newX.t * direction) > (x.t * direction)) {
-      (lower, newX)
-
-    } else {
-      throw new RuntimeException("Slice size shrank to zero.")
-    }
-  }
-}
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/criteria/ConfidenceBound.scala b/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/criteria/ConfidenceBound.scala
deleted file mode 100644
index c6fd9c21..00000000
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/criteria/ConfidenceBound.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.criteria
-
-import breeze.linalg.DenseVector
-import breeze.numerics.sqrt
-
-import com.linkedin.photon.ml.hyperparameter.estimators.PredictionTransformation
-
-/**
- * Confidence bound selection criterion. This transformation produces a lower confidence bound.
- *
- * @param explorationFactor a factor that determines the trade-off between exploration and exploitation during search:
- *   higher values favor exploration.
- */
-class ConfidenceBound(explorationFactor: Double = 2.0) extends PredictionTransformation {
-
-  // Minimize CB to minimize the evaluation value.
-  def isMaxOpt: Boolean = false
-
-  /**
-   * Applies the confidence bound transformation to the model output
-   *
-   * @param predictiveMeans predictive mean output from the model
-   * @param predictiveVariances predictive variance output from the model
-   * @return the lower confidence bounds
-   */
-  def apply(
-      predictiveMeans: DenseVector[Double],
-      predictiveVariances: DenseVector[Double]): DenseVector[Double] = {
-
-    // PBO Eq. 3
-    val confidenceBounds = explorationFactor * sqrt(predictiveVariances)
-    predictiveMeans - confidenceBounds
-  }
-}
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/criteria/ExpectedImprovement.scala b/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/criteria/ExpectedImprovement.scala
deleted file mode 100644
index 12e2bb2a..00000000
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/criteria/ExpectedImprovement.scala
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.criteria
-
-import breeze.linalg.DenseVector
-import breeze.numerics.sqrt
-import breeze.stats.distributions.Gaussian
-
-import com.linkedin.photon.ml.hyperparameter.estimators.PredictionTransformation
-
-/**
- * Expected improvement selection criterion. This transformation produces the expected improvement of the model
- * predictions (over the current "best" value).
- *
- * @see "Practical Bayesian Optimization of Machine Learning Algorithms" (PBO),
- *   https://papers.nips.cc/paper/4522-practical-bayesian-optimization-of-machine-learning-algorithms.pdf
- *
- * @param bestEvaluation The current best evaluation
- */
-class ExpectedImprovement(bestEvaluation: Double) extends PredictionTransformation {
-
-  // Maximize EI to minimize the evaluation value.
-  def isMaxOpt: Boolean = true
-
-  private val standardNormal = new Gaussian(0, 1)
-
-  /**
-   * Applies the expected improvement transformation to the model output.
-   *
-   * @param predictiveMeans Predictive mean output from the model
-   * @param predictiveVariances Predictive variance output from the model
-   * @return The expected improvement over the current best evaluation
-   */
-  def apply(
-      predictiveMeans: DenseVector[Double],
-      predictiveVariances: DenseVector[Double]): DenseVector[Double] = {
-
-    val std = sqrt(predictiveVariances)
-
-    // PBO Eq. 1
-    val gamma = - (predictiveMeans - bestEvaluation) / std
-
-    // Eq. 2
-    std :* ((gamma :* gamma.map(standardNormal.cdf)) + gamma.map(standardNormal.pdf))
-  }
-}
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/GaussianProcessEstimator.scala b/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/GaussianProcessEstimator.scala
deleted file mode 100644
index 62feb6c8..00000000
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/GaussianProcessEstimator.scala
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.estimators
-
-import breeze.linalg.{DenseMatrix, DenseVector}
-import breeze.stats.mean
-
-import com.linkedin.photon.ml.hyperparameter.SliceSampler
-import com.linkedin.photon.ml.hyperparameter.estimators.kernels._
-
-/**
- * Estimates a Gaussian Process regression model
- *
- * @see "Gaussian Processes for Machine Learning" (GPML), http://www.gaussianprocess.org/gpml/, Chapter 2
- *
- * @param kernel the covariance kernel
- * @param normalizeLabels if true, the estimator normalizes labels to a mean of zero before fitting
- * @param noisyTarget learn a target function with noise
- * @param predictionTransformation transformation function to apply for predictions
- * @param monteCarloNumBurnInSamples the number of samples to draw during the burn-in phase of kernel parameter
- *   estimation
- * @param monteCarloNumSamples the number of samples to draw for estimating kernel parameters
- */
-class GaussianProcessEstimator(
-    kernel: Kernel = new RBF,
-    normalizeLabels: Boolean = false,
-    noisyTarget: Boolean = false,
-    predictionTransformation: Option[PredictionTransformation] = None,
-    monteCarloNumBurnInSamples: Int = 100,
-    monteCarloNumSamples: Int = 10,
-    seed: Long = System.currentTimeMillis) {
-
-  val defaultNoise = 1e-4
-
-  /**
-   * Produces a Gaussian Process regression model from the input features and labels
-   *
-   * @param x the observed features
-   * @param y the observed labels
-   * @return the estimated model
-   */
-  def fit(x: DenseMatrix[Double], y: DenseVector[Double]): GaussianProcessModel = {
-    require(x.rows > 0 && x.cols > 0, "Empty input.")
-    require(x.rows == y.length, "Training feature sets and label sets must have the same number of elements")
-
-    // Normalize labels
-    val (yTrain, yMean) = if (normalizeLabels) {
-      val m = mean(y)
-      (y - m, m)
-    } else {
-      (y, 0.0)
-    }
-
-    val kernels = estimateKernelParams(x, yTrain)
-    new GaussianProcessModel(x, yTrain, yMean, kernels, predictionTransformation)
-  }
-
-  /**
-   * Estimates kernel parameters by sampling from the kernel parameter likelihood function
-   *
-   * We assume a uniform prior over the kernel parameters $\theta$ and observed features $x$, therefore:
-   *
-   *   $l(\theta|x,y) = p(y|theta,x) \propto p(theta|x,y)$
-   *
-   * Since the slice sampling algorithm requires that the function be merely proportional to the target distribution,
-   * sampling from this function is equivalent to sampling from p(\theta|x,y). These samples can then be used to compute
-   * a Monte Carlo estimate of the response for a new query point $q'$ by integrating over values of $\theta$:
-   *
-   *   $\int r(x', \theta) p(\theta) d\theta$
-   *
-   * In this way we (approximately) marginalize over all $\theta$ and arrive at a more robust estimate than would be
-   * produced by computing a maximum likelihood point estimate of the parameters.
-   *
-   * @param x the observed features
-   * @param y the observed labels
-   * @return a collection of covariance kernels corresponding to the sampled kernel parameters
-   */
-  protected[estimators] def estimateKernelParams(
-      x: DenseMatrix[Double],
-      y: DenseVector[Double]): List[Kernel] = {
-
-    val initialTheta = kernel.getInitialKernel(x, y).getParams
-
-    // Sampler burn-in. Since Markov chain samplers like slice sampler exhibit serial dependence between samples, the
-    // first n samples are biased by the initial choice of parameter vector. Here we perform a "burn in" procedure to
-    // mitigate this.
-    val thetaAfterBurnIn = (0 until monteCarloNumBurnInSamples)
-      .foldLeft(initialTheta) { (currTheta, _) =>
-        sampleNext(currTheta, x, y)
-      }
-
-    // Now draw the actual samples from the distribution
-    val (_, samples) = (0 until monteCarloNumSamples)
-      .foldLeft((thetaAfterBurnIn, List.empty[DenseVector[Double]])) { case ((currTheta, ls), _) =>
-        val nextTheta = sampleNext(currTheta, x, y)
-        (nextTheta, ls :+ nextTheta)
-      }
-
-    samples.map(kernel.withParams(_))
-  }
-
-  /**
-   * Samples the next theta, given the previous one
-   *
-   * @param theta the previous sample
-   * @param x the observed features
-   * @param y the observed labels
-   * @return the next theta sample
-   */
-  protected[estimators] def sampleNext(
-      theta: DenseVector[Double],
-      x: DenseMatrix[Double],
-      y: DenseVector[Double]): DenseVector[Double] = {
-
-    // Log likelihood wrapper function for learning length scale (holds noise and amplitude constant)
-    def lengthScaleLogp(amplitudeNoise: DenseVector[Double]) =
-      (ls: DenseVector[Double]) =>
-        kernel
-          .withParams(DenseVector.vertcat(amplitudeNoise, ls))
-          .logLikelihood(x, y)
-
-    // Log likelihood wrapper function for learning amplitude (holds noise and length scale constant)
-    def amplitudeLogp(ls: DenseVector[Double]) =
-      (amplitude: DenseVector[Double]) =>
-        kernel
-          .withParams(DenseVector.vertcat(amplitude, DenseVector(defaultNoise), ls))
-          .logLikelihood(x, y)
-
-    // Log likelihood wrapper function for learning amplitude and noise (holds length scale constant)
-    def amplitudeNoiseLogp(ls: DenseVector[Double]) =
-      (amplitudeNoise: DenseVector[Double]) =>
-        kernel
-          .withParams(DenseVector.vertcat(amplitudeNoise, ls))
-          .logLikelihood(x, y)
-
-    // Separate amplitude / noise, and length scale into separate vectors so that they can be sampled
-    // separately. There is some interplay between these parameters, so the algorithm is a bit more well behaved if
-    // they're sampled separately.
-    val currAmplitudeNoise = theta.slice(0, 2)
-    val currLengthScale = theta.slice(2, theta.length)
-    val sampler = new SliceSampler(seed = seed)
-
-    // Sample amplitude and noise
-    val amplitudeNoise = if (noisyTarget) {
-      sampler.draw(currAmplitudeNoise, amplitudeNoiseLogp(currLengthScale))
-
-    } else {
-      // If we're not sampling noise, just sample amplitude and concat on the default noise
-      DenseVector.vertcat(
-        sampler.draw(currAmplitudeNoise.slice(0, 1), amplitudeLogp(currLengthScale)),
-        DenseVector(defaultNoise))
-    }
-
-    // Sample length scale
-    val lengthScale = sampler.drawDimensionWise(currLengthScale, lengthScaleLogp(amplitudeNoise))
-
-    DenseVector.vertcat(amplitudeNoise, lengthScale)
-  }
-
-}
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/GaussianProcessModel.scala b/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/GaussianProcessModel.scala
deleted file mode 100644
index 302aed13..00000000
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/GaussianProcessModel.scala
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.estimators
-
-import breeze.linalg.{DenseMatrix, DenseVector, cholesky, diag}
-
-import com.linkedin.photon.ml.hyperparameter.estimators.kernels._
-import com.linkedin.photon.ml.util.Linalg.{choleskySolve, vectorMean}
-
-/**
- * Gaussian Process regression model that predicts mean and variance of response for new observations
- *
- * @see Gaussian Processes for Machine Learning (GPML), http://www.gaussianprocess.org/gpml/, Chapter 2
- *
- * @param xTrain the observed training features
- * @param yTrain the observed training labels
- * @param yMean the mean of the observed labels
- * @param kernels the sampled kernels
- * @param predictionTransformation optional transformation function that will be applied to the predicted response for
- *   each sampled kernel
- */
-class GaussianProcessModel protected[estimators] (
-    xTrain: DenseMatrix[Double],
-    yTrain: DenseVector[Double],
-    yMean: Double,
-    kernels: Seq[Kernel],
-    predictionTransformation: Option[PredictionTransformation]) {
-
-  require(xTrain.rows > 0 && xTrain.cols > 0, "Empty training set.")
-  require(xTrain.rows == yTrain.length, "Training feature sets and label sets must have the same number of elements")
-
-  val featureDimension = xTrain.cols
-
-  // Precompute items that don't depend on new data
-  private val precomputedKernelVals = kernels.map { kernel =>
-    val k = kernel(xTrain)
-
-    // GPML Algorithm 2.1, Line 2
-    // Since we know the kernel function produces symmetric and positive definite matrices, we can use the Cholesky
-    // factorization to solve the system $kx = yTrain$ faster than a general purpose solver (e.g. LU) could
-    val l = cholesky(k)
-
-    // Line 3
-    val alpha = choleskySolve(l, yTrain)
-
-    kernel -> (l, alpha)
-  }.toMap
-
-  /**
-   * Predicts mean and variance of response for new observations
-   *
-   * @param x the observed features
-   * @return predicted mean and variance of response
-   */
-  def predict(x: DenseMatrix[Double]): (DenseVector[Double], DenseVector[Double]) = {
-    require(x.rows > 0 && x.cols > 0, "Empty input.")
-    require(x.cols == featureDimension, s"Model was trained for $featureDimension features, but input has ${x.cols}")
-
-    val (means, vars) = kernels.map(predictWithKernel(x, _)).unzip
-    (vectorMean(means), vectorMean(vars))
-  }
-
-  /**
-   * Predicts and transforms the response for the new observations
-   *
-   * @param x the observed features
-   * @return the transformed response prediction
-   */
-  def predictTransformed(x: DenseMatrix[Double]): DenseVector[Double] = {
-    require(x.rows > 0 && x.cols > 0, "Empty input.")
-    require(x.cols == featureDimension, s"Model was trained for $featureDimension features, but input has ${x.cols}")
-
-    vectorMean(kernels
-      .map(predictWithKernel(x, _))
-      .map { case (means, covs) =>
-        predictionTransformation.map(_(means, covs)).getOrElse(means)
-      })
-  }
-
-  /**
-   * Computes the predicted mean and variance of response for the new observation, given a single kernel
-   *
-   * @param x the observed features
-   * @param kernel the covariance kernel
-   * @return predicted mean and variance of response
-   */
-  protected[estimators] def predictWithKernel(
-      x: DenseMatrix[Double],
-      kernel: Kernel): (DenseVector[Double], DenseVector[Double]) = {
-
-    val (l, alpha) = precomputedKernelVals(kernel)
-    val ktrans = kernel(x, xTrain).t
-
-    // GPML Algorithm 2.1, Line 4
-    val yPred = ktrans.t * alpha
-
-    // Line 5
-    val v = l \ ktrans
-
-    // Line 6
-    val kx = kernel(x)
-    val yCov = kx - v.t * v
-
-    (yPred + yMean, diag(yCov))
-  }
-}
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/PredictionTransformation.scala b/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/PredictionTransformation.scala
deleted file mode 100644
index c75b50fd..00000000
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/PredictionTransformation.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.estimators
-
-import breeze.linalg.DenseVector
-
-/**
- * Base trait for prediction transformations. A prediction transformation is applied to a model's predictions before
- * they're returned or integrated over.
- */
-trait PredictionTransformation {
-
-  def isMaxOpt: Boolean
-
-  /**
-   * Applies the transformation. Implementing classes should provide specific transformations here.
-   *
-   * @param predictiveMeans predictive mean output from the model
-   * @param predictiveVariances predictive variance output from the model
-   * @return the transformed predictions
-   */
-  def apply(
-      predictiveMeans: DenseVector[Double],
-      predictiveVariances: DenseVector[Double]): DenseVector[Double]
-}
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/Kernel.scala b/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/Kernel.scala
deleted file mode 100644
index 83788cef..00000000
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/Kernel.scala
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.estimators.kernels
-
-import breeze.linalg.{DenseMatrix, DenseVector}
-
-/**
- * Base trait for covariance kernel functions
- *
- * In Gaussian processes estimators and models, the covariance kernel determines the similarity between points in the
- * space. We assume that similarity in domain entails similarity in range, hence the kernel also encodes our prior
- * assumptions about how the function behaves.
- *
- * @see "Gaussian Processes for Machine Learning" (GPML), http://www.gaussianprocess.org/gpml/, Chapter 4
- */
-trait Kernel {
-
-  /**
-   * Applies the kernel function to the given points
-   *
-   * @param x the matrix of points, where each of the m rows is a point in the space
-   * @return the m x m covariance matrix
-   */
-  def apply(x: DenseMatrix[Double]): DenseMatrix[Double]
-
-  /**
-   * Applies the kernel functions to the two sets of points
-   *
-   * @param x1 the matrix containing the first set of points, where each of the m rows is a point in the space
-   * @param x2 the matrix containing the second set of points, where each of the p rows is a point in the space
-   * @return the m x p covariance matrix
-   */
-  def apply(x1: DenseMatrix[Double], x2: DenseMatrix[Double]): DenseMatrix[Double]
-
-  /**
-   * Creates a new kernel function of the same type, with the given parameters
-   *
-   * @param theta the parameter vector for the new kernel function
-   * @return the new kernel function
-   */
-  def withParams(theta: DenseVector[Double]): Kernel
-
-  /**
-   * Returns the kernel parameters as a vector
-   *
-   * @return the kernel parameters
-   */
-  def getParams: DenseVector[Double]
-
-  /**
-   * Builds a kernel with initial settings, based on the observations
-   *
-   * @param x the observed features
-   * @param y the observed labels
-   * @return the initial kernel
-   */
-  def getInitialKernel(x: DenseMatrix[Double], y: DenseVector[Double]): Kernel
-
-  /**
-   * Computes the log likelihood of the kernel parameters
-   *
-   * @param x the observed features
-   * @param y the observed labels
-   * @return the log likelihood
-   */
-  def logLikelihood(x: DenseMatrix[Double], y: DenseVector[Double]): Double
-
-  /**
-   * If only one parameter value has been specified, builds a new vector with the single value repeated to fill all
-   * dimensions
-   *
-   * @param param the initial parameters
-   * @param dim the dimensions of the final vector
-   * @return the vector with all dimensions specified
-   */
-  def expandDimensions(param: DenseVector[Double], dim: Int): DenseVector[Double] = {
-    require(param.length == 1 || param.length == dim,
-      "Parameter must contain one global scale or a scale for each feature")
-
-    if (param.length != dim) {
-      DenseVector(Array.fill(dim)(param(0)))
-    } else {
-      param
-    }
-  }
-}
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/Matern52.scala b/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/Matern52.scala
deleted file mode 100644
index 83d4ece8..00000000
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/Matern52.scala
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.estimators.kernels
-
-import breeze.linalg.{DenseMatrix, DenseVector}
-import breeze.numerics.{exp, sqrt}
-import breeze.stats.stddev
-
-/**
- * Implements the Matérn 5/2 covariance kernel.
- *
- * The Matern kernel is a generalization of the RBF kernel with an additional parameter $\nu$ that allows controlling
- * smoothness. At $\nu = \infty$, the Matern kernel is equivalent to RBF. At $\nu = 0.5$, it's equivalent to the
- * absolute exponential kernel. It's noted in the literature that $\nu = 2.5$ allows the kernel to closely approximate
- * hyperparameter spaces where the smoothness of RBF causes issues (see PBO). Here we hard-code to the 5/2 value because
- * the computation is much simpler than allowing a user-defined $\nu$.
- *
- *     $K(x,x') = \big(\sqrt{5r^2(x,x')} + \frac{5}{3} r^2(x,x') + 1\big) \exp(-\sqrt{5r^2(x,x')})$
- *
- * Where $r(x,x')$ is the Euclidean distance between $x$ and $x'$.
- *
- * @see "Gaussian Processes for Machine Learning" (GPML), http://www.gaussianprocess.org/gpml/, Chapter 4
- * @see "Practical Bayesian Optimization of Machine Learning Algorithms" (PBO),
- *   https://papers.nips.cc/paper/4522-practical-bayesian-optimization-of-machine-learning-algorithms.pdf
- *
- * @param amplitude the covariance amplitude
- * @param noise the observation noise
- * @param lengthScale the length scale of the kernel. This controls the complexity of the kernel, or the degree to which
- *   it can vary within a given region of the function's domain. Higher values allow less variation, and lower values
- *   allow more.
- */
-class Matern52(
-    amplitude: Double = 1.0,
-    noise: Double = 1e-4,
-    lengthScale: DenseVector[Double] = DenseVector(1.0))
-  extends StationaryKernel(amplitude, noise, lengthScale) {
-
-  /**
-   * Computes the Matern 5/2 kernel function from the pairwise distances between points.
-   *
-   * @param dists the m x p matrix of pairwise distances between m and p points
-   * @return the m x p covariance matrix
-   */
-  protected[kernels] override def fromPairwiseDistances(dists: DenseMatrix[Double]): DenseMatrix[Double] = {
-    val f = sqrt(dists * 5.0)
-    (f + (dists * 5.0 / 3.0) + 1.0) :* exp(-f)
-  }
-
-  /**
-   * Creates a new kernel function of the same type, with the given parameters
-   *
-   * @param theta the parameter vector for the new kernel function
-   * @return the new kernel function
-   */
-  override def withParams(theta: DenseVector[Double]): Kernel = new Matern52(
-    amplitude = theta(0),
-    noise = theta(1),
-    lengthScale = theta.slice(2, theta.length))
-
-  /**
-   * Builds a kernel with initial settings, based on the observations
-   *
-   * @param x the observed features
-   * @param y the observed labels
-   * @return the initial kernel
-   */
-  override def getInitialKernel(x: DenseMatrix[Double], y: DenseVector[Double]): Kernel =
-    new Matern52(amplitude = stddev(y))
-
-}
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/RBF.scala b/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/RBF.scala
deleted file mode 100644
index adca761c..00000000
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/RBF.scala
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.estimators.kernels
-
-import breeze.linalg.{DenseMatrix, DenseVector}
-import breeze.numerics.exp
-import breeze.stats.stddev
-
-/**
- * Implements the radial basis function (RBF) kernel.
- *
- *     $K(x,x') = \exp(-\frac{1}{2} r(x,x')^2)$
- *
- * Where $r(x,x')$ is the Euclidean distance between $x$ and $x'$.
- *
- * @param amplitude the covariance amplitude
- * @param noise the observation noise
- * @param lengthScale the length scale of the kernel. This controls the complexity of the kernel, or the degree to which
- *   it can vary within a given region of the function's domain. Higher values allow less variation, and lower values
- *   allow more.
- */
-class RBF(
-    amplitude: Double = 1.0,
-    noise: Double = 1e-4,
-    lengthScale: DenseVector[Double] = DenseVector(1.0))
-  extends StationaryKernel(amplitude, noise, lengthScale) {
-
-  /**
-   * Computes the RBF kernel function from the pairwise distances between points.
-   *
-   * @param dists the m x p matrix of pairwise distances between m and p points
-   * @return the m x p covariance matrix
-   */
-  protected[kernels] override def fromPairwiseDistances(dists: DenseMatrix[Double]): DenseMatrix[Double] =
-    exp(dists * -0.5)
-
-  /**
-   * Creates a new kernel function of the same type, with the given parameters
-   *
-   * @param theta the parameter vector for the new kernel function
-   * @return the new kernel function
-   */
-  override def withParams(theta: DenseVector[Double]): Kernel = new RBF(
-    amplitude = theta(0),
-    noise = theta(1),
-    lengthScale = theta.slice(2, theta.length))
-
-  /**
-   * Builds a kernel with initial settings, based on the observations
-   *
-   * @param x the observed features
-   * @param y the observed labels
-   * @return the initial kernel
-   */
-  override def getInitialKernel(x: DenseMatrix[Double], y: DenseVector[Double]): Kernel =
-    new RBF(amplitude = stddev(y))
-
-}
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/StationaryKernel.scala b/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/StationaryKernel.scala
deleted file mode 100644
index 114ea067..00000000
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/StationaryKernel.scala
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.estimators.kernels
-
-import breeze.linalg._
-import breeze.numerics.constants.Pi
-import breeze.numerics.{log, pow, sqrt}
-
-import com.linkedin.photon.ml.util.Linalg.choleskySolve
-
-/**
- * Base trait for stationary covariance kernel functions
- *
- * Stationary kernels depend on the relative positions of points (e.g. distance), rather than on their absolute
- * positions.
- *
- * @param amplitude the covariance amplitude
- * @param noise the observation noise
- * @param lengthScale the length scale of the kernel. This controls the complexity of the kernel, or the degree to which
- *   it can vary within a given region of the function's domain. Higher values allow less variation, and lower values
- *   allow more.
- */
-abstract class StationaryKernel(
-    amplitude: Double = 1.0,
-    noise: Double = 1e-4,
-    lengthScale: DenseVector[Double] = DenseVector(1.0))
-  extends Kernel {
-
-  // Amplitude lognormal prior
-  val amplitudeScale = 1.0
-
-  // Noise horseshoe prior
-  val noiseScale = 0.1
-
-  // Length scale tophat prior
-  val lengthScaleMax = 2.0
-
-  /**
-   * Computes the kernel function from the pairwise distances between points. Implementing classes should override this
-   * to provide the specific kernel computation.
-   *
-   * @param dists the m x p matrix of pairwise distances between m and p points
-   * @return the m x p covariance matrix
-   */
-  protected[kernels] def fromPairwiseDistances(dists: DenseMatrix[Double]): DenseMatrix[Double]
-
-  /**
-   * Applies the kernel function to the given points
-   *
-   * @param x the matrix of points, where each of the m rows is a point in the space
-   * @return the m x m covariance matrix
-   */
-  override def apply(x: DenseMatrix[Double]): DenseMatrix[Double] = {
-    require(x.rows > 0 && x.cols > 0, "Empty input.")
-
-    val ls = expandDimensions(lengthScale, x.cols)
-    val dists = pairwiseDistances(x(*,::) / ls)
-
-    (amplitude * fromPairwiseDistances(dists)) +
-      (noise * DenseMatrix.eye[Double](x.rows))
-  }
-
-  /**
-   * Applies the kernel functions to the two sets of points
-   *
-   * @param x1 the matrix containing the first set of points, where each of the m rows is a point in the space
-   * @param x2 the matrix containing the second set of points, where each of the p rows is a point in the space
-   * @return the m x p covariance matrix
-   */
-  override def apply(x1: DenseMatrix[Double], x2: DenseMatrix[Double]): DenseMatrix[Double] = {
-    require(x1.rows > 0 && x1.cols > 0 && x2.rows > 0, "Empty input.")
-    require(x1.cols == x2.cols, "Inputs must have the same number of columns")
-
-    val ls = expandDimensions(lengthScale, x1.cols)
-    val dists = pairwiseDistances(x1(*,::) / ls, x2(*,::) / ls)
-
-    amplitude * fromPairwiseDistances(dists)
-  }
-
-  /**
-   * Returns the kernel parameters as a vector
-   *
-   * @return the kernel parameters
-   */
-  override def getParams: DenseVector[Double] = DenseVector.vertcat(DenseVector(amplitude, noise), lengthScale)
-
-  /**
-   * Computes the log likelihood of the kernel parameters
-   *
-   * @param x the observed features
-   * @param y the observed labels
-   * @return the log likelihood
-   */
-  override def logLikelihood(x: DenseMatrix[Double], y: DenseVector[Double]): Double = {
-    // Bounds checks
-    if (amplitude < 0.0 ||
-        noise < 0.0 ||
-        any(lengthScale :< 0.0)) {
-      return Double.NegativeInfinity
-    }
-
-    // Tophat prior for length scale
-    if (any(lengthScale :> lengthScaleMax)) {
-      return Double.NegativeInfinity
-    }
-
-    // Apply the kernel to the input
-    val k = apply(x)
-
-    // Compute log likelihood. See GPML Algorithm 2.1
-    try {
-      // Line 2
-      // Since we know the kernel function produces symmetric and positive definite matrices, we can use the Cholesky
-      // factorization to solve the system $kx = y$ faster than a general purpose solver (e.g. LU) could.
-      val l = cholesky(k)
-
-      // Line 3
-      val alpha = choleskySolve(l, y)
-
-      // GPML algorithm 2.1 Line 7, equation 2.30
-      val likelihood = -0.5 * (y.t * alpha) - sum(log(diag(l))) - k.rows/2.0 * log(2*Pi)
-
-      // Add in lognormal prior for amplitude and horseshoe prior for noise
-      likelihood +
-        -0.5 * pow(log(sqrt(amplitude / amplitudeScale)), 2) +
-        (if (noise > 0) {
-          log(log(1.0 + pow(noiseScale / noise, 2)))
-        } else {
-          0
-        })
-
-    } catch {
-      case e: Exception => Double.NegativeInfinity
-    }
-  }
-
-  /**
-   * Computes the pairwise squared distances between all points
-   *
-   * @param x the matrix of points, where each of the m rows is a point in the space
-   * @return the m x m matrix of distances
-   */
-  protected[kernels] def pairwiseDistances(x: DenseMatrix[Double]): DenseMatrix[Double] = {
-    val out = DenseMatrix.zeros[Double](x.rows, x.rows)
-
-    for (i <- 0 until x.rows) {
-      // Lower triangular first, then reflect it across the diagonal rather than recomputing
-      for (j <- 0 until i) {
-        val dist = squaredDistance(x(i, ::).t, x(j, ::).t)
-        out(i, j) = dist
-        out(j, i) = dist
-      }
-    }
-
-    out
-  }
-
-  /**
-   * Computes the pairwise squared distance between the points in two sets
-   *
-   * @param x1 the matrix containing the first set of points, where each of the m rows is a point in the space
-   * @param x2 the matrix containing the second set of points, where each of the p rows is a point in the space
-   * @return the m x p matrix of distances
-   */
-  protected[kernels] def pairwiseDistances(x1: DenseMatrix[Double], x2: DenseMatrix[Double]): DenseMatrix[Double] = {
-    val out = DenseMatrix.zeros[Double](x1.rows, x2.rows)
-
-    for (i <- 0 until x1.rows) {
-      for (j <- 0 until x2.rows) {
-        out(i, j) = squaredDistance(x1(i, ::).t, x2(j, ::).t)
-      }
-    }
-
-    out
-  }
-
-}
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/search/GaussianProcessSearch.scala b/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/search/GaussianProcessSearch.scala
deleted file mode 100644
index 87d3eb65..00000000
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/search/GaussianProcessSearch.scala
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.search
-
-import breeze.linalg.{DenseMatrix, DenseVector}
-import breeze.stats.mean
-
-import com.linkedin.photon.ml.hyperparameter.EvaluationFunction
-import com.linkedin.photon.ml.hyperparameter.criteria.ExpectedImprovement
-import com.linkedin.photon.ml.hyperparameter.estimators.{GaussianProcessEstimator, GaussianProcessModel, PredictionTransformation}
-import com.linkedin.photon.ml.hyperparameter.estimators.kernels.{Matern52, StationaryKernel}
-
-/**
- * Performs a guided random search of the given ranges, where the search is guided by a Gaussian Process estimated from
- * evaluations of the actual evaluation function. Since we assume that the evaluation function is very costly (as it
- * often is for doing a full train / cycle evaluation of a machine learning model), it makes sense to spend time doing
- * what would otherwise be considered an expensive computation to reduce the number of times we need to evaluate the
- * function.
- *
- * At a high level, the search routine proceeds as follows:
- *
- *  1) Assume a uniform prior over the evaluation function
- *  2) Receive a new observation, and use it along with any previous observations to train a new Gaussian Process
- *     regression model for the evaluation function. This approximation is the new posterior over the evaluation
- *     function.
- *  3) Sample candidates uniformly, evaluate the posterior for each, and select the candidate with the highest predicted
- *     evaluation.
- *  4) Evaluate the best candidate with the actual evaluation function to acquire a new observation.
- *  5) Repeat from step 2.
- *
- * @param numParams the dimensionality of the hyper-parameter tuning problem
- * @param evaluationFunction the function that evaluates points in the space to real values
- * @param discreteParams specifies the indices of discrete parameters and their numbers of discrete values
- * @param kernel specifies the covariance kernel for hyper-parameters
- * @param candidatePoolSize the number of candidate points to draw at each iteration. Larger numbers give more precise
- *   results, but also incur higher computational cost.
- * @param noisyTarget whether to include observation noise in the evaluation function model
- * @param seed the random seed value
- */
-class GaussianProcessSearch[T](
-    numParams: Int,
-    evaluationFunction: EvaluationFunction[T],
-    discreteParams: Map[Int, Int] = Map(),
-    kernel: StationaryKernel = new Matern52,
-    candidatePoolSize: Int = 250,
-    noisyTarget: Boolean = true,
-    seed: Long = System.currentTimeMillis)
-  extends RandomSearch[T](numParams, evaluationFunction, discreteParams, kernel, seed){
-
-  private var observedPoints: Option[DenseMatrix[Double]] = None
-  private var observedEvals: Option[DenseVector[Double]] = None
-  private var bestEval: Double = Double.PositiveInfinity
-  private var priorObservedPoints: Option[DenseMatrix[Double]] = None
-  private var priorObservedEvals: Option[DenseVector[Double]] = None
-  private var priorBestEval: Double = Double.PositiveInfinity
-  private var lastModel: GaussianProcessModel = _
-
-  /**
-   * Produces the next candidate, given the last. In this case, we fit a Gaussian Process to the previous observations,
-   * and use it to predict the value of uniformly-drawn candidate points. The candidate with the best predicted
-   * evaluation is chosen.
-   *
-   * @param lastCandidate the last candidate
-   * @param lastObservation the last observed value
-   * @return the next candidate
-   */
-  protected[search] override def next(
-      lastCandidate: DenseVector[Double],
-      lastObservation: Double): DenseVector[Double] = {
-
-    onObservation(lastCandidate, lastObservation)
-
-    (observedPoints, observedEvals) match {
-      case (Some(points), Some(evals)) if points.rows > numParams =>
-        val candidates = drawCandidates(candidatePoolSize)
-
-        // Finding the overall bestEval
-        val currentMean = mean(evals)
-        val overallBestEval = Math.min(priorBestEval, bestEval - currentMean)
-
-        // Expected improvement transformation
-        val transformation = new ExpectedImprovement(overallBestEval)
-
-        val estimator = new GaussianProcessEstimator(
-          kernel = kernel,
-          normalizeLabels = false,
-          noisyTarget = noisyTarget,
-          predictionTransformation = Some(transformation),
-          seed = seed)
-
-        // Union of points and evals with priorData
-        val (overallPoints, overallEvals) = (priorObservedPoints, priorObservedEvals) match {
-          case (Some(priorPoints), Some(priorEvals)) => (
-            DenseMatrix.vertcat(points, priorPoints),
-            DenseVector.vertcat(evals - currentMean, priorEvals))
-          case _ =>
-            (points, evals - currentMean)
-        }
-
-        val model = estimator.fit(overallPoints, overallEvals)
-        lastModel = model
-
-        val predictions = model.predictTransformed(candidates)
-
-        selectBestCandidate(candidates, predictions, transformation)
-
-      // If we've received fewer observations than the number of parameters, fall back to a uniform search, to ensure
-      // that the problem is not under-determined.
-      case _ => super.next(lastCandidate, lastObservation)
-    }
-  }
-
-  /**
-   * Handler callback for each observation. In this case, we record the observed point and values.
-   *
-   * @param point the observed point in the space
-   * @param eval the observed value
-   */
-  protected[search] override def onObservation(point: DenseVector[Double], eval: Double): Unit = {
-    observedPoints = observedPoints
-      .map(DenseMatrix.vertcat(_, point.toDenseMatrix))
-      .orElse(Some(point.toDenseMatrix))
-
-    observedEvals = observedEvals
-      .map(DenseVector.vertcat(_, DenseVector(eval)))
-      .orElse(Some(DenseVector(eval)))
-
-    bestEval = Math.min(bestEval, eval)
-  }
-
-  /**
-    * Handler callback for each observation in the prior data. In this case, we record the observed point and values.
-    *
-    * @param point the observed point in the space
-    * @param eval the observed value
-    */
-  protected[search] override def onPriorObservation(point: DenseVector[Double], eval: Double): Unit = {
-    priorObservedPoints = priorObservedPoints
-      .map(DenseMatrix.vertcat(_, point.toDenseMatrix))
-      .orElse(Some(point.toDenseMatrix))
-
-    priorObservedEvals = priorObservedEvals
-      .map(DenseVector.vertcat(_, DenseVector(eval)))
-      .orElse(Some(DenseVector(eval)))
-
-    priorBestEval = Math.min(priorBestEval, eval)
-  }
-
-  /**
-   * Selects the best candidate according to the predicted values, where "best" is determined by the given
-   * transformation. In the case of EI, we always search for the max; In the case of CB, we always search for the min.
-   *
-   * @param candidates matrix of candidates
-   * @param predictions predicted values for each candidate
-   * @param transformation prediction transformation function
-   * @return the candidate with the best value
-   */
-  protected[search] def selectBestCandidate(
-      candidates: DenseMatrix[Double],
-      predictions: DenseVector[Double],
-      transformation: PredictionTransformation): DenseVector[Double] = {
-
-    val init = (candidates(0,::).t, predictions(0))
-
-    val direction = if (transformation.isMaxOpt) 1 else -1
-
-    val (selectedCandidate, _) = (1 until candidates.rows).foldLeft(init) {
-      case ((bestCandidate, bestPrediction), i) =>
-        if (predictions(i) * direction > bestPrediction * direction) {
-          (candidates(i,::).t, predictions(i))
-        } else {
-          (bestCandidate, bestPrediction)
-        }
-    }
-
-    selectedCandidate
-  }
-
-  /**
-   * Returns the last model trained during search
-   *
-   * @return the last model
-   */
-  def getLastModel: GaussianProcessModel = lastModel
-}
diff --git a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/search/RandomSearch.scala b/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/search/RandomSearch.scala
deleted file mode 100644
index 96251b2b..00000000
--- a/photon-lib/src/main/scala/com/linkedin/photon/ml/hyperparameter/search/RandomSearch.scala
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.search
-
-import scala.math.floor
-
-import breeze.linalg.{DenseMatrix, DenseVector}
-import org.apache.commons.math3.random.SobolSequenceGenerator
-
-import com.linkedin.photon.ml.hyperparameter.EvaluationFunction
-import com.linkedin.photon.ml.hyperparameter.estimators.kernels.{Matern52, StationaryKernel}
-
-/**
- * Performs a random search of the bounded space.
- *
- * @param numParams The dimensionality of the hyper-parameter tuning problem
- * @param evaluationFunction The function that evaluates points in the space to real values
- * @param discreteParams Specifies the indices of discrete parameters and their numbers of discrete values
- * @param kernel Specifies the indices and transformation function of hyper-parameters
- * @param seed A random seed
- */
-class RandomSearch[T](
-    numParams: Int,
-    evaluationFunction: EvaluationFunction[T],
-    discreteParams: Map[Int, Int] = Map(),
-    kernel: StationaryKernel = new Matern52,
-    seed: Long = System.currentTimeMillis) {
-
-  require(numParams > 0, "Number of parameters must be non-negative.")
-
-  /**
-   * Sobol generator for uniformly choosing roughly equidistant points.
-   */
-  private val paramDistributions = {
-    val sobol = new SobolSequenceGenerator(numParams)
-    sobol.skipTo((seed % (Int.MaxValue.toLong + 1)).toInt)
-
-    sobol
-  }
-
-  /**
-   * Searches and returns n points in the space, given prior observations from this data set and past data sets.
-   *
-   * @param n The number of points to find
-   * @param observations Observations made prior to searching, from this data set (not mean-centered)
-   * @param priorObservations Observations made prior to searching, from past data sets (mean-centered)
-   * @return The found points
-   */
-  def findWithPriors(
-      n: Int,
-      observations: Seq[(DenseVector[Double], Double)],
-      priorObservations: Seq[(DenseVector[Double], Double)]): Seq[T] = {
-
-    require(n > 0, "The number of results must be greater than zero.")
-    require(observations.nonEmpty, "There must be at least one observation.")
-
-    // Load the initial observations
-    observations.init.foreach { case (candidate, value) =>
-      onObservation(candidate, value)
-    }
-
-    // Load the prior data observations
-    priorObservations.foreach { case (candidate, value) =>
-      onPriorObservation(candidate, value)
-    }
-
-    val (results, _) = (0 until n).foldLeft((List.empty[T], observations.last)) {
-      case ((models, (lastCandidate, lastObservation)), _) =>
-
-        val candidate = next(lastCandidate, lastObservation)
-
-        // Discretize values specified as discrete
-        val candidateWithDiscrete = discretizeCandidate(candidate, discreteParams)
-
-        val (observation, model) = evaluationFunction(candidateWithDiscrete)
-
-        (models :+ model, (candidateWithDiscrete, observation))
-    }
-
-    results
-  }
-
-  /**
-   * Searches and returns n points in the space, given prior observations from past data sets.
-   *
-   * @param n The number of points to find
-   * @param priorObservations Observations made prior to searching, from past data sets (mean-centered)
-   * @return The found points
-   */
-  def findWithPriorObservations(n: Int, priorObservations: Seq[(DenseVector[Double], Double)]): Seq[T] = {
-
-    require(n > 0, "The number of results must be greater than zero.")
-
-    val candidate = drawCandidates(1)(0, ::).t
-
-    // Make values discrete as specified
-    val candidateWithDiscrete = discretizeCandidate(candidate, discreteParams)
-
-    val (_, model) = evaluationFunction(candidateWithDiscrete)
-    val initialObservation = evaluationFunction.convertObservations(Seq(model))
-
-    Seq(model) ++ (if (n == 1) Seq() else findWithPriors(n - 1, initialObservation, priorObservations))
-  }
-
-
-  /**
-   * Searches and returns n points in the space.
-   *
-   * @param n The number of points to find
-   * @return The found points
-   */
-  def find(n: Int): Seq[T] = findWithPriorObservations(n, Seq())
-
-  /**
-   * Produces the next candidate, given the last. In this case, the next candidate is chosen uniformly from the space.
-   *
-   * @param lastCandidate the last candidate
-   * @param lastObservation the last observed value
-   * @return the next candidate
-   */
-  protected[search] def next(lastCandidate: DenseVector[Double], lastObservation: Double): DenseVector[Double] =
-    drawCandidates(1)(0,::).t
-
-  /**
-   * Handler callback for each observation. In this case, we do nothing.
-   *
-   * @param point the observed point in the space
-   * @param eval the observed value
-   */
-  protected[search] def onObservation(point: DenseVector[Double], eval: Double): Unit = {}
-
-  /**
-   * Handler callback for each observation in the prior data. In this case, we do nothing.
-   *
-   * @param point the observed point in the space
-   * @param eval the observed value
-   */
-  protected[search] def onPriorObservation(point: DenseVector[Double], eval: Double): Unit = {}
-
-  /**
-   * Draw candidates from the distributions along each dimension in the space
-   *
-   * @param n the number of candidates to draw
-   */
-  protected[search] def drawCandidates(n: Int): DenseMatrix[Double] = {
-    // Draw candidates from a Sobol generator, which produces values in the range [0, 1]
-    (1 until n).foldLeft(DenseMatrix(paramDistributions.nextVector)) { case (acc, _) =>
-      DenseMatrix.vertcat(acc, DenseMatrix(paramDistributions.nextVector))
-    }
-  }
-
-  /**
-   * Discretize candidates with specified indices.
-   *
-   * @param candidate candidate with values in [0, 1]
-   * @param discreteParams Map that specifies the indices of discrete parameters and their numbers of discrete values
-   * @return candidate with the specified discrete values
-   */
-  protected[search] def discretizeCandidate(
-      candidate: DenseVector[Double],
-      discreteParams: Map[Int, Int]): DenseVector[Double] = {
-
-    val candidateWithDiscrete = candidate.copy
-
-    discreteParams.foreach { case (index, numDiscreteValues) =>
-      candidateWithDiscrete(index) = floor(candidate(index) * numDiscreteValues) / numDiscreteValues
-    }
-
-    candidateWithDiscrete
-  }
-}
diff --git a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/HyperparameterSerializationTest.scala b/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/HyperparameterSerializationTest.scala
deleted file mode 100644
index d9bcf7bc..00000000
--- a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/HyperparameterSerializationTest.scala
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Copyright 2018 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter
-
-import breeze.linalg.DenseVector
-import com.linkedin.photon.ml.HyperparameterTuningMode
-import org.testng.Assert.{assertEquals, assertNotEquals, assertTrue}
-import org.testng.annotations.Test
-
-/**
- * Unit tests for [[HyperparameterSerialization]].
- */
-class HyperparameterSerializationTest {
-
-  /**
-   * Test that prior observation data can be loaded from JSON.
-   */
-  @Test
-  def testPriorFromJson(): Unit = {
-
-    val priorDataJson =
-      """
-        |{
-        |  "records": [
-        |    {
-        |      "alpha": "1.0",
-        |      "lambda": "2.0",
-        |      "gamma": "3.0",
-        |      "evaluationValue": "0.01"
-        |    },
-        |    {
-        |      "alpha": "0.5",
-        |      "evaluationValue": "0.02"
-        |    }
-        |  ]
-        |}
-      """.stripMargin
-    val priorDefault: Map[String, String] = Map("alpha" -> "1.0", "lambda" -> "4.0", "gamma" -> "8.0")
-    val hyperParameterList = Seq("alpha", "lambda", "gamma")
-
-    val priorData = HyperparameterSerialization.priorFromJson(priorDataJson, priorDefault, hyperParameterList)
-    val expectedData = Seq(
-      (DenseVector(Array(1.0, 2.0 ,3.0)), 0.01),
-      (DenseVector(Array(0.5, 4.0, 8.0)), 0.02)
-    )
-
-    assertEquals(priorData, expectedData)
-  }
-
-  /**
-   * Unit test to set hyper-parameter configuration by default.
-   */
-  @Test
-  def testConfigFromJson(): Unit = {
-
-    val config: String =
-      """
-        |{ "tuning_mode" : "BAYESIAN",
-        |  "variables" : {
-        |    "global_regularizer" : {
-        |      "type" : "FLOAT",
-        |      "transform" : "LOG",
-        |      "min" : -3,
-        |      "max" : 3
-        |    },
-        |    "member_regularizer" : {
-        |      "type" : "FLOAT",
-        |      "transform" : "LOG",
-        |      "min" : -3,
-        |      "max" : 3
-        |    },
-        |    "item_regularizer" : {
-        |      "type" : "FLOAT",
-        |      "transform" : "LOG",
-        |      "min" : -3,
-        |      "max" : 3
-        |    }
-        |  }
-        |}
-      """.stripMargin
-
-    val hyperParams = HyperparameterSerialization.configFromJson(config)
-
-    assertEquals(hyperParams.tuningMode, HyperparameterTuningMode.BAYESIAN)
-    assertNotEquals(hyperParams.tuningMode, HyperparameterTuningMode.NONE)
-
-    // Testing matching variables name
-    assertEquals(hyperParams.names.toSet, Set("global_regularizer", "member_regularizer", "item_regularizer"))
-
-    // Testing the corresponding ranges.
-    val matchingRanges = hyperParams.ranges.zipWithIndex.map(
-      row => (hyperParams.names(row._2), (row._1.start, row._1.end)))
-
-    assertEquals(
-      matchingRanges.toSet,
-      Set(
-        ("global_regularizer", (-3, 3)),
-        ("member_regularizer", (-3, 3)),
-        ("item_regularizer", (-3, 3))))
-
-    assertTrue(hyperParams.discreteParams.isEmpty)
-
-    // Testing the transformation Map.
-    assertEquals(
-      hyperParams.transformMap.toSet,
-      Set(
-        hyperParams.names.indexOf("global_regularizer") -> "LOG",
-        hyperParams.names.indexOf("member_regularizer") -> "LOG",
-        hyperParams.names.indexOf("item_regularizer") -> "LOG"))
-  }
-}
diff --git a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/SliceSamplerTest.scala b/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/SliceSamplerTest.scala
deleted file mode 100644
index 5e4ecdca..00000000
--- a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/SliceSamplerTest.scala
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter
-
-import breeze.linalg.DenseVector
-import breeze.numerics.log
-import breeze.stats.distributions.{ContinuousDistr, Gaussian, Laplace}
-import org.apache.commons.math3.distribution.{LaplaceDistribution, NormalDistribution, RealDistribution}
-import org.apache.commons.math3.random.MersenneTwister
-import org.apache.commons.math3.stat.inference.KolmogorovSmirnovTest
-import org.testng.Assert._
-import org.testng.annotations.{DataProvider, Test}
-
-/**
- * Test cases for the SliceSampler class
- */
-class SliceSamplerTest {
-
-  val alpha = 0.001
-  val numBurnInSamples = 100
-  val numSamples = 2000
-  val seed = 0
-
-  @DataProvider
-  def distributionDataProvider() =
-    Array(
-      Array(new Gaussian(0.0, 0.5), new NormalDistribution(0.0, 0.5)),
-      Array(new Gaussian(0.0, 1.0), new NormalDistribution(0.0, 1.0)),
-      Array(new Gaussian(1.0, 4.0), new NormalDistribution(1.0, 4.0)),
-      Array(new Gaussian(-2.0, 2.1), new NormalDistribution(-2.0, 2.1)),
-      Array(new Gaussian(2.5, 2.3), new NormalDistribution(2.5, 2.3)),
-      Array(new Laplace(0.0, 0.5), new LaplaceDistribution(0.0, 0.5)),
-      Array(new Laplace(0.0, 1.0), new LaplaceDistribution(0.0, 1.0)),
-      Array(new Laplace(1.0, 4.0), new LaplaceDistribution(1.0, 4.0)),
-      Array(new Laplace(-2.0, 2.1), new LaplaceDistribution(-2.0, 2.1)),
-      Array(new Laplace(2.5, 2.3), new LaplaceDistribution(2.5, 2.3)))
-
-  @Test(dataProvider = "distributionDataProvider")
-  def testSampledDistribution(
-      sourceDistribution: ContinuousDistr[Double],
-      testDistribution: RealDistribution): Unit = {
-
-    def logp(x: DenseVector[Double]) = log(sourceDistribution.pdf(x(0)))
-    val sampler = new SliceSampler(seed = seed)
-
-    // Sampler burn-in
-    val init = (0 until numBurnInSamples)
-      .foldLeft(DenseVector(0.0)) { (currX, _) =>
-        sampler.draw(currX, logp)
-      }
-
-    // Draw the real samples
-    val (_, samples) = (0 until numSamples)
-      .foldLeft((init, List.empty[Double])) { case ((currX, ls), _) =>
-        val x = sampler.draw(currX, logp)
-        (x, ls :+ x(0))
-      }
-
-    // Run a Kolmogorov-Smirnov test to confirm the distribution of the samples
-    val tester = new KolmogorovSmirnovTest(new MersenneTwister(seed))
-    val pval = tester.kolmogorovSmirnovTest(testDistribution, samples.toArray)
-    assertTrue(pval > alpha)
-  }
-}
diff --git a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/criteria/ConfidenceBoundTest.scala b/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/criteria/ConfidenceBoundTest.scala
deleted file mode 100644
index 5c0bdbb7..00000000
--- a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/criteria/ConfidenceBoundTest.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright 2018 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.criteria
-
-import breeze.linalg.DenseVector
-import org.testng.annotations.{DataProvider, Test}
-
-import com.linkedin.photon.ml.test.Assertions.assertIterableEqualsWithTolerance
-
-/**
- * Test cases for the ConfidenceBound class
- */
-class ConfidenceBoundTest {
-
-  val TOL = 1e-3
-
-  /**
-   * Test data
-   */
-  @DataProvider
-  def modelDataProvider() =
-    Array(
-      Array(DenseVector(1.0, 2.0, 3.0), DenseVector(1.0, 2.0, 3.0), 0),
-      Array(DenseVector(-4.0, 5.0, -6.0), DenseVector(3.0, 2.0, 1.0), 1))
-
-  /**
-   * Unit tests for [[ConfidenceBound.apply]]
-   */
-  @Test(dataProvider = "modelDataProvider")
-  def testApply(mu: DenseVector[Double], sigma: DenseVector[Double], testSetIndex: Int): Unit = {
-
-    val confidenceBound = new ConfidenceBound
-    val predicted = confidenceBound(mu, sigma)
-
-    val expected = testSetIndex match {
-      case 0 => DenseVector(-1.0000, -0.8284, -0.4641)
-      case 1 => DenseVector(-7.4641, 2.1716, -8.0000)
-    }
-
-    assertIterableEqualsWithTolerance(predicted.toArray, expected.toArray, TOL)
-  }
-}
diff --git a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/criteria/ExpectedImprovementTest.scala b/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/criteria/ExpectedImprovementTest.scala
deleted file mode 100644
index 5c51c00c..00000000
--- a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/criteria/ExpectedImprovementTest.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright 2018 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.criteria
-
-import breeze.linalg.DenseVector
-import org.testng.annotations.{DataProvider, Test}
-
-import com.linkedin.photon.ml.test.Assertions.assertIterableEqualsWithTolerance
-
-/**
- * Unit tests for [[ExpectedImprovement]].
- */
-class ExpectedImprovementTest {
-
-  val TOL = 1e-3
-
-  /**
-   * Provide test data.
-   */
-  @DataProvider
-  def modelDataProvider() =
-    Array(
-      Array(DenseVector(1.0, 2.0, 3.0), DenseVector(1.0, 2.0, 3.0), DenseVector(0.0833, 0.0503, 0.0292)),
-      Array(DenseVector(-4.0, 5.0, -6.0), DenseVector(3.0, 2.0, 1.0), DenseVector(4.0062, 0.0000, 6.0000)))
-
-  /**
-   * Test that the expected improvement over an evaluation can be correctly predicted for a vector of means and standard
-   * deviations.
-   *
-   * @param mu Vector of means
-   * @param sigma Vector of standard deviations
-   * @param expectedResult Vector of expected improvements
-   */
-  @Test(dataProvider = "modelDataProvider")
-  def testApply(mu: DenseVector[Double], sigma: DenseVector[Double], expectedResult: DenseVector[Double]): Unit = {
-
-    val bestCandidate = 0.0
-    val expectedImprovement = new ExpectedImprovement(bestCandidate)
-    val predicted = expectedImprovement(mu, sigma)
-
-    assertIterableEqualsWithTolerance(predicted.toArray, expectedResult.toArray, TOL)
-  }
-}
diff --git a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/estimators/GaussianProcessEstimatorTest.scala b/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/estimators/GaussianProcessEstimatorTest.scala
deleted file mode 100644
index f956b190..00000000
--- a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/estimators/GaussianProcessEstimatorTest.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.estimators
-
-import scala.math.sin
-
-import breeze.linalg.{DenseMatrix, DenseVector}
-import breeze.linalg.linspace
-import breeze.numerics.pow
-import breeze.stats.mean
-import org.testng.Assert._
-import org.testng.annotations.{DataProvider, Test}
-
-import com.linkedin.photon.ml.hyperparameter.estimators.kernels.Matern52
-
-/**
- * Test cases for the GaussianProcessEstimator class
- */
-class GaussianProcessEstimatorTest {
-
-  private val seed = 0L
-  private val estimator = new GaussianProcessEstimator(
-    kernel = new Matern52(noise = 0.0),
-    seed = seed)
-  private val tol = 1e-7
-
-  @Test
-  def testFit(): Unit = {
-    val x = linspace(0, 10, 100)
-    val y = x.map(i => i * sin(i))
-
-    val obsPoints = List(5, 15, 35, 50, 75, 78, 90)
-    val xTrain = DenseMatrix(obsPoints.map(i => x(i)):_*)
-    val yTrain = DenseVector(obsPoints.map(i => y(i)):_*)
-
-    val model = estimator.fit(xTrain, yTrain)
-
-    val (means, vars) = model.predict(x.toDenseMatrix.t)
-
-    val mse = mean(pow(y - means, 2))
-    assertTrue(mse < 2.0)
-  }
-}
diff --git a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/estimators/GaussianProcessModelTest.scala b/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/estimators/GaussianProcessModelTest.scala
deleted file mode 100644
index 03bfd921..00000000
--- a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/estimators/GaussianProcessModelTest.scala
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.estimators
-
-import breeze.linalg.{DenseMatrix, DenseVector}
-import breeze.numerics.sqrt
-import org.testng.Assert._
-import org.testng.annotations.{DataProvider, Test}
-
-import com.linkedin.photon.ml.hyperparameter.estimators.kernels.RBF
-import com.linkedin.photon.ml.test.Assertions.assertIterableEqualsWithTolerance
-
-/**
- * Test cases for the GaussianProcessModel class
- */
-class GaussianProcessModelTest {
-  private val tol = 1e-7
-  private val kernel = new RBF(noise = 0.0, lengthScale = DenseVector(1.0))
-
-  /**
-   * Test data and results generated from reference implementation in scikit-learn
-   */
-  @DataProvider
-  def predictionProvider() =
-    Array(
-      Array(
-        DenseMatrix((0.00773725, -0.31298875, 0.27183008),
-          (-0.68440447, -0.8561772, -0.78500855),
-          (-0.02330709, -1.92979733, 0.43287544),
-          (-0.85140297, -1.49877559, -1.63778668)),
-        DenseVector(-0.34459489, -0.0485107, -1.29375589, 1.11622403),
-        DenseMatrix((-0.31800735, 1.34422005, -1.55408361),
-          (-0.60237846, -1.00816597, -0.09440482),
-          ( 0.31517342, -1.11984756, -0.9466699 ),
-          ( 0.11024813, -1.43619905, 0.67390101)),
-        DenseVector(-0.01325603, -0.66403465, -0.10878228, -1.10488029),
-        DenseVector(0.99747502, 0.44726687, 0.79425794, 0.44201904)),
-      Array(
-        DenseMatrix((0.69567278, -0.41581942, 0.85500744),
-          ( 0.98204282, -0.29115782, -0.22831259),
-          (-0.46622083, -0.68199927, -0.09467517),
-          ( 0.12449017, -0.37616456, -0.27992044)),
-        DenseVector(-0.11453575,  0.95807664, -0.7181996 , -0.29513717),
-        DenseMatrix(( 1.21362357,  0.18562891, -1.62395987),
-          (-0.75193848, 0.48940236, -0.98794203),
-          (-0.43582962, 1.83947234, 0.0808053 ),
-          (-0.73004528, -1.83643245, -0.33303083)),
-        DenseVector(0.46723757, -0.34857392, -0.05126064, -0.24301167),
-        DenseVector(0.92967279, 0.91067249, 0.99688996, 0.83459746)),
-      Array(
-        DenseMatrix((-0.46055067, 0.93364116, -1.09573962),
-          (-1.20787535, 0.33594068, -1.95753059),
-          (-0.84306614, -0.6812687 , -0.74283257),
-          (-0.95882761, 0.51132399, -0.13720216)),
-        DenseVector(-0.98494485, 0.186753, -0.65985498, 0.52334382),
-        DenseMatrix((-1.00757146, 0.78187748, -0.78197457),
-          (1.52226612, 0.43348454, -1.31427541),
-          (0.21296738, -0.77575617, 1.46077293),
-          (0.35616412, -0.01987576, -1.05690365)),
-        DenseVector(-0.16836956, -0.22862767, 0.04165401, -0.77207482),
-        DenseVector(0.3791334, 0.99059374, 0.99728549, 0.83955005)))
-
-  @Test(dataProvider = "predictionProvider")
-  def testPredict(
-      xTrain: DenseMatrix[Double],
-      yTrain: DenseVector[Double],
-      x: DenseMatrix[Double],
-      expectedMeans: DenseVector[Double],
-      expectedStd: DenseVector[Double]): Unit = {
-
-    val model = new GaussianProcessModel(
-      xTrain,
-      yTrain,
-      yMean = 0.0,
-      kernels = Seq(kernel),
-      predictionTransformation = None)
-
-    val (means, vars) = model.predict(x)
-
-    assertIterableEqualsWithTolerance(means.toArray, expectedMeans.toArray, tol)
-    assertIterableEqualsWithTolerance(sqrt(vars).toArray, expectedStd.toArray, tol)
-  }
-
-  @Test(dataProvider = "predictionProvider")
-  def testPredictTransformed(
-      xTrain: DenseMatrix[Double],
-      yTrain: DenseVector[Double],
-      x: DenseMatrix[Double],
-      expectedMeans: DenseVector[Double],
-      expectedStd: DenseVector[Double]): Unit = {
-
-    def trans(means: DenseVector[Double], std: DenseVector[Double]) =
-      1.7 * means / std
-
-    val transformation = new PredictionTransformation {
-      def isMaxOpt: Boolean = true
-      def apply(
-          predictiveMeans: DenseVector[Double],
-          predictiveVariances: DenseVector[Double]): DenseVector[Double] =
-        trans(predictiveMeans, sqrt(predictiveVariances))
-    }
-
-    val model = new GaussianProcessModel(
-      xTrain,
-      yTrain,
-      yMean = 0.0,
-      kernels = Seq(kernel),
-      predictionTransformation = Some(transformation))
-
-    val vals = model.predictTransformed(x)
-
-    assertIterableEqualsWithTolerance(vals.toArray, trans(expectedMeans, expectedStd).toArray, tol)
-  }
-}
diff --git a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/Matern52Test.scala b/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/Matern52Test.scala
deleted file mode 100644
index 37b85be0..00000000
--- a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/Matern52Test.scala
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.estimators.kernels
-
-import breeze.linalg.{DenseMatrix, DenseVector}
-import org.testng.Assert._
-import org.testng.annotations.{DataProvider, Test}
-
-import com.linkedin.photon.ml.test.Assertions.assertIterableEqualsWithTolerance
-
-/**
- * Test cases for the Matern52 class
- */
-class Matern52Test {
-
-  private val tol = 1e-7
-  private val kernel = new Matern52(noise = 0.0)
-
-  /**
-   * Test data and results generated from reference implementation in scikit-learn
-   */
-  @DataProvider
-  def kernelSourceProvider() =
-    Array(
-      Array(
-        DenseMatrix((1.16629448, 2.06716533, -0.92010277),
-          (0.32491615, -0.50086458, 0.15349931),
-          (-1.29952204, 1.22238724, -0.0238411)),
-        DenseMatrix((1.0, 0.03239932, 0.04173912),
-          (0.03239932, 1.0, 0.07761498),
-          (0.04173912, 0.07761498, 1.0))),
-      Array(
-        DenseMatrix((0.32817291, -0.62739075, -0.15141223),
-          (-0.33697839, -0.49970007, -0.30290632),
-          (-0.49786383, 0.34232845, 0.11775675),
-          (-0.86069848, -0.60832783, 0.13357631)),
-        DenseMatrix((1.0, 0.71067495, 0.36649838, 0.40439812),
-          (0.71067495, 1.0, 0.55029418, 0.71297005),
-          (0.36649838, 0.55029418, 1.0, 0.51385965),
-          (0.40439812, 0.71297005, 0.51385965, 1.0))),
-      Array(
-        DenseMatrix((-0.40944433, 0.39704702, -0.48894766),
-          (1.03282411, -1.0380654, 0.65404646),
-          (1.21080337, 0.5587334, 0.59055366),
-          (1.33081, 1.20478412, 0.8560233)),
-        DenseMatrix((1.0, 0.08284709, 0.14862395,0.08162984),
-          (0.08284709, 1.0, 0.24441232, 0.09136301),
-          (0.14862395, 0.24441232, 1.0, 0.70149793),
-          (0.08162984, 0.09136301, 0.70149793, 1.0))))
-
-  /**
-   * Test data and results generated from reference implementation in scikit-learn
-   */
-  @DataProvider
-  def kernelTwoSourceProvider() =
-    Array(
-      Array(
-        DenseMatrix((0.32817291, -0.62739075, -0.15141223),
-          (-0.33697839, -0.49970007, -0.30290632),
-          (-0.49786383, 0.34232845, 0.11775675),
-          (-0.86069848, -0.60832783, 0.13357631)),
-        DenseMatrix((-0.40944433, 0.39704702, -0.48894766),
-          (1.03282411, -1.0380654, 0.65404646),
-          (1.21080337, 0.5587334, 0.59055366),
-          (1.33081, 1.20478412, 0.8560233)),
-        DenseMatrix((0.36431909, 0.44333958, 0.22917335, 0.08481237),
-          (0.57182815, 0.19854279, 0.12340393, 0.04963231),
-          (0.75944682, 0.11384187, 0.19003345, 0.10995123),
-          (0.38353084, 0.13654483, 0.07208932, 0.03096713))),
-      Array(
-        DenseMatrix((0.32817291, -0.62739075, -0.15141223),
-          (-0.33697839, -0.49970007, -0.30290632),
-          (-0.49786383,  0.34232845,  0.11775675),
-          (-0.86069848, -0.60832783,  0.13357631)),
-        DenseMatrix((-0.92499106, 0.34302631, 0.84799782),
-          (-0.83857738, -1.20129995,  0.06613189),
-          (-1.6107072 , -0.8280462 ,  0.52490887),
-          (-0.30898909, -1.13793004, -1.34480429)),
-        DenseMatrix((0.16726349, 0.35900106, 0.12602633, 0.30430555),
-          (0.26721572, 0.56024858, 0.26314172, 0.40466601),
-          (0.61601685, 0.25344041, 0.2255722, 0.1210904),
-          (0.42003005, 0.77070301, 0.59884283, 0.22590494))))
-
-  @Test(dataProvider = "kernelSourceProvider")
-  def testKernelApply(
-      x: DenseMatrix[Double],
-      expectedResult: DenseMatrix[Double]): Unit = {
-
-    val result = kernel(x)
-    assertIterableEqualsWithTolerance(result.toArray, expectedResult.toArray, tol)
-  }
-
-  @Test(dataProvider = "kernelTwoSourceProvider")
-  def testKernelTwoSourceApply(
-      x1: DenseMatrix[Double],
-      x2: DenseMatrix[Double],
-      expectedResult: DenseMatrix[Double]): Unit = {
-
-    val result = kernel(x1, x2)
-    assertIterableEqualsWithTolerance(result.toArray, expectedResult.toArray, tol)
-  }
-
-  @Test(expectedExceptions = Array(classOf[IllegalArgumentException]))
-  def testEmptyInput(): Unit = {
-    kernel(DenseMatrix.zeros[Double](0, 0))
-  }
-
-  @Test(expectedExceptions = Array(classOf[IllegalArgumentException]))
-  def testDimensionMismatch(): Unit = {
-    kernel(DenseMatrix.zeros[Double](2, 2), DenseMatrix.zeros[Double](2, 3))
-  }
-}
diff --git a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/RBFTest.scala b/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/RBFTest.scala
deleted file mode 100644
index 09e75f6b..00000000
--- a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/RBFTest.scala
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.estimators.kernels
-
-import breeze.linalg.{DenseMatrix, DenseVector}
-import org.testng.Assert._
-import org.testng.annotations.{DataProvider, Test}
-
-import com.linkedin.photon.ml.test.Assertions.assertIterableEqualsWithTolerance
-
-/**
- * Test cases for the RBF class
- */
-class RBFTest {
-
-  private val tol = 1e-7
-  private val kernel = new RBF(noise = 0.0)
-
-  /**
-   * Test data and results generated from reference implementation in scikit-learn
-   */
-  @DataProvider
-  def kernelSourceProvider() =
-    Array(
-      Array(
-        DenseMatrix((1.16629448, 2.06716533, -0.92010277),
-          (0.32491615, -0.50086458, 0.15349931),
-          (-1.29952204, 1.22238724, -0.0238411)),
-        DenseMatrix((1.0, 0.01458651, 0.02240227),
-          (0.01458651, 1.0, 0.05961054),
-          (0.02240227, 0.05961054, 1.0))),
-      Array(
-        DenseMatrix((0.32817291, -0.62739075, -0.15141223),
-          (-0.33697839, -0.49970007, -0.30290632),
-          (-0.49786383, 0.34232845, 0.11775675),
-          (-0.86069848, -0.60832783, 0.13357631)),
-        DenseMatrix((1.0, 0.78596674, 0.42845397, 0.47354965),
-          (0.78596674, 1.0, 0.63386024, 0.78796634),
-          (0.42845397, 0.63386024, 1.0, 0.59581605),
-          (0.47354965, 0.78796634, 0.59581605, 1.0))),
-      Array(
-        DenseMatrix((-0.40944433, 0.39704702, -0.48894766),
-          (1.03282411, -1.0380654, 0.65404646),
-          (1.21080337, 0.5587334, 0.59055366),
-          (1.33081, 1.20478412, 0.8560233)),
-        DenseMatrix((1.0, 0.06567344, 0.14832728, 0.06425244),
-          (0.06567344, 1.0, 0.27451835, 0.07577536),
-          (0.14832728, 0.27451835, 1.0, 0.7779223),
-          (0.06425244, 0.07577536, 0.7779223,  1.0))))
-
-  /**
-   * Test data and results generated from reference implementation in scikit-learn
-   */
-  @DataProvider
-  def kernelTwoSourceProvider() =
-    Array(
-      Array(
-        DenseMatrix((0.32817291, -0.62739075, -0.15141223),
-          (-0.33697839, -0.49970007, -0.30290632),
-          (-0.49786383, 0.34232845, 0.11775675),
-          (-0.86069848, -0.60832783, 0.13357631)),
-        DenseMatrix((-0.40944433, 0.39704702, -0.48894766),
-          (1.03282411, -1.0380654, 0.65404646),
-          (1.21080337, 0.5587334, 0.59055366),
-          (1.33081, 1.20478412, 0.8560233)),
-        DenseMatrix((0.42581894, 0.518417, 0.25455962, 0.06798038),
-          (0.65572813, 0.21417167, 0.11566118, 0.02974926),
-          (0.82741311, 0.10351387, 0.2029175, 0.09862232),
-          (0.44889221, 0.13258973, 0.0533442, 0.0134873))),
-      Array(
-        DenseMatrix((0.32817291, -0.62739075, -0.15141223),
-          (-0.33697839, -0.49970007, -0.30290632),
-          (-0.49786383,  0.34232845,  0.11775675),
-          (-0.86069848, -0.60832783,  0.13357631)),
-        DenseMatrix((-0.92499106, 0.34302631, 0.84799782),
-          (-0.83857738, -1.20129995,  0.06613189),
-          (-1.6107072 , -0.8280462 ,  0.52490887),
-          (-0.30898909, -1.13793004, -1.34480429)),
-        DenseMatrix((0.17282516, 0.41936999, 0.11901991, 0.35154935),
-          (0.30414111,  0.64402575,  0.29887283,  0.47386342),
-          (0.69918137,  0.28628435,  0.24982739,  0.11270722),
-          (0.49174098,  0.83666878,  0.6825188 ,  0.25026486))))
-
-  @Test(dataProvider = "kernelSourceProvider")
-  def testKernelApply(
-      x: DenseMatrix[Double],
-      expectedResult: DenseMatrix[Double]): Unit = {
-
-    val result = kernel(x)
-    assertIterableEqualsWithTolerance(result.toArray, expectedResult.toArray, tol)
-  }
-
-  @Test(dataProvider = "kernelTwoSourceProvider")
-  def testKernelTwoSourceApply(
-      x1: DenseMatrix[Double],
-      x2: DenseMatrix[Double],
-      expectedResult: DenseMatrix[Double]): Unit = {
-
-    val result = kernel(x1, x2)
-    assertIterableEqualsWithTolerance(result.toArray, expectedResult.toArray, tol)
-  }
-
-  @Test(expectedExceptions = Array(classOf[IllegalArgumentException]))
-  def testEmptyInput(): Unit = {
-    kernel(DenseMatrix.zeros[Double](0, 0))
-  }
-
-  @Test(expectedExceptions = Array(classOf[IllegalArgumentException]))
-  def testDimensionMismatch(): Unit = {
-    kernel(DenseMatrix.zeros[Double](2, 2), DenseMatrix.zeros[Double](2, 3))
-  }
-}
diff --git a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/StationaryKernelTest.scala b/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/StationaryKernelTest.scala
deleted file mode 100644
index ef59709c..00000000
--- a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/estimators/kernels/StationaryKernelTest.scala
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright 2018 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.estimators.kernels
-
-import breeze.linalg.{DenseMatrix, DenseVector}
-import org.testng.Assert._
-import org.testng.annotations.{DataProvider, Test}
-
-/**
- * Test cases for the StationaryKernel class
- */
-class StationaryKernelTest {
-
-  private val tol = 1e-7
-  private val kernel = new RBF(noise = 0.0)
-
-  /**
-   * Test data and results generated from reference implementation in scikit-learn
-   */
-  @DataProvider
-  def logLikelihoodProvider() =
-    Array(
-      Array(
-        DenseMatrix((1.0, 2.0), (3.0, 4.0)),
-        DenseVector(1.0, 0.0),
-        DenseVector(1.0, 0.0, 1.0),
-        -2.3378770946057106),
-      Array(
-        DenseMatrix((0.01388442, -0.45110147, -1.19551816,  0.67570627),
-          (0.03187212,  0.26128739,  0.09664432,  0.51288568),
-          (-0.1892356 , -0.37331575, -0.63040424, -0.91800163),
-          (-0.41915027, -0.63310943, -0.33619354, -1.54669407)),
-        DenseVector(1.97492722, -0.14301486, -0.17681195, -0.25272319),
-        DenseVector(1.0, 0.0, 1.0),
-        -5.6170735015658586),
-      Array(
-        DenseMatrix((-0.80210875,  1.01741171,  0.14846267,  1.15931876),
-          (1.40446672, -0.06407389, -0.06608613, -0.55499189),
-          (0.65113077, -0.53180815, -0.51595562,  0.08615354),
-          (0.33328126,  0.89654475,  0.99865134, -0.34262719)),
-        DenseVector(0.01540465,  0.93196629, -0.83929026,  0.39678908),
-        DenseVector(1.0, 0.0, 0.36787944117144233),
-        -4.5455256227943401),
-      Array(
-        DenseMatrix((0.32817291, -0.62739075, -0.15141223),
-          (-0.33697839, -0.49970007, -0.30290632),
-          (-0.49786383, 0.34232845, 0.11775675),
-          (-0.86069848, -0.60832783, 0.13357631)),
-        DenseVector(0.16192959, 0.86015525, -0.14415703, -0.46888909),
-        DenseVector(1.0, 0.0, 1.0),
-        -6.0655926036008498))
-
-  @Test(dataProvider = "logLikelihoodProvider")
-  def testLogLikelihood(
-      x: DenseMatrix[Double],
-      y: DenseVector[Double],
-      theta: DenseVector[Double],
-      expectedLikelihood: Double): Unit = {
-
-    val likelihood = kernel.withParams(theta).logLikelihood(x, y)
-    assertEquals(likelihood, expectedLikelihood, tol)
-  }
-}
diff --git a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/search/GaussianProcessSearchTest.scala b/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/search/GaussianProcessSearchTest.scala
deleted file mode 100644
index acafd1b9..00000000
--- a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/search/GaussianProcessSearchTest.scala
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.search
-
-import breeze.linalg.{DenseMatrix, DenseVector}
-import org.testng.Assert._
-import org.testng.annotations.{DataProvider, Test}
-
-import com.linkedin.photon.ml.hyperparameter.EvaluationFunction
-import com.linkedin.photon.ml.hyperparameter.criteria.{ConfidenceBound, ExpectedImprovement}
-import com.linkedin.photon.ml.hyperparameter.estimators.kernels.Matern52
-
-/**
- * Unit tests for [[GaussianProcessSearch]]
- */
-class GaussianProcessSearchTest {
-
-  import GaussianProcessSearchTest._
-
-  val searcher = new GaussianProcessSearch[TestModel](
-    DIM,
-    EVALUATION_FUNCTION,
-    DISCRETE_PARAMS,
-    KERNEL,
-    seed = SEED)
-
-  var observedPoints: Option[DenseMatrix[Double]] = None
-  var observedEvals: Option[DenseVector[Double]] = None
-  var bestEval: Double = 0.0
-  var priorObservedPoints: Option[DenseMatrix[Double]] = None
-  var priorObservedEvals: Option[DenseVector[Double]] = None
-  var priorBestEval: Double = 0.0
-
-  @DataProvider
-  def priorDataProvider: Array[Array[Any]] = {
-
-    val candidate1 = (DenseVector(0.25, 0.125, 0.999), 0.1)
-    val candidate2 = (DenseVector(0.2, 0.2, 0.2), 0.2)
-    val candidate3 = (DenseVector(0.3, 0.3, 0.3), -0.3)
-    val candidate4 = (DenseVector(0.2, 0.2, 0.2), 0.4)
-    val candidate5 = (DenseVector(0.3, 0.3, 0.3), -0.4)
-    val observations = Seq(candidate1, candidate2, candidate3)
-    val priorObservations = Seq(candidate4, candidate5)
-
-    Array(
-      Array(observations, Seq(), 0),
-      Array(observations, priorObservations, 1))
-  }
-
-  @Test(dataProvider = "priorDataProvider")
-  def testFindWithPriors(
-      currentCandidates: Seq[(DenseVector[Double], Double)],
-      priorCandidates: Seq[(DenseVector[Double], Double)],
-      testSetIndex: Int): Unit = {
-
-    val candidates = searcher.findWithPriors(N, currentCandidates, priorCandidates)
-
-    assertEquals(candidates.length, N)
-    assertEquals(candidates.toSet.size, N)
-    assertTrue(candidates.forall(_.params.toArray.forall(x => x >= 0 && x < 1)))
-  }
-
-  @Test(dataProvider = "priorDataProvider", dependsOnMethods = Array[String]("testFindWithPriors"))
-  def testFindWithPriorObservations(
-      currentCandidates: Seq[(DenseVector[Double], Double)],
-      priorCandidates: Seq[(DenseVector[Double], Double)],
-      testSetIndex: Int): Unit = {
-
-    val candidates = searcher.findWithPriorObservations(N, priorCandidates)
-
-    assertEquals(candidates.length, N)
-    assertTrue(candidates.forall(_.params.toArray.forall(x => x >= 0 && x < 1)))
-    assertEquals(candidates.size, N)
-  }
-
-  @Test(dependsOnMethods = Array[String]("testFindWithPriorObservations"))
-  def testFind(): Unit = {
-    val candidates = searcher.find(N)
-
-    assertEquals(candidates.length, N)
-    assertEquals(candidates.toSet.size, N)
-    assertTrue(candidates.forall(_.params.toArray.forall(x => x >= 0 && x < 1)))
-  }
-
-  @DataProvider
-  def bestCandidateDataProvider: Array[Array[Any]] = {
-    val candidate1 = DenseVector(1.0)
-    val candidate2 = DenseVector(2.0)
-    val candidate3 = DenseVector(3.0)
-    val candidates = DenseMatrix.vertcat(
-      candidate1.asDenseMatrix,
-      candidate2.asDenseMatrix,
-      candidate3.asDenseMatrix)
-
-    Array(
-      Array(candidates, DenseVector(2.0, 1.0, 0.0), candidate1, candidate3),
-      Array(candidates, DenseVector(1.0, 2.0, 0.0), candidate2, candidate3),
-      Array(candidates, DenseVector(0.0, 1.0, 2.0), candidate3, candidate1))
-  }
-
-  /**
-   * Unit test for [[GaussianProcessSearch.selectBestCandidate]].
-   * If transformation is EI, select the best candidate that maximize the prediction.
-   * If transformation is CB, select the best candidate that minimize the prediction.
-   */
-  @Test(dataProvider = "bestCandidateDataProvider")
-  def testSelectBestCandidate(
-      candidates: DenseMatrix[Double],
-      predictions: DenseVector[Double],
-      expectedMax: DenseVector[Double],
-      expectedMin: DenseVector[Double]): Unit = {
-
-    val bestCandidate = 0.0
-    val transformationEI = new ExpectedImprovement(bestCandidate)
-    val selectedMax = searcher.selectBestCandidate(candidates, predictions, transformationEI)
-    assertEquals(selectedMax, expectedMax)
-
-    val transformationCB = new ConfidenceBound
-    val selectedMin = searcher.selectBestCandidate(candidates, predictions, transformationCB)
-    assertEquals(selectedMin, expectedMin)
-  }
-
-  @Test(dataProvider = "priorDataProvider")
-  def testOnPriorObservation(
-      currentCandidates: Seq[(DenseVector[Double], Double)],
-      priorCandidates: Seq[(DenseVector[Double], Double)],
-      testSetIndex: Int): Unit = {
-
-    // Load the initial observations
-    currentCandidates.foreach { case (candidate, value) =>
-      observedPoints = observedPoints
-        .map(DenseMatrix.vertcat(_, candidate.toDenseMatrix))
-        .orElse(Some(candidate.toDenseMatrix))
-
-      observedEvals = observedEvals
-        .map(DenseVector.vertcat(_, DenseVector(value)))
-        .orElse(Some(DenseVector(value)))
-
-      bestEval = Math.min(bestEval, value)
-    }
-
-    priorCandidates.foreach { case (candidate, value) =>
-      priorObservedPoints = priorObservedPoints
-        .map(DenseMatrix.vertcat(_, candidate.toDenseMatrix))
-        .orElse(Some(candidate.toDenseMatrix))
-
-      priorObservedEvals = priorObservedEvals
-        .map(DenseVector.vertcat(_, DenseVector(value)))
-        .orElse(Some(DenseVector(value)))
-
-      priorBestEval = Math.min(priorBestEval, value)
-    }
-
-    testSetIndex match {
-      case 0 =>
-        assertEquals(observedPoints.get.rows, 3)
-        assertEquals(observedPoints.get.cols, 3)
-        assertEquals(observedEvals.get.length, 3)
-        assertEquals(bestEval, -0.3)
-        assertFalse(priorObservedPoints.isDefined)
-        assertFalse(priorObservedEvals.isDefined)
-      case 1 =>
-        assertEquals(observedPoints.get.rows, 6)
-        assertEquals(observedPoints.get.cols, 3)
-        assertEquals(observedEvals.get.length, 6)
-        assertEquals(bestEval, -0.3)
-        assertEquals(priorObservedPoints.get.rows, 2)
-        assertEquals(priorObservedPoints.get.cols, 3)
-        assertEquals(priorObservedEvals.get.length, 2)
-        assertEquals(priorBestEval, -0.4)
-    }
-  }
-}
-
-object GaussianProcessSearchTest {
-
-  val SEED = 1L
-  val DIM = 3
-  val N = 5
-  val DISCRETE_PARAMS = Map(0 -> 5, 2 -> 9)
-  val KERNEL = new Matern52
-  val TOLERANCE = 1E-12
-
-  case class TestModel(params: DenseVector[Double], evaluation: Double)
-
-  val EVALUATION_FUNCTION: EvaluationFunction[TestModel] = new EvaluationFunction[TestModel] {
-
-    def apply(hyperParameters: DenseVector[Double]): (Double, TestModel) = {
-      (0.0, TestModel(hyperParameters, 0.0))
-    }
-    def convertObservations(results: Seq[TestModel]): Seq[(DenseVector[Double], Double)] = {
-      results.map { result =>
-        val candidate = vectorizeParams(result)
-        val value = getEvaluationValue(result)
-        (candidate, value)
-      }
-    }
-    def vectorizeParams(result: TestModel): DenseVector[Double] = result.params
-    def getEvaluationValue(result: TestModel): Double = result.evaluation
-  }
-}
diff --git a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/search/RandomSearchTest.scala b/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/search/RandomSearchTest.scala
deleted file mode 100644
index 9b204a61..00000000
--- a/photon-lib/src/test/scala/com/linkedin/photon/ml/hyperparameter/search/RandomSearchTest.scala
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Copyright 2017 LinkedIn Corp. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package com.linkedin.photon.ml.hyperparameter.search
-
-import breeze.linalg.{DenseVector, norm}
-import org.testng.Assert._
-import org.testng.annotations.{DataProvider, Test}
-
-import com.linkedin.photon.ml.hyperparameter.EvaluationFunction
-import com.linkedin.photon.ml.hyperparameter.estimators.kernels.Matern52
-
-/**
- * Unit tests for [[RandomSearch]].
- */
-class RandomSearchTest {
-
-  import RandomSearchTest._
-
-  val searcher = new RandomSearch[TestModel](DIM, EVALUATION_FUNCTION, DISCRETE_PARAMS, KERNEL, SEED)
-
-  @DataProvider
-  def priorDataProvider: Array[Array[Any]] = {
-
-    val candidate1 = (DenseVector(0.25, 0.125, 0.999), 0.1)
-    val candidate2 = (DenseVector(0.2, 0.2, 0.2), 0.2)
-    val candidate3 = (DenseVector(0.3, 0.3, 0.3), -0.3)
-    val candidate4 = (DenseVector(0.2, 0.2, 0.2), 0.4)
-    val candidate5 = (DenseVector(0.3, 0.3, 0.3), -0.4)
-    val observations = Seq(candidate1, candidate2, candidate3)
-    val priorObservations = Seq(candidate4, candidate5)
-
-    Array(Array(observations, priorObservations))
-  }
-
-  /**
-   * Test that [[RandomSearch]] can generate multiple points in the search space.
-   */
-  @Test(dataProvider = "priorDataProvider")
-  def testFindWithPriors(
-      observations: Seq[(DenseVector[Double], Double)],
-      priorObservations: Seq[(DenseVector[Double], Double)]): Unit = {
-
-    val candidates = searcher.findWithPriors(N, observations, priorObservations)
-
-    assertEquals(candidates.length, N)
-    assertEquals(candidates.toSet.size, N)
-    assertTrue(candidates.forall(_.params.toArray.forall(x => x >= 0 && x < 1)))
-  }
-
-  /**
-   * Test that [[RandomSearch]] can generate multiple points in the search space.
-   */
-  @Test(dataProvider = "priorDataProvider", dependsOnMethods = Array[String]("testFindWithPriors"))
-  def testFindWithPriorObservations(
-      observations: Seq[(DenseVector[Double], Double)],
-      priorObservations: Seq[(DenseVector[Double], Double)]): Unit = {
-
-    val candidates = searcher.findWithPriorObservations(N, priorObservations)
-
-    assertEquals(candidates.length, N)
-    assertEquals(candidates.toSet.size, N)
-    assertTrue(candidates.forall(_.params.toArray.forall(x => x >= 0 && x < 1)))
-  }
-
-  /**
-   * Test that observations and prior observations don't affect [[RandomSearch]].
-   */
-  @Test(dataProvider = "priorDataProvider", dependsOnMethods = Array[String]("testFindWithPriorObservations"))
-  def testFind(
-      observations: Seq[(DenseVector[Double], Double)],
-      priorObservations: Seq[(DenseVector[Double], Double)]): Unit = {
-
-    val searcher1 = new RandomSearch[TestModel](DIM, EVALUATION_FUNCTION, DISCRETE_PARAMS, KERNEL, SEED)
-    val candidates1 = searcher1.find(N)
-
-    assertEquals(candidates1.length, N)
-    assertEquals(candidates1.toSet.size, N)
-    assertTrue(candidates1.forall(_.params.toArray.forall(x => x >= 0 && x < 1)))
-
-    val searcher2 = new RandomSearch[TestModel](DIM, EVALUATION_FUNCTION, DISCRETE_PARAMS, KERNEL, SEED)
-    val candidates2 = searcher2.findWithPriors(N, observations, priorObservations)
-
-    candidates1.zip(candidates2).foreach { case (candidate1, candidate2) =>
-      assertTrue(candidate1.params == candidate2.params)
-      assertEquals(candidate1.evaluation, candidate2.evaluation, TOLERANCE)
-    }
-  }
-
-  /**
-   * Test that the candidates of integer hyper-parameters are discretized correctly.
-   */
-  @Test(dataProvider = "priorDataProvider")
-  def testDiscretizeCandidate(
-      observations: Seq[(DenseVector[Double], Double)],
-      priorObservations: Seq[(DenseVector[Double], Double)]): Unit = {
-
-    val candidate = observations.head._1
-    val expectedData = DenseVector(0.2, 0.125, 8.0 / 9.0)
-
-    val candidateWithDiscrete = searcher.discretizeCandidate(candidate, DISCRETE_PARAMS)
-    assertEquals(norm(candidateWithDiscrete), norm(expectedData), TOLERANCE)
-  }
-}
-
-object RandomSearchTest {
-
-  val SEED = 1L
-  val DIM = 3
-  val N = 5
-  val DISCRETE_PARAMS = Map(0 -> 5, 2 -> 9)
-  val KERNEL = new Matern52
-  val TOLERANCE = 1E-12
-
-  case class TestModel(params: DenseVector[Double], evaluation: Double)
-
-  val EVALUATION_FUNCTION: EvaluationFunction[TestModel] = new EvaluationFunction[TestModel] {
-
-    def apply(hyperParameters: DenseVector[Double]): (Double, TestModel) = {
-      (0.0, TestModel(hyperParameters, 0.0))
-    }
-
-    def convertObservations(results: Seq[TestModel]): Seq[(DenseVector[Double], Double)] = {
-      results.map { result =>
-        val candidate = vectorizeParams(result)
-        val value = getEvaluationValue(result)
-        (candidate, value)
-      }
-    }
-    def vectorizeParams(result: TestModel): DenseVector[Double] = result.params
-    def getEvaluationValue(result: TestModel): Double = result.evaluation
-  }
-}