Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Enable coefficient variance computation #141

Merged
merged 1 commit into from
Jul 22, 2016
Merged

Enable coefficient variance computation #141

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions photon-ml/src/integTest/scala/com/linkedin/photon/ml/cli/game/training/DriverTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,13 @@
*/
package com.linkedin.photon.ml.cli.game.training

import java.nio.file.{FileSystems, Files, Path}

import collection.JavaConversions._
import org.apache.spark.SparkConf
import org.testng.Assert._
import org.testng.annotations.Test

import com.linkedin.photon.ml.SparkContextConfiguration
import com.linkedin.photon.ml.avro.AvroIOUtils
import com.linkedin.photon.ml.avro.data.NameAndTerm
Expand All @@ -23,18 +29,12 @@ import com.linkedin.photon.ml.avro.model.ModelProcessingUtils
import com.linkedin.photon.ml.data.{FixedEffectDataSet, RandomEffectDataSet}
import com.linkedin.photon.ml.io.ModelOutputMode
import com.linkedin.photon.ml.supervised.TaskType
import com.linkedin.photon.ml.supervised.regression.LinearRegressionModel
import com.linkedin.photon.ml.test.{CommonTestUtils, SparkTestUtils, TestTemplateWithTmpDir}
import com.linkedin.photon.ml.util.{Utils, PhotonLogger}
import org.apache.spark.SparkConf
import org.testng.annotations.Test
import org.testng.Assert._

import java.nio.file.{Files, FileSystems, Path}
import com.linkedin.photon.ml.util.{PhotonLogger, Utils}

class DriverTest extends SparkTestUtils with TestTemplateWithTmpDir {
import DriverTest._
import CommonTestUtils._
import DriverTest._

@Test
def testFixedEffectsWithIntercept() = sparkTest("testFixedEffectsWithIntercept", useKryo = true) {
Expand Down
58 changes: 34 additions & 24 deletions photon-ml/src/main/scala/com/linkedin/photon/ml/cli/game/training/Driver.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,12 @@
*/
package com.linkedin.photon.ml.cli.game.training

import com.linkedin.photon.ml.optimization.{
GeneralizedLinearOptimizationProblem, SmoothedHingeLossLinearSVMOptimizationProblem, PoissonRegressionOptimizationProblem, LogisticRegressionOptimizationProblem, LinearRegressionOptimizationProblem}
import com.linkedin.photon.ml.optimization.game.GLMOptimizationConfiguration
import com.linkedin.photon.ml.{RDDLike, SparkContextConfiguration}
import scala.collection.Map

import org.apache.hadoop.fs.Path
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD

import com.linkedin.photon.ml.algorithm._
import com.linkedin.photon.ml.avro.AvroUtils
import com.linkedin.photon.ml.avro.data.{DataProcessingUtils, NameAndTerm, NameAndTermFeatureSetContainer}
Expand All @@ -28,16 +30,13 @@ import com.linkedin.photon.ml.evaluation._
import com.linkedin.photon.ml.function.DiffFunction
import com.linkedin.photon.ml.io.ModelOutputMode
import com.linkedin.photon.ml.model.GAMEModel
import com.linkedin.photon.ml.optimization.game.{FactoredRandomEffectOptimizationProblem, RandomEffectOptimizationProblem}
import com.linkedin.photon.ml.optimization.game.{GLMOptimizationConfiguration, FactoredRandomEffectOptimizationProblem, RandomEffectOptimizationProblem}
import com.linkedin.photon.ml.optimization.{GeneralizedLinearOptimizationProblem, LinearRegressionOptimizationProblem, LogisticRegressionOptimizationProblem, PoissonRegressionOptimizationProblem, SmoothedHingeLossLinearSVMOptimizationProblem}
import com.linkedin.photon.ml.projector.IdentityProjection
import com.linkedin.photon.ml.supervised.TaskType._
import com.linkedin.photon.ml.supervised.model.GeneralizedLinearModel
import com.linkedin.photon.ml.util._
import org.apache.hadoop.fs.Path
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD

import scala.collection.Map
import com.linkedin.photon.ml.{RDDLike, SparkContextConfiguration}

/**
* The driver class, which provides the main entrance to GAME model training
Expand All @@ -48,8 +47,6 @@ final class Driver(val params: Params, val sparkContext: SparkContext, val logge

private val hadoopConfiguration = sparkContext.hadoopConfiguration

private val isTrackingState = true

/**
* Builds feature name-and-term to index maps according to configuration
*
Expand Down Expand Up @@ -321,15 +318,14 @@ final class Driver(val params: Params, val sparkContext: SparkContext, val logge
trainingEvaluator: Evaluator,
validatingDataAndEvaluatorOption: Option[(RDD[(Long, GameDatum)], Evaluator)]): Map[String, GAMEModel] = {

val optimizationProblemBuilder:
Function3[GLMOptimizationConfiguration, Int, Boolean,
GeneralizedLinearOptimizationProblem[GeneralizedLinearModel, DiffFunction[LabeledPoint]]] = taskType match {
val optimizationProblemBuilder: (GLMOptimizationConfiguration, Int, Boolean, Boolean) =>
GeneralizedLinearOptimizationProblem[GeneralizedLinearModel, DiffFunction[LabeledPoint]] = taskType match {

case LOGISTIC_REGRESSION => LogisticRegressionOptimizationProblem.buildOptimizationProblem _
case LINEAR_REGRESSION => LinearRegressionOptimizationProblem.buildOptimizationProblem _
case POISSON_REGRESSION => PoissonRegressionOptimizationProblem.buildOptimizationProblem _
case LOGISTIC_REGRESSION => LogisticRegressionOptimizationProblem.buildOptimizationProblem
case LINEAR_REGRESSION => LinearRegressionOptimizationProblem.buildOptimizationProblem
case POISSON_REGRESSION => PoissonRegressionOptimizationProblem.buildOptimizationProblem
case SMOOTHED_HINGE_LOSS_LINEAR_SVM =>
SmoothedHingeLossLinearSVMOptimizationProblem.buildOptimizationProblem _
SmoothedHingeLossLinearSVMOptimizationProblem.buildOptimizationProblem

case _ => throw new Exception(s"Loss function for taskType $taskType is currently not supported.")
}
Expand All @@ -354,20 +350,28 @@ final class Driver(val params: Params, val sparkContext: SparkContext, val logge
val optimizationConfiguration = fixedEffectOptimizationConfiguration(coordinateId)
// If number of features is from moderate to large (>200000), then use tree aggregate,
// otherwise use aggregate.
val treeAggregateDepth = if (fixedEffectDataSet.numFeatures < 200000) 1 else 2
val treeAggregateDepth = if (fixedEffectDataSet.numFeatures < 200000) {
Driver.DEFAULT_TREE_AGGREGATE_DEPTH
} else {
Driver.DEEP_TREE_AGGREGATE_DEPTH
}
val optimizationProblem = optimizationProblemBuilder(
optimizationConfiguration,
treeAggregateDepth,
isTrackingState)
Driver.TRACK_STATE,
computeVariance)
new FixedEffectCoordinate(fixedEffectDataSet, optimizationProblem)

case randomEffectDataSetInProjectedSpace: RandomEffectDataSetInProjectedSpace =>
// Random effect coordinate
val optimizationConfiguration = randomEffectOptimizationConfiguration(coordinateId)
val randomEffectOptimizationProblem = RandomEffectOptimizationProblem.buildRandomEffectOptimizationProblem(
val randomEffectOptimizationProblem = RandomEffectOptimizationProblem
.buildRandomEffectOptimizationProblem(
optimizationProblemBuilder,
optimizationConfiguration,
randomEffectDataSetInProjectedSpace)
randomEffectDataSetInProjectedSpace,
Driver.DEFAULT_TREE_AGGREGATE_DEPTH,
computeVariance)
.setName(s"Random effect optimization problem of coordinate $coordinateId")
.persistRDD(StorageLevel.INFREQUENT_REUSE_RDD_STORAGE_LEVEL)
new RandomEffectCoordinateInProjectedSpace(
Expand All @@ -385,7 +389,10 @@ final class Driver(val params: Params, val sparkContext: SparkContext, val logge
randomEffectOptimizationConfiguration,
latentFactorOptimizationConfiguration,
mfOptimizationConfiguration,
randomEffectDataSet)
randomEffectDataSet,
Driver.DEFAULT_TREE_AGGREGATE_DEPTH,
Driver.TRACK_STATE,
computeVariance)
.setName(s"Factored random effect optimization problem of coordinate $coordinateId")
.persistRDD(StorageLevel.INFREQUENT_REUSE_RDD_STORAGE_LEVEL)
new FactoredRandomEffectCoordinate(randomEffectDataSet, factoredRandomEffectOptimizationProblem)
Expand Down Expand Up @@ -517,6 +524,9 @@ final class Driver(val params: Params, val sparkContext: SparkContext, val logge
}

object Driver {
val DEFAULT_TREE_AGGREGATE_DEPTH = 1
val DEEP_TREE_AGGREGATE_DEPTH = 2
val TRACK_STATE = false
val LOGS = "logs"

/**
Expand Down
9 changes: 9 additions & 0 deletions photon-ml/src/main/scala/com/linkedin/photon/ml/cli/game/training/Params.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,11 @@ class Params {
*/
var numIterations: Int = 1

/**
* Whether to compute coefficient variance
*/
var computeVariance: Boolean = false

/**
* Updating order of the ordinates (separated by commas) in the coordinate descent algorithm.
*/
Expand Down Expand Up @@ -192,6 +197,7 @@ class Params {
.map(_.mkString("\n")).mkString("\n")}\n" +
s"randomEffectDataConfigurations:\n${randomEffectDataConfigurations.mkString("\n")}\n" +
s"taskType: $taskType\n" +
s"computeVariance: $computeVariance\n" +
s"modelOutputOption: $modelOutputMode\n" +
s"numberOfOutputFilesForRandomEffectModel: $numberOfOutputFilesForRandomEffectModel\n" +
s"deleteOutputDirIfExists: $deleteOutputDirIfExists\n" +
Expand Down Expand Up @@ -355,6 +361,9 @@ object Params {
}
.toMap
)
opt[Boolean]("compute-variance")
.text(s"Whether to compute the coefficient variance, default: ${defaultParams.computeVariance}")
.foreach(x => params.computeVariance = x)
opt[Boolean]("save-models-to-hdfs")
.text(s"DEPRECATED -- USE model-output-mode")
.foreach(x => params.modelOutputMode = if (x) ALL else NONE)
Expand Down
72 changes: 36 additions & 36 deletions photon-ml/src/main/scala/com/linkedin/photon/ml/function/LogisticLossFunction.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,48 +19,48 @@ import com.linkedin.photon.ml.normalization.{NoNormalization, NormalizationConte
import com.linkedin.photon.ml.util.Utils

/**
* Class for the logistic loss function:
* sum_i (w_i*(y_i*log(1 + exp(-(theta'x_i + o_i))) + (1-y_i)*log(1 + exp(theta'x_i + o_i)))),
* where \theta is the coefficients of the data features to be estimated, (y_i, x_i, o_i, w_i) are the tuple
* for label, features, offset, and weight of the i'th labeled data point, respectively.
* Note that the above equation assumes the label y_i \in {0, 1}. However, the code below would also work when
* y_i \in {-1, 1}.
*/
* Class for the logistic loss function:
* sum_i (w_i*(y_i*log(1 + exp(-(theta'x_i + o_i))) + (1-y_i)*log(1 + exp(theta'x_i + o_i)))),
* where \theta is the coefficients of the data features to be estimated, (y_i, x_i, o_i, w_i) are the tuple
* for label, features, offset, and weight of the i'th labeled data point, respectively.
* Note that the above equation assumes the label y_i \in {0, 1}. However, the code below would also work when
* y_i \in {-1, 1}.
*/
class LogisticLossFunction(normalizationContext: ObjectProvider[NormalizationContext] =
new SimpleObjectProvider[NormalizationContext](NoNormalization))
extends GeneralizedLinearModelLossFunction(PointwiseLogisticLossFunction, normalizationContext)

/**
* A single logistic loss function
*
* l(z, y) = - log [1/(1+exp(-z))] if this is a positive sample
*
* or - log [1 - 1/(1+exp(-z))] if this is a negative sample
*/
* A single logistic loss function
*
* l(z, y) = - log [1 / (1 + exp(-z))] if this is a positive sample
*
* - log [1 - (1 / (1 + exp(-z)))] if this is a negative sample
*/
@SerialVersionUID(1L)
object PointwiseLogisticLossFunction extends PointwiseLossFunction {
/**
* The sigmoid function 1/(1+exp(-z))
*
* @param z z
* @return The value
*/
* The sigmoid function 1 / (1 + exp(-z))
*
* @param z z
* @return The value
*/
private def sigmoid(z: Double): Double = 1.0 / (1.0 + math.exp(-z))


/**
* l(z, y) = - log [1 / (1 + exp(-z))] = log [1 + exp(-z)] if this is a positive sample
*
* - log [1 - 1/(1+exp(-z))] = log [1 + exp(z)] if this is a negative sample
*
* dl/dz = - 1 / (1 + exp(z)) if this is a positive sample
*
* 1 / (1 + exp(-z)) if this is a negative sample
*
* @param margin The margin, i.e. z in l(z, y)
* @param label The label, i.e. y in l(z, y)
* @return The value and the 1st derivative
*/
* l(z, y) = - log [1 / (1 + exp(-z))] = log [1 + exp(-z)] if this is a positive sample
*
* - log [1 - (1 / (1 + exp(-z)))] = log [1 + exp(z)] if this is a negative sample
*
* dl/dz = -1 / (1 + exp(z)) if this is a positive sample
*
* 1 / (1 + exp(-z)) if this is a negative sample
*
* @param margin The margin, i.e. z in l(z, y)
* @param label The label, i.e. y in l(z, y)
* @return The value and the 1st derivative
*/
override def loss(margin: Double, label: Double): (Double, Double) = {
if (label > 0) {
// The following is equivalent to log(1 + exp(-margin)) but more numerically stable.
Expand All @@ -71,12 +71,12 @@ object PointwiseLogisticLossFunction extends PointwiseLossFunction {
}

/**
* d^2^l/dz^2^ = sigmoid(z) * (1 - sigmoid(z))
*
* @param margin The margin, i.e. z in l(z, y)
* @param label The label, i.e. y in l(z, y)
* @return The value and the 2st derivative with respect to z
*/
* d^2^l/dz^2^ = sigmoid(z) * (1 - sigmoid(z))
*
* @param margin The margin, i.e. z in l(z, y)
* @param label The label, i.e. y in l(z, y)
* @return The value and the 2st derivative with respect to z
*/
override def d2lossdz2(margin: Double, label: Double): Double = {
val s = sigmoid(margin)
s * (1 - s)
Expand Down
9 changes: 3 additions & 6 deletions photon-ml/src/main/scala/com/linkedin/photon/ml/function/PoissonLossFunction.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,13 @@
*/
package com.linkedin.photon.ml.function


import com.linkedin.photon.ml.data.{SimpleObjectProvider, ObjectProvider}
import com.linkedin.photon.ml.data.{ObjectProvider, SimpleObjectProvider}
import com.linkedin.photon.ml.normalization.{NoNormalization, NormalizationContext}


/**
* Class for the Poisson loss function: sum_i (w_i*(exp(theta'x_i + o_i) - y_i*(theta'x_i + o_i))),
* where \theta is the coefficients of the data features to be estimated, (y_i, x_i, o_i, w_i) are the tuple
* for label, features, offset, and weight of the i'th labeled data point, respectively.
* @author asaha
* @author dpeng
*/
class PoissonLossFunction(
normalizationContext: ObjectProvider[NormalizationContext] =
Expand All @@ -42,8 +38,8 @@ class PoissonLossFunction(
object PointwisePoissonLossFunction extends PointwiseLossFunction {
/**
* l(z, y) = exp(z) - y * z
* dl/dz = exp(z) - y
*
* dl/dz = exp(z) - y
* @param margin The margin, i.e. z in l(z, y)
* @param label The label, i.e. y in l(z, y)
* @return The value and the 1st derivative
Expand All @@ -55,6 +51,7 @@ object PointwisePoissonLossFunction extends PointwiseLossFunction {

/**
* d^2^l/dz^2^ = exp(z)
*
* @param margin The margin, i.e. z in l(z, y)
* @param label The label, i.e. y in l(z, y)
* @return The value and the 2st derivative with respect to z
Expand Down
5 changes: 1 addition & 4 deletions photon-ml/src/main/scala/com/linkedin/photon/ml/function/SquaredLossFunction.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,11 @@ package com.linkedin.photon.ml.function
import com.linkedin.photon.ml.data.{ObjectProvider, SimpleObjectProvider}
import com.linkedin.photon.ml.normalization.{NoNormalization, NormalizationContext}


/**
* Class for the squared loss function: sum_i w_i/2*(theta'x_i + o_i - y_i)**2, where theta is the weight coefficients
* of the data features to be estimated, (y_i, x_i, o_i, w_i) are the label, features, offset, and weight of
* the i'th labeled data point, respectively.
* @author xazhang
* @author dpeng
*/

class SquaredLossFunction(
normalizationContext: ObjectProvider[NormalizationContext] =
new SimpleObjectProvider[NormalizationContext](NoNormalization))
Expand Down Expand Up @@ -56,6 +52,7 @@ object PointwiseSquareLossFunction extends PointwiseLossFunction {

/**
* d^2^l/dz^2^ = 1
*
* @param margin The margin, i.e. z in l(z, y)
* @param label The label, i.e. y in l(z, y)
* @return The value and the 2st derivative with respect to z
Expand Down
Loading