diff --git a/NAMESPACE b/NAMESPACE index 91dabff5..fa86367c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -8,6 +8,7 @@ S3method(logLik,cyclopsFit) S3method(predict,cyclopsFit) S3method(print,cyclopsData) S3method(print,cyclopsFit) +S3method(residuals,cyclopsFit) S3method(summary,cyclopsData) S3method(survfit,cyclopsFit) S3method(vcov,cyclopsFit) @@ -70,6 +71,7 @@ importFrom(stats,predict) importFrom(stats,qchisq) importFrom(stats,qnorm) importFrom(stats,rbinom) +importFrom(stats,residuals) importFrom(stats,rexp) importFrom(stats,rnorm) importFrom(stats,rpois) diff --git a/NEWS.md b/NEWS.md index 56f2259e..ded39d8f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +develop +============== + +1. add Schoenfeld residual output for Cox models + Cyclops v3.5.0 ============== diff --git a/R/ModelFit.R b/R/ModelFit.R index 8f382091..507e596d 100644 --- a/R/ModelFit.R +++ b/R/ModelFit.R @@ -897,7 +897,7 @@ confint.cyclopsFit <- function(object, parm, level = 0.95, #control, #' #' @param object Fitted Cyclops model object #' @param parm Specification of which parameter requires profiling, -#' either a vector of numbers of covariateId names +#' either a vector of numbers or covariateId names #' @param x Vector of values of the parameter #' @param bounds Pair of values to bound adaptive profiling #' @param tolerance Absolute tolerance allowed for adaptive profiling diff --git a/R/RcppExports.R b/R/RcppExports.R index a2987971..5df701b5 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -65,6 +65,10 @@ invisible(.Call(`_Cyclops_cyclopsLogResult`, inRcppCcdInterface, fileName, withASE)) } +.cyclopsGetSchoenfeldResiduals <- function(inRcppCcdInterface, sexpBitCovariates) { + .Call(`_Cyclops_cyclopsGetSchoenfeldResiduals`, inRcppCcdInterface, sexpBitCovariates) +} + .cyclopsGetFisherInformation <- function(inRcppCcdInterface, sexpBitCovariates) { .Call(`_Cyclops_cyclopsGetFisherInformation`, inRcppCcdInterface, sexpBitCovariates) } diff --git a/R/Residuals.R b/R/Residuals.R new file mode 100644 index 00000000..b96c229c --- /dev/null +++ b/R/Residuals.R @@ -0,0 +1,51 @@ +# @file Residuals.R +# +# Copyright 2024 Observational Health Data Sciences and Informatics +# +# This file is part of cyclops +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#' @method residuals cyclopsFit +#' @title Model residuals +#' +#' @description +#' \code{residuals.cyclopsFit} computes model residuals for Cox model-based Cyclops objects +#' +#' @param object A Cyclops model fit object +#' @param parm A specification of which parameters require residuals, +#' either a vector of numbers or covariateId names +#' @param type Character string indicating the type of residual desires. Possible +#' values are "schoenfeld". +#' +#' @importFrom stats residuals +#' +#' @export +residuals.cyclopsFit <- function(object, parm, type = "schoenfeld", ...) { + modelType <- object$cyclopsData$modelType + if (modelType != "cox") { + stop("Residuals for only Cox models are implemented") + } + if (type != "schoenfeld") { + stop("Only Schoenfeld residuals are implemented") + } + + .checkInterface(object$cyclopsData, testOnly = TRUE) + + res <- .cyclopsGetSchoenfeldResiduals(cyclopsFitRight$interface, NULL) + + result <- res$residuals + names(result) <- res$times + + return(rev(result)) +} diff --git a/man/getCyclopsProfileLogLikelihood.Rd b/man/getCyclopsProfileLogLikelihood.Rd index 9432d916..388e10a7 100644 --- a/man/getCyclopsProfileLogLikelihood.Rd +++ b/man/getCyclopsProfileLogLikelihood.Rd @@ -20,7 +20,7 @@ getCyclopsProfileLogLikelihood( \item{object}{Fitted Cyclops model object} \item{parm}{Specification of which parameter requires profiling, -either a vector of numbers of covariateId names} +either a vector of numbers or covariateId names} \item{x}{Vector of values of the parameter} diff --git a/man/residuals.cyclopsFit.Rd b/man/residuals.cyclopsFit.Rd new file mode 100644 index 00000000..cc850ea9 --- /dev/null +++ b/man/residuals.cyclopsFit.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/Predict.R +\name{residuals.cyclopsFit} +\alias{residuals.cyclopsFit} +\title{Model residuals} +\usage{ +\method{residuals}{cyclopsFit}(object, parm, type = "schoenfeld", ...) +} +\arguments{ +\item{object}{A Cyclops model fit object} + +\item{parm}{A specification of which parameters require residuals, +either a vector of numbers or covariateId names} + +\item{type}{Character string indicating the type of residual desires. Possible +values are "schoenfeld".} +} +\description{ +\code{residuals.cyclopsFit} computes model residuals for Cox model-based Cyclops objects +} diff --git a/src/RcppCyclopsInterface.cpp b/src/RcppCyclopsInterface.cpp index cad45a0f..a8a93646 100644 --- a/src/RcppCyclopsInterface.cpp +++ b/src/RcppCyclopsInterface.cpp @@ -214,6 +214,39 @@ void cyclopsLogResult(SEXP inRcppCcdInterface, const std::string& fileName, bool interface->logResultsToFile(fileName, withASE); } +// [[Rcpp::export(".cyclopsGetSchoenfeldResiduals")]] +Rcpp::DataFrame cyclopsGetSchoenfeldResiduals(SEXP inRcppCcdInterface, + const SEXP sexpBitCovariates) { + using namespace bsccs; + XPtr interface(inRcppCcdInterface); + + std::vector indices; + if (!Rf_isNull(sexpBitCovariates)) { + const std::vector& bitCovariates = as>(sexpBitCovariates); + ProfileVector covariates = reinterpret_cast&>(bitCovariates); // as(sexpCovariates); + for (auto it = covariates.begin(); it != covariates.end(); ++it) { + size_t index = interface->getModelData().getColumnIndex(*it); + indices.push_back(index); + } + } else { + indices.push_back(0); + } + if (indices.size() != 1) { + Rcpp::stop("Not yet implemented"); + } + + std::vector residuals; + std::vector times; + + interface->getCcd().getSchoenfeldResiduals(indices[0], + residuals, times); + + return DataFrame::create( + Named("residuals") = residuals, + Named("times") = times + ); +} + // [[Rcpp::export(".cyclopsGetFisherInformation")]] Eigen::MatrixXd cyclopsGetFisherInformation(SEXP inRcppCcdInterface, const SEXP sexpBitCovariates) { using namespace bsccs; diff --git a/src/RcppExports.cpp b/src/RcppExports.cpp index bf8ec7f3..b7de34a6 100644 --- a/src/RcppExports.cpp +++ b/src/RcppExports.cpp @@ -189,6 +189,18 @@ BEGIN_RCPP return R_NilValue; END_RCPP } +// cyclopsGetSchoenfeldResiduals +Rcpp::DataFrame cyclopsGetSchoenfeldResiduals(SEXP inRcppCcdInterface, const SEXP sexpBitCovariates); +RcppExport SEXP _Cyclops_cyclopsGetSchoenfeldResiduals(SEXP inRcppCcdInterfaceSEXP, SEXP sexpBitCovariatesSEXP) { +BEGIN_RCPP + Rcpp::RObject rcpp_result_gen; + Rcpp::RNGScope rcpp_rngScope_gen; + Rcpp::traits::input_parameter< SEXP >::type inRcppCcdInterface(inRcppCcdInterfaceSEXP); + Rcpp::traits::input_parameter< const SEXP >::type sexpBitCovariates(sexpBitCovariatesSEXP); + rcpp_result_gen = Rcpp::wrap(cyclopsGetSchoenfeldResiduals(inRcppCcdInterface, sexpBitCovariates)); + return rcpp_result_gen; +END_RCPP +} // cyclopsGetFisherInformation Eigen::MatrixXd cyclopsGetFisherInformation(SEXP inRcppCcdInterface, const SEXP sexpBitCovariates); RcppExport SEXP _Cyclops_cyclopsGetFisherInformation(SEXP inRcppCcdInterfaceSEXP, SEXP sexpBitCovariatesSEXP) { @@ -862,6 +874,7 @@ static const R_CallMethodDef CallEntries[] = { {"_Cyclops_cyclopsGetNewPredictiveLogLikelihood", (DL_FUNC) &_Cyclops_cyclopsGetNewPredictiveLogLikelihood, 2}, {"_Cyclops_cyclopsGetLogLikelihood", (DL_FUNC) &_Cyclops_cyclopsGetLogLikelihood, 1}, {"_Cyclops_cyclopsLogResult", (DL_FUNC) &_Cyclops_cyclopsLogResult, 3}, + {"_Cyclops_cyclopsGetSchoenfeldResiduals", (DL_FUNC) &_Cyclops_cyclopsGetSchoenfeldResiduals, 2}, {"_Cyclops_cyclopsGetFisherInformation", (DL_FUNC) &_Cyclops_cyclopsGetFisherInformation, 2}, {"_Cyclops_cyclopsSetPrior", (DL_FUNC) &_Cyclops_cyclopsSetPrior, 6}, {"_Cyclops_cyclopsTestParameterizedPrior", (DL_FUNC) &_Cyclops_cyclopsTestParameterizedPrior, 4}, diff --git a/src/cyclops/CyclicCoordinateDescent.cpp b/src/cyclops/CyclicCoordinateDescent.cpp index b2cf992b..0507700c 100644 --- a/src/cyclops/CyclicCoordinateDescent.cpp +++ b/src/cyclops/CyclicCoordinateDescent.cpp @@ -1084,8 +1084,8 @@ void CyclicCoordinateDescent::findMode( modelSpecifics.setPriorParams(temp); //modelSpecifics.resetBeta(); } - - auto cycle = [this,&lastObjFunc,&lastObjFuncVec,&iteration,algorithmType,&allDelta,&doItAll] { + + auto cycle = [this,&iteration,algorithmType,&allDelta,&doItAll] { /* if (iteration%10==0) { std::cout<<"iteration " << iteration << " "; @@ -1172,7 +1172,7 @@ void CyclicCoordinateDescent::findMode( if (delta != 0.0) { sufficientStatisticsKnown = false; updateSufficientStatistics(delta, index); - } + } } log(index); } @@ -1815,6 +1815,17 @@ void CyclicCoordinateDescent::turnOffSyncCV() { modelSpecifics.turnOffSyncCV(); } +void CyclicCoordinateDescent::getSchoenfeldResiduals(const IdType index, + std::vector& residuals, + std::vector& times) { + + checkAllLazyFlags(); + + modelSpecifics.computeSchoenfeldResiduals(index, + residuals, times, + false); +} + std::vector CyclicCoordinateDescent::getPredictiveLogLikelihood(std::vector>& weightsPool) { xBetaKnown = false; if (usingGPU && syncCV) xBetaKnown = true; diff --git a/src/cyclops/CyclicCoordinateDescent.h b/src/cyclops/CyclicCoordinateDescent.h index 36992836..e277f41f 100644 --- a/src/cyclops/CyclicCoordinateDescent.h +++ b/src/cyclops/CyclicCoordinateDescent.h @@ -130,13 +130,17 @@ class CyclicCoordinateDescent { std::vector getCensorWeights(); // ESK: + void getSchoenfeldResiduals(const IdType index, + std::vector& residuals, + std::vector& times); + void setLogisticRegression(bool idoLR); // template void setBeta(const std::vector& beta); void setStartingBeta(const std::vector& inStartingBeta); - + void setBeta(int i, double beta); // void double getHessianComponent(int i, int j); diff --git a/src/cyclops/engine/AbstractModelSpecifics.h b/src/cyclops/engine/AbstractModelSpecifics.h index 18c17b2a..020fe425 100644 --- a/src/cyclops/engine/AbstractModelSpecifics.h +++ b/src/cyclops/engine/AbstractModelSpecifics.h @@ -77,6 +77,14 @@ class AbstractModelSpecifics { virtual void computeFisherInformation(int indexOne, int indexTwo, double *oinfo, bool useWeights) = 0; // pure virtual + virtual void computeSchoenfeldResiduals(int indexOne, + std::vector& residuals, + std::vector& times, + // double* residuals, + // double* numerators, + // double* denominators, + bool useWeights) = 0; // pure virtual + virtual void updateXBeta(double realDelta, int index, bool useWeights) = 0; // pure virtual virtual void computeXBeta(double* beta, bool useWeights) = 0; // pure virtual diff --git a/src/cyclops/engine/ModelSpecifics.h b/src/cyclops/engine/ModelSpecifics.h index 52db0af7..1256a50d 100644 --- a/src/cyclops/engine/ModelSpecifics.h +++ b/src/cyclops/engine/ModelSpecifics.h @@ -197,6 +197,11 @@ class ModelSpecifics : public AbstractModelSpecifics, BaseModel { void computeFisherInformation(int indexOne, int indexTwo, double *oinfo, bool useWeights); + void computeSchoenfeldResiduals(int indexOne, + std::vector& residuals, std::vector& times, + // double* residuals, double* numerators, double* denominators, + bool useWeights); + void updateXBeta(double delta, int index, bool useWeights); void computeRemainingStatistics(bool useWeights); @@ -322,6 +327,12 @@ class ModelSpecifics : public AbstractModelSpecifics, BaseModel { template void computeFisherInformationImpl(int indexOne, int indexTwo, double *oinfo, Weights w); + template + void getSchoenfeldResidualsImpl(int index, + std::vector& residuals, + std::vector& times, + Weights w); + template SparseIterator getSubjectSpecificHessianIterator(int index); @@ -1260,7 +1271,7 @@ struct CoxProportionalHazards : public Storage, OrderedData, SurvivalP return static_cast(yi); } - template + template inline void incrementGradientAndHessian( // TODO Make static again? const IteratorType& it, Weights false_signature, diff --git a/src/cyclops/engine/ModelSpecifics.hpp b/src/cyclops/engine/ModelSpecifics.hpp index cf845ad0..71b671e5 100644 --- a/src/cyclops/engine/ModelSpecifics.hpp +++ b/src/cyclops/engine/ModelSpecifics.hpp @@ -732,6 +732,94 @@ void ModelSpecifics::getPredictiveEstimates(double* y, doubl // TODO How to remove code duplication above? } +template template +void ModelSpecifics::getSchoenfeldResidualsImpl(int index, + std::vector& residuals, + std::vector& times, + Weights w) { + + residuals.clear(); + times.clear(); + // TODO: only written for accummulive models (Cox, Fine/Grey) + + // std::cerr << "start " << index << "\n"; + + // int outcomeIndex = 0; + + // if (sparseIndices[index] == nullptr || sparseIndices[index]->size() > 0) { + + // IteratorType it(sparseIndices[index].get(), N); + IteratorType it(hX, index); + + RealType accNumerator = static_cast(0); + RealType accDenominator = static_cast(0); + + // find start relavent accumulator reset point + auto reset = begin(accReset); + while( *reset < it.index() ) { + ++reset; + } + + for (; it; ) { + int i = it.index(); + + if (*reset <= i) { + accNumerator = static_cast(0); + accDenominator = static_cast(0); + ++reset; + } + + const auto expXBeta = exp(hXBeta[i]); + + accNumerator += expXBeta * it.value(); + accDenominator += expXBeta; + + // std::cerr << hXBeta[i] << " " << expXBeta << " " << accDenominator << "\n"; + + if (hY[i] == 1) { + residuals.push_back(it.value() - accNumerator / accDenominator); + times.push_back(hOffs[i]); + // residuals[outcomeIndex] = it.value() - accNumerator / accDenominator; + // ++outcomeIndex; + } + + // residuals[i] = it.value() - accNumerator / accDenominator; + + ++it; + + if (IteratorType::isSparse) { + + const int next = it ? it.index() : N; + for (++i; i < next; ++i) { + + if (*reset <= i) { + accNumerator = static_cast(0); + accDenominator = static_cast(0); + ++reset; + } + + const auto expXBeta = exp(hXBeta[i]); + + accDenominator += expXBeta; + + if (hY[i] == 1) { + residuals.push_back(-accNumerator / accDenominator); + times.push_back(hOffs[i]); + // residuals[outcomeIndex] = -accNumerator / accDenominator; + // ++outcomeIndex; + } + + // residuals[i] = -accNumerator / accDenominator; + } + } + } + // } else { + // // TODO: fill residuals with 0s + // } + + // std::cerr << "stop\n"; +} + // TODO The following function is an example of a double-dispatch, rewrite without need for virtual function template void ModelSpecifics::computeGradientAndHessian(int index, double *ogradient, @@ -1366,6 +1454,32 @@ void ModelSpecifics::computeThirdDerivativeImpl(int index, d #endif } +template +void ModelSpecifics::computeSchoenfeldResiduals(int indexOne, + std::vector& residuals, + std::vector& times, + bool useWeights) { + if (useWeights) { + throw new std::logic_error("Weights are not yet implemented in Schoenfeld residual calculations"); + } else { // no weights + switch (hX.getFormatType(indexOne)) { + case INDICATOR : + getSchoenfeldResidualsImpl>(indexOne, residuals, times, weighted); + break; + case SPARSE : + getSchoenfeldResidualsImpl>(indexOne, residuals, times, weighted); + break; + case DENSE : + getSchoenfeldResidualsImpl>(indexOne, residuals, times, weighted); + break; + case INTERCEPT : + getSchoenfeldResidualsImpl>(indexOne, residuals, times, weighted); + break; + } + } +} + + template void ModelSpecifics::computeFisherInformation(int indexOne, int indexTwo, double *oinfo, bool useWeights) {