Skip to content

Latest commit

 

History

History
1790 lines (1388 loc) · 42 KB

code.org

File metadata and controls

1790 lines (1388 loc) · 42 KB

headers

#include <RcppArmadillo.h>
using namespace Rcpp;
using namespace arma;
// [[Rcpp::depends(RcppArmadillo)]]

utils

 // log probability of r given p and lambda

 // log likelihood kernel sans constants in data r1!, r2!, ...
 inline double llkr( int rsum, int n, double p, double lambda ){
   double llk;
   if (ISNA(lambda)) {
     llk = 0.0;
   }
   else
     {
	if (lambda<DBL_MIN) lambda = DBL_MIN;
	llk = (double) rsum * (log(lambda)+log(p)) -
	  (double) n*(lambda*p + log1p(-exp(-lambda*p)));
     }
   return llk;
 }

 // lambda vectorized log probability of r given p and lambda

 inline arma::rowvec logprob_p( int r, double p, arma::rowvec lambda ){
   rowvec result(lambda.size());
   for (int i = 0L; i<lambda.size(); i++) 
     result[i] = llkr( r, 1L, p, lambda[i] );
   return result;
 }    

 // p vectorized log probability of r given p and lambda

  inline arma::rowvec logprob_l( int r, arma::rowvec p,  double lambda ){
   rowvec result(p.size());
   for (int i = 0L; i<p.size(); i++) 
     result[i] = llkr( r, 1L, p[i], lambda );
   return result;
 }    
 // log probability vector (sans multinomial coefficient)

 inline arma::rowvec logprob(arma::irowvec& tabrow, arma::mat& om,
			      arma::mat& eta, arma::rowvec& etaN,
			      int etaLast, double lambda){
   rowvec logpr(etaLast);
   int J = eta.n_rows;
   int K = om.n_cols;
   for (int rc = 0; rc<etaLast; rc++){
     double rhosum = 0.0;
     int tabsum = 0L;
     double logprc = 0.0;

     for (int k = 0; k<K; k++){
	tabsum+=tabrow(k);
	double rhoelt = 0.0;
	for (int j=0;j<J;j++) rhoelt+=om(j,k)*eta(j,rc);
	logprc+= tabrow(k) * log(rhoelt);
	rhosum+= rhoelt;
     }
     logprc-= (double) tabsum * log(rhosum);
     logprc+= llkr( tabsum, 1L, rhosum, lambda );
     logprc+= log( etaN(rc));
     logpr(rc) = logprc;
   }
   return logpr;
 }
// uniform dirichlet random numbers

inline vec rdirich( int n, double dprior=1.0 ){
  vec rg( n );
  for (int i = 0; i<n; i++) {
    rg(i)=Rf_rgamma(dprior, 1.0);
  }
  return rg/sum(rg);
}
// sample one index
inline int newIndex(arma::rowvec logpr){
  double maxlogpr = max(logpr);
  double prcum = 0.0;
  for (int i =0;i<logpr.size(); i++){
    prcum += exp(logpr(i)-maxlogpr);
    logpr(i) = prcum;
  }  
  double ur = Rf_runif(0.0,prcum);
  int index=0L;
  for (; logpr[index] < ur && index<logpr.size(); index++);
  return index;
}
 #define LOWLIM 8
 <<piy>>
 <<piky>>

 inline double log_piy(double y, double rho, double alpha, double beta){
   double log_pi0 = log_pi_ky( 0, y, rho, alpha, beta);
   double pitotal = 1.0 +
     exp(log_pi_Y_approx(y, (double) LOWLIM - 0.5,  rho, alpha, beta) - log_pi0);
   for (int i = 1; i<LOWLIM; i++){
     pitotal +=
	exp( log_pi_ky( i, y, rho, alpha, beta) - log_pi0 );
   }
   return log_pi0 + log(pitotal);
 }
 #define LOWLIM 8

 inline double piy(double y, double rho, double alpha, double beta){
   double piwhy =  exp(log_pi_Y_approx(y, (double) LOWLIM - 0.5,  rho, alpha, beta));
   for (int i = 0; i<LOWLIM; i++){
     piwhy +=
	exp( log_pi_ky( i, y, rho, alpha, beta));
   }
   return piwhy;
 }
inline double log_pi_Y_approx(double y, double lowlim, double rho, double alpha, double beta){
  return log(beta) * alpha - lgamma(alpha) + log(rho)*(y-1.0)
    - LOGFACTORIAL(y) + lgamma( alpha + y ) - log (alpha + y - 1.0)
    - log( beta + rho * (lowlim + 1.0)) * (alpha + y - 1.0);
}
inline double log_pi_ky(int k, double y,  double rho, double alpha, double beta){
  return log(beta)*alpha - lgamma(alpha) +
    log(rho) * y - LOGFACTORIAL(y) + lgamma(alpha+y) -
    log(beta+rho*(((double) k + 1.0))) * (alpha+y);
}
 #define INDEF_INT( x )   -pow((beta+rho*(1+x)),-(alpha+r-1))/(rho*(alpha+r-1))

 inline double rkReject (double rho,double alpha,double beta, int r,int z){
   vec kpz(z+1L, fill::zeros);
   double kpztot = 0.0;
   int k;
   for (k = 0L; k < z; k++){
     kpz[k] = pow(beta+rho*(1+k),-(alpha+r));
     kpztot += kpz[k];
   }
   kpz[ z ] = -INDEF_INT(z-0.5);
   kpztot += kpz[z];
   bool  reject = true;
   while( reject ){
     double kx = Rf_runif(0.0,kpztot);
     double kpsum=kpz[ 0L ];
     for (k = 0L; k<z && kpsum < kx; k++)
	kpsum += kpz[k+1L];
     if (k==z){
	double x = Rf_runif(0.0,1.0);
	double kcand =
	  (beta/rho + z + 0.5)/pow(x, 1.0/(alpha+r-1.0)) - beta/rho - 0.5;
	kcand = Rf_ftrunc(kcand);
	double v = INDEF_INT(kcand+0.5)-INDEF_INT(kcand-0.5);
	double y = Rf_runif(0.0,v);
	reject = y >= pow(beta+rho*(1+kcand),-(alpha+r));
	k = (int) kcand;
     } else {
	reject = false;

     }
   }
   return (double) k;
 }

This next function takes a lot of time. Maybe it helps to trim it.

#define KMAX 5L

inline double rlamGivenR(double rho,double alpha,double beta,int r){

  double indx = rkReject(rho, alpha, beta, r, KMAX);

  double rg = Rf_rgamma( alpha+r, 1.0/(beta+rho*(1+indx)));

  return rg;

}

Resizing is rather time consuming.

 /*
   update a matrix and associated N vector in place
   Author: Charles C. Berry
   Date: 12-04-2020

 */

 #define MORESIZE 10L
 #define MAXSIZE 500L


 int updateXX( int newind,
		 int i,
		 arma::mat& XX,
		 arma::rowvec& XXN,
		 arma::ivec& diToXX,
		 int XXN1,	// singleton flag
		 int& XXM,
		 int& decXXN,  
		 int& incXXNnew,
		 int& incXXNold,
		 int auxXXM,
		 int verbose)
 {

   int ndat = diToXX.n_elem;
   int di2XX = diToXX[ i ];
   int XXSize = XX.n_cols;

   if (XXN1){
     //singleton case
     if (verbose>2L) Rprintf("singleton\n");
     if (newind == di2XX)
	{
	  // retain di2XX
	  XXN( di2XX ) = 1;
	}
     else if (newind < XXM)
	{
	  // use existing element in place of this one and move di2e
	  XXN( newind )++;
	  diToXX( i ) = newind;
	  // shift left
	  XXN.shed_col(di2XX);
	  XX.shed_col(di2XX);
	  if (verbose>2L)
	    Rprintf("XXN.n_cols=%d XX.n_cols=%d\n", XXN.n_cols, XX.n_cols);
	  XXM--;
	  XXSize--;
	  decXXN++;
	  for (int idi=0; idi<ndat; idi++) 
	    if (diToXX[ idi ] >= di2XX) diToXX[ idi ]--;
	  if (verbose > 2L) Rprintf("diToXX=%d\n",diToXX[ i ]);
	}
     else
	{
	  // use new element
	  // copy to di2XX
	  XXN( di2XX ) = 1; incXXNnew++;
	  XX.col( di2XX ) = XX.col( newind );
	} 
   }
   else
     // initial run or XXN[ di2XX ] >= 2
     {
	if (verbose>2L) Rprintf("initial run or N>=2\n");
	if (newind >= XXM)
	  {
	    //use new element
	    XXN( XXM ) =1;
	    if (newind>XXM) XX.col(XXM) = XX.col(newind);
	    if (verbose>2L) Rprintf("XX(0,XXM)=%f\n", XX(0,XXM));
	    diToXX( i ) = XXM;
	    XXM++;
	    incXXNnew++;
	    if (verbose>2L) Rprintf("XXM=%d\n", XXM);
	  }
	else
	  {
	    // use existing element
	    XXN( newind )++;
	    diToXX( i ) = newind;
	    incXXNold++;
	  }

     }
   // check size and pad as needed
   if (XXM+auxXXM > XXSize){
     if (verbose) {
	Rprintf("XX has %d Elts ", XX.n_elem);
	Rprintf("XXSize = %d XXM = %d auxXXM = %d\n",
		XXSize, XXM, auxXXM);
     }
     int addSize = MORESIZE;
     if (XXSize + addSize <= MAXSIZE){
	XXSize += addSize;
	XX.resize( XX.n_rows, XXSize ); 
	XXN.resize( XXSize );
     } else {
	Rcpp::stop("Cannot resize XX");
     }
     if (verbose) Rprintf("XX has %d Elts\n", XX.n_elem);
   }
   return XXSize;
 }
inline void rmultnm(int n, double* prob, int k, int* rn){
  double prsum = 0.0;
  for (int i=0; i<k; i++) prsum += prob[i];
  for (int i=0; i<k; i++) prob[i] /= prsum;
  Rf_rmultinom(n, prob,k,rn);
}
inline int rN0GivenN1( int N1, double lambda, double rhoplus ){
  return  Rf_rnbinom( (double) N1, -expm1(-lambda * rhoplus ) );
}

auxGibbs

 /*
   auxGibbs.cpp

   Auxiliary Gibbs Sampler for negative multinomial sampler of cell
   type proportions.

   Author: Charles C. Berry
   Date:
   28-07-2020
   24-01-2020
   10-01-2020
   22-04-2019
   16-06-2019
 */

 /* assume
    imat tab = wtab["tab"];
    ivec di = wtab["data.index"];
    di = di - 1L;
    di, dataTo[Eta|Lambda] are zero based
 */

 // [[Rcpp::export]]
 List auxGibbs(arma::imat& tab, arma::ivec& di, arma::mat& om, 
		arma::mat eta,
		arma::rowvec etaN,
		arma::ivec diToEta,
		arma::rowvec lambda,
		arma::rowvec lambdaN,
		arma::ivec diToLambda,
		int etaM = 0L,
		int auxM = 5L, double alpha = 100.0,
		int lambdaM = 0L,
		int auxLambdaM = 1L, double alphaLambda = 5.0,
		int ijvals = 0L,
		int verbose = 0L,
		double dprior=1.0,
		double lambdaShape=1.0,
		double lambdaRate=0.01) {
   // we get a list from R
   // pull std::vector<double> from R list
   // this is achieved through an implicit
   // call to Rcpp::as

   int etaCols = eta.n_cols;
   int lambdaSize = lambda.size();
   int J = om.n_rows;
   int ndat = di.size();
   int decN = 0L;
   int incNnew = 0L;
   int incNold = 0L;

   int decLambdaN = 0L;
   int incLambdaNnew = 0L;
   int incLambdaNold = 0L;

   for (int i=ijvals;
	 i<ndat && etaM+auxM <= etaCols && lambdaM+auxLambdaM <= lambdaSize;
	 i++){

     if (verbose>1L) Rprintf("i = %d",i);

     int di2e = diToEta[ i ];
     int etaN1; // singletons need one less 
     if (di2e >= 0L && etaN( di2e ) == 1.0){
	etaN1 = 1;
	etaN( di2e ) = alpha/auxM;
     }
     else
	{ etaN1 = 0; 
	  if (di2e>=0L) etaN( di2e )--;
	}

     int di2lam = diToLambda[ i ];
     int lambdaN1; // singletons need one less 
     if (di2lam >= 0 && lambdaN( di2lam ) == 1.0){
	lambdaN1 = 1;
	lambdaN( di2lam ) = alphaLambda;
     }
     else
	{ lambdaN1 = 0; 
	  if (di2lam>=0L) lambdaN( di2lam )--;
	}

     // sample auxM from prior
     for (int j = 0; j < auxM-etaN1; j++){
	eta.col(j + etaM ) = rdirich(J, dprior);
	etaN( j+etaM ) = alpha/auxM;
     }

     // rho and logprob
     // initially use lambdaVal = NA_REAL;
     double lambdaVal = (di2lam < 0 ) ? NA_REAL : lambda( di2lam );

     irowvec tr = tab.row(di( i ));
     int newind =
	newIndex(logprob( tr, om, eta, etaN,
			  etaM + auxM - etaN1, lambdaVal));

     // update-eta

     etaCols = updateXX(  newind, i, eta, etaN, diToEta, etaN1, etaM,
		 decN, incNnew, incNold, auxM, verbose);

     // update-lambda

     // sample lambdaM from posterior
     double rhosum = (double) accu( trans(eta.col(newind))*om );
     int tabsum = arma::sum( tr );
     int lambdaElts = (auxLambdaM==0 || lambdaN1) ? lambdaM : lambdaM + 1;
     arma::rowvec lambdaProbs =
	logprob_p( tabsum, rhosum, lambda.head(lambdaElts) ) +
	log( lambdaN.head( lambdaElts));
     int intFGindx = lambdaN1 ? di2lam : lambdaM;
     if (lambdaN1 || auxLambdaM){
	lambdaProbs(intFGindx) =
	  log_piy((double) tabsum , rhosum, lambdaShape, lambdaRate) + log(alphaLambda);
     }

     newind = newIndex(lambdaProbs);

     if (newind == intFGindx)
	  lambda(newind) = rlamGivenR( rhosum, lambdaShape, lambdaRate, tabsum);

     lambdaSize = updateXX(  newind, i, lambda, lambdaN, diToLambda, lambdaN1, lambdaM,
		 decLambdaN, incLambdaNnew, incLambdaNold, auxLambdaM, verbose);
     if (verbose>1L) Rprintf(" done\n");
   }
   if (verbose)  {
     Rprintf("delete Eta= %d add = %d use existing = %d ",
	      decN, incNnew, incNold);
     Rprintf("delete Lambda= %d add = %d use existing = %d\n",
	      decLambdaN, incLambdaNnew, incLambdaNold);
   }

   // return an R list; this is achieved
   // through an implicit call to Rcpp::wrap
   return List::create(_["eta"] = eta,
			_["etaN"] = etaN,
			_["dataToEta"] = diToEta,
			_["etaM"] = etaM,
			_["lambda"] = lambda,
			_["lambdaN"] = lambdaN,
			_["dataToLambda"] = diToLambda,
			_["lambdaM"] = lambdaM
			);
 }

tests

Many src blocks can use Rcpp::sourceCpp(code=”…”).

However, strings with embedded backslash escapes will cause issues and sometimes errors.

To obviate those, tangling to a temp file and using Rcpp::sourceCpp(“tempfilename.cpp”) should work.

rdirich

Rcpp::sourceCpp(code='
<<arma-headers>>
<<rdirich>>
// [[Rcpp::export]]
vec call_rdirich( int n, double dprior=1.0 ){
return rdirich( n, dprior );}
')

## test here
set.seed(123)
cr <- call_rdirich(5L,1.0)
set.seed(123)
rr <- prop.table(rgamma(5,1))

if (all(cr==rr)) "PASS" else "FAIL"

logprob_Rplus

Rcpp::sourceCpp(code='
<<arma-headers>>
<<logprob_Rplus>>
// [[Rcpp::export]]
double call_llkr( int rsum, int n, double p, double lambda ){
return llkr( rsum, n, p,  lambda );}
// [[Rcpp::export]]
arma::rowvec call_logprob_p( int r, double p, arma::rowvec lambda ){
  return logprob_p(  r,  p,  lambda );}
// [[Rcpp::export]]
arma::rowvec call_logprob_l( int r, arma::rowvec p,  double lambda ){
return logprob_l( r, p,  lambda );}
')

## test here
R_llkr <- function(rsum, n, p, lambda){   
  lambda <- pmax(.Machine$double.xmin, lambda)
  res <-   rsum * (log(lambda)+log(p)) -
    n*(lambda*p + log1p(-exp(-lambda*p)))
  res[is.na(res)] <- 0.0
  res
}

cllkr <- call_logprob_p(3L,0.8,c(1.2,NA))
rllkr <- R_llkr( 3L, 1L, 0.8, c(1.2,NA))

if (all(cllkr == rllkr )) "PASS" else "FAIL"

cllkr <- call_logprob_l(3L,c(0.8,0.9),1.2)
rllkr <- R_llkr( 3L, 1L, c(0.8,0.9), 1.2)

if (all(cllkr == rllkr )) "PASS" else "FAIL"

logprob_Multi

Rcpp::sourceCpp(code='
<<arma-headers>>
<<logprob_Rplus>>
<<logprobMulti>>
// [[Rcpp::export]]
arma::rowvec call_logprob(arma::irowvec& tabrow, arma::mat& om,
arma::mat& eta, arma::rowvec& etaN,
int etaLast, double lambda){
return logprob(tabrow, om, eta, etaN, etaLast, lambda);}
')
## test here

## logprob(tabrow, om, eta, etaN, etaLast, lambda);}

tabrow <- c(1,2,3)
om<- (diag(3)+.05)/2
eta <- prop.table(cbind(1:3,1,3:1),2)
etaN <- 1:3
etaLast <- 3
lambda <- 2.0


clp <- call_logprob(tabrow, om, eta, etaN, etaLast, lambda)

Rlogprob <- function(tabrow, om, eta, etaN, etaLast, lambda){
  tabsum <- sum(tabrow)
  rho <- t( t(eta)%*%om )
  logprc <- tabrow %*% log(prop.table(rho,2)) +
    R_llkr(tabsum, 1L, colSums(rho), lambda) +
    log( etaN )
  logprc
}

Rlp <- Rlogprob(tabrow, om, eta, etaN, etaLast, lambda)

if (isTRUE(all.equal(clp,Rlp))) "PASS" else "FAIL"

newIndex

Rcpp::sourceCpp(code='
<<arma-headers>>
<<newIndex>>
// [[Rcpp::export]]
int call_newIndex(arma::rowvec logpr){
  return newIndex(arma::rowvec logpr);}
')

## test here

rkReject

    Rcpp::sourceCpp(code='
<<arma-headers>>
<<rkReject>>
// [[Rcpp::export]]
  double call_rkReject (double rho, double alpha, double beta, int r=1,int z=4){
 return rkReject ( rho, alpha, beta,  r, z);
}
')

rlamGivenR

Rcpp::sourceCpp(code='
<<arma-headers>>
<<rkReject>>
<<rlamGivenR>>
// [[Rcpp::export]]
double call_rlamGivenR(double rho,double alpha,double beta,int r){
double res;
res = rlamGivenR( rho, alpha, beta, r);
return res;
}
')

## test here
rlambdaGivenR <- function(rho, alpha, beta, r){
  KMAX <- 1000L
  lkp <-  -(r+alpha)*log(beta+rho*(1.0+0:KMAX));
  indx <- sample(0:KMAX,1,prob=exp(lkp))
  rgamma(1L, alpha+r, rate=(beta+rho*(1+indx)))
}

set.seed(1234)
clam <- replicate(1000, call_rlamGivenR(.8, 1.0,.01,1))
rlam <- replicate(1000,rlambdaGivenR(.8, 1.0,.01,1))

all.equal(rlam,clam)
microbenchmark::microbenchmark(call_rlamGivenR(.8, 1.0,.01,1))

## apparently setting KMAX too a smaller value will do the trick, so
## adaptively figure out what value to use.
## look at rk.reject to figure this out.

updateXX

<<arma-headers>>
<<updateXX>>
// [[Rcpp::export]]
List call_updateXX( int newind, int i, List xlist){
  arma::mat XX = xlist["XX"];
  arma::rowvec XXN = xlist["XXN"];
  arma::ivec diToXX = xlist["diToXX"];
  int XXN1 = xlist["XXN1"];
  IntegerVector XXM = xlist["XXM"];
  IntegerVector decXXN = xlist["decXXN"];
  IntegerVector incXXNnew = xlist["incXXNnew"];
  IntegerVector incXXNold = xlist["incXXNold"];
  int auxXXM = xlist["auxXXM"];
  int verbose = xlist["verbose"];
  updateXX(newind, i, XX, XXN, diToXX, XXN1, XXM[0L],
	     decXXN[0L], incXXNnew[0L], incXXNold[0L], auxXXM, verbose);
  return List::create(
			_["XX"] = XX,
			_["XXN"] = XXN,
			_["diToXX"] = diToXX,
			_["XXN1"] = XXN1,
			_["XXM"] = XXM,
			_["decXXN"] = decXXN,
			_["incXXNnew"] = incXXNnew,
			_["incXXNold"] = incXXNold,
			_["auxXXM"] = auxXXM,
			_["verbose"] = verbose);
}
Rcpp::sourceCpp("nobuild/test-updateXX.cpp")
## TODO: this exercises updateXX, but requires manual inspection of
## results to verify correctness

xlist <- 
  list(
    i = 0L,
    XX = matrix(as.double(1:30),nrow=3),
    XXN = rep(0.0, 10),
    diToXX = rep(-1L,5),
    XXN1 = 0L,
    XXM = 0L,
    decXXN = 0L,
    incXXNnew = 0L,
    incXXNold = 0L,
    auxXXM = 5L,
    verbose = 3L
  )

vlist <- 
  list(
    XX = matrix(as.double(1:10),nrow=1),
    XXN = rep(0.0, 10),
    diToXX = rep(-1L,5),
    XXN1 = 0L,
    XXM = 0L,
    decXXN = 0L,
    incXXNnew = 0L,
    incXXNold = 0L,
    auxXXM = 5L,
    verbose = 3L
  )


ylist <- rlang::duplicate(xlist)

## choose 1
zlist <- ylist

zlist <- vlist

## cases  (4, 5, 4, 5, 4)  
newinds <- c(4L,0L,5L,1L,6L)


for (i in 0L:4L){
  zlist <- call_updateXX(newinds[i+1],i,zlist)
}


## update singleton existing case 2 

i  <- 4L
d2x <- zlist$diToXX[ i + 1L ]
zlist$XXN[ d2x+1L ] <-   zlist$XXN[ d2x+1L ] - 1.0
zlist$XXN1  <- if (zlist$XXN[ d2x + 1L] == 0L) 1L else 0L
zlist <- call_updateXX(0L,i,zlist)

## add new case 4

i  <- 4L
d2x <- zlist$diToXX[ i + 1L ]
zlist$XXN[ d2x+1L ] <-   zlist$XXN[ d2x+1L ] - 1.0
zlist$XXN1  <- if (zlist$XXN[ d2x + 1L] == 0L) 1L else 0L
zlist <- call_updateXX(5L,i,zlist)

## revise in place case 1

d2x <- zlist$diToXX[ i + 1L ]
zlist$XXN[ d2x+1L ] <-   zlist$XXN[ d2x+1L ] - 1.0
zlist$XXN1  <- if (zlist$XXN[ d2x + 1L] == 0L) 1L else 0L
zlist <- call_updateXX(2L,i,zlist)

## remove early singleton case 2

ni <- 1L
for (i in 0:1){
  d2x <- zlist$diToXX[ i + 1L ]
  zlist$XXN[ d2x+1L ] <-   zlist$XXN[ d2x+1L ] - 1.0
  zlist$XXN1  <- if (zlist$XXN[ d2x + 1L] == 0L) 1L else 0L
  zlist <- call_updateXX(ni, i, zlist)
  }


## update singleton new case 3
i <- 4L
d2x <- zlist$diToXX[ i + 1L ]
zlist$XXN[ d2x+1L ] <-   zlist$XXN[ d2x+1L ] - 1.0
zlist$XXN1  <- if (zlist$XXN[ d2x + 1L] == 0L) 1L else 0L
zlist <- call_updateXX(2L,i,zlist)



## test resizing

i <- 0L


for (j in 1:20){
  ## use last
  d2x <- zlist$diToXX[ i + 1L ]
  zlist$XXN[ d2x+1L ] <-   zlist$XXN[ d2x+1L ] - 1.0
  zlist$XXN1  <- if (zlist$XXN[ d2x + 1L] == 0L) 1L else 0L
  zlist <- call_updateXX(length(zlist$XXN)-1L, i, zlist)  
  ## discard last
  d2x <- zlist$diToXX[ i + 1L ]
  zlist$XXN[ d2x+1L ] <-   zlist$XXN[ d2x+1L ] - 1.0
  zlist$XXN1  <- if (zlist$XXN[ d2x + 1L] == 0L) 1L else 0L
  zlist <- call_updateXX(1L, i, zlist)
}

auxGibbs

<<arma-headers>>
<<rdirich>>
<<logprob_Rplus>>
<<logprobMulti>>
<<newIndex>>
<<rkReject>>
<<rlamGivenR>>
<<updateXX>>
<<auxGibbs>>
Rcpp::sourceCpp("nobuild/test-auxGibbs.cpp")

The results here seems plausible. But keep an eye on the lambda values for small r.

source("R/gibbsDPP.R")
load("~/projects/bushman/WAS/derep-11-17/wttabs.etc.RData")
wtab <- wttabs[[11]]
om <- with(param.list[[11]],diag(upsilon)%*%omega%*%diag(psi))

tmp <- gibbsDPP(wtab,om,verbose=1L)
str(tmp)

tmp <- gibbsDPP(wtab,om,alphaEta=1.0,alphaLambda=0.5, verbose=1L)

with(tmp,cbind(lambda,lambdaN)[order(lambda),])

rmultnm

Rcpp::sourceCpp(code='
<<arma-headers>>
<<rmultnm>>
// [[Rcpp::export]]
ivec call_rmultnm(int n, NumericVector prob){
int k = LENGTH(prob);
ivec rn(k);
rmultnm(n, REAL(prob), k, &rn[0]);
  return rn;}
  ')
## test here
set.seed(123)
cr <- replicate(1000,call_rmultnm(5L,1:5))
set.seed(123)
rr <- replicate(1000, rmultinom(1,5,1:5))
if (all(cr==rr)) "PASS" else "FAIL"

rN0GivenN1

Rcpp::sourceCpp(
	  code ='
<<arma-headers>>
<<rN0GivenN1>>
// [[Rcpp::export]]
int call_rN0GivenN1( int N1, double lambda, double rhoplus ){
return rN0GivenN1( N1,  lambda, rhoplus );
}
')

rho <- 0.8
lambda <- 1.0
prob <- -expm1(-rho*lambda)

set.seed(123)
if (isTRUE(all.equal(
  3 * (1-prob)/prob,
  mean(replicate(10000,call_rN0GivenN1(3,lambda,rho))),
  tol=0.01))) "PASS" else "FAIL"
microbenchmark::microbenchmark(call_rN0GivenN1(3,lambda,rho))

int FG

Checks on the integrals are done here:

 Rcpp::sourceCpp(code="
 <<arma-headers>>
 #define LOGFACTORIAL(x) lgamma(x+1.0) 
 <<piy>>
 <<piky>>
 <<intFG>>
 // [[Rcpp::export]]
 double intFG(double y, double rho, double alpha, double beta){
 return piy(y, rho, alpha, beta);
 }
 ")


 pi.ky <- function(k,y,rho,alpha,beta)
     beta^alpha /
	gamma(alpha) *
	rho^y/factorial(y) *
	gamma(alpha+y) /
	(beta+rho*(k+1))^(alpha+y)
 ## this approximates the sum from k to Inf
 int.pi.ky <- function(k,y,rho,alpha,beta)
   beta^alpha/ gamma(alpha) * rho^(y-1) / factorial(y) *
     gamma(alpha+y)/(alpha+y-1) /
     (beta + rho*(k+1))^(alpha+y-1)



 tmp <- (pi.ky(0,1:100,.5,1,.1))+
   (pi.ky(1,1:100,.5,1,.1))+
   (pi.ky(2,1:100,.5,1,.1))+
   (pi.ky(3,1:100,.5,1,.1))+
   (int.pi.ky(3.5, 1:100, .5, 1,.1))

 tmp2  <- sapply(1:100,function(x) intFG(x,.5,1,.1))

 all.equal(tmp,tmp2)

 ## kernel only

 Rcpp::sourceCpp(code="
 <<arma-headers>>
 #define LOGFACTORIAL(x) 0.0 
 <<piy>>
 <<piky>>
 <<intFG>>
 // [[Rcpp::export]]
 double intFG2(double y, double rho, double alpha, double beta){
 return piy(y, rho, alpha, beta);
 }
 ")

 tmp3  <- sapply(1:100,function(x) intFG2(x,.5,1,.1))
 all.equal(tmp3/factorial(1:100),tmp2)


 Rcpp::sourceCpp(code="
 <<arma-headers>>
 #define LOGFACTORIAL(x) lgamma(x+1.0) 
 <<logIntFG>>
 // [[Rcpp::export]]
 double logIntFG(double y, double rho, double alpha, double beta){
 return log_piy(y, rho, alpha, beta);
 }
 ")

 tmp4  <- sapply(1:100,function(x) logIntFG(x,.5,1,.1))
 all.equal(exp(tmp4),tmp2)


 intFG3 <- function(y,rho,alpha,beta,tol=1e-5){
   fun <- function(x) dpois(y,rho*x)/ppois(0,rho*x,lower.tail=FALSE)*dgamma(x,alpha,rate=beta)
   res1 <- integrate(fun,0,10,rel.tol=tol)
   res2 <- integrate(fun,10,Inf,rel.tol=tol)
   res1$value+res2$value
 }

 # the integrate function is close
 tmp5 <- sapply(1:100, function(x) intFG3(x,.5,1,.1,tol=1e-12))
 all.equal(tmp5,tmp3/factorial(1:100),tolerance=0.001)

prob k

 Rcpp::sourceCpp(code=
		    "
 <<arma-headers>>
 #define LOGFACTORIAL(x) lgamma(x+1.0) 
 <<piky>>
 // [[Rcpp::export]]
 double piky(int k, double y, double rho, double alpha, double beta){
  return exp(log_pi_ky(k,y,rho,alpha,beta));
 }")

 pi.ky2 <- function(k,y,rho,alpha,beta){
     lres  <-log(beta)*alpha -
	lgamma(alpha) +
	log(rho)*y - lfactorial(y) +
	lgamma(alpha+y) -
	log(beta+rho*(k+1))*(alpha+y)
     exp(lres)
 }

 all.equal(pi.ky2(0:10,1,.5,1,.1),
	    sapply(0:10,function(x) piky(x,1.0,0.5,1.0,0.1)))

Approximate integral

Rcpp::sourceCpp(code=
		   "
<<arma-headers>>
#define LOGFACTORIAL(x) lgamma(x+1.0) 
<<piy>>
// [[Rcpp::export]]
double logpiy(double y, double lowlim, double rho, double alpha, double beta){
return log_pi_Y_approx(y, lowlim, rho, alpha, beta);}
")

log.int.pi.ky <- function(k,y,rho,alpha,beta)
  log(beta)*alpha - lgamma(alpha) + log(rho)*(y-1) - lfactorial(y) +
    lgamma(alpha+y)-log(alpha+y-1) -
    log( beta + rho*(k+1) )*(alpha+y-1)

all.equal(
  log.int.pi.ky(-0.5, 1:10, .5, 1,.1),
  sapply(1:10, function(x) logpiy(x,-0.5,0.5,1.0,0.1)))
pi.y.giv <- function(y,lambda,rho,alpha,beta)
  dpois(y,rho*lambda)/ppois(0,rho*lambda,lower.tail=FALSE)*dgamma(lambda,alpha,rate=beta)

pi.y.giv2 <- function(y,lambda,rho,alpha,beta)
  exp(-rho*lambda)*(rho*lambda)^y/(1-exp(-rho*lambda))*beta^alpha/gamma(alpha)*lambda^(alpha-1)*exp(-lambda*beta)/factorial(y)

pi.y.giv3 <- function(lambda,k,y,rho,alpha,beta)
  beta^alpha /
    gamma(alpha) *
    lambda^(alpha-1) *
    (rho*lambda)^y *
    exp(-rho*lambda) *
    sum(exp(-(k)*rho*lambda)) *
    exp(-lambda*beta)/factorial(y)



pi.y.giv(1,.5,.5,.1,.1)
pi.y.giv2(1,.5,.5,.1,.1)
sum(pi.y.giv3(.5,0:100,1,.5,.1,.1))

intFG <- function(y,rho,alpha,beta,tol=1e-5){
  fun <- function(x) dpois(y,rho*x)/ppois(0,rho*x,lower.tail=FALSE)*dgamma(x,alpha,rate=beta)
  res1 <- integrate(fun,0,10,rel.tol=tol)
  res2 <- integrate(fun,10,Inf,rel.tol=tol)
  res1$value+res2$value
}

intFG(1,.1,.1,.01)

intFG(10,.1,.1,.01)
sum(sapply(1:1000,function(x) intFG(x,.1,.1,.1)))
debugonce(intFG)
intFG(50,.1,.1,.01)
intFG(50,.1,.1,.01,1e-15)

pi.lky <- function(lambda,k,y,rho,alpha,beta)
  beta^alpha / gamma(alpha) * (rho*lambda)^y * lambda^(alpha-1)*exp(-lambda*(beta+rho*(k+1)))/factorial(y)

pi.lky(.5,0,1,.5,.1,.1)
pi.y.giv3(.5,0,1,.5,.1,.1)


sum(pi.lky(0.5,1,1:100,.5,1,.1))

pi.lky2 <- function(x,k,y,rho,alpha,beta)
  dpois(y,rho*x*(k+1))*dgamma(x,alpha,rate=beta)


pi.ky <- function(k,y,rho,alpha,beta)
  beta^alpha /
    gamma(alpha) *
    rho^y/factorial(y) *
    gamma(alpha+y) /
    (beta+rho*(k+1))^(alpha+y)

all.equal(
  integrate(function(x) pi.lky(x,0,1,.5,.1,.1),0,Inf)$value,
  pi.ky(0,1,.5,.1,.1),
  tol=1e-5)

## more accurate version
pi.ky2 <- function(k,y,rho,alpha,beta){
  lres  <-log(beta)*alpha -
    lgamma(alpha) +
    log(rho)*y - lfactorial(y) +
    lgamma(alpha+y) -
    log(beta+rho*(k+1))*(alpha+y)
  exp(lres)
}


intFG(1,.5,1,.1)
sum(pi.ky(0:1000,1,.5,1,.1))

## this approximates the sum from k to Inf
int.pi.ky <- function(k,y,rho,alpha,beta)
  beta^alpha/ gamma(alpha) * rho^(y-1) / factorial(y) *
    gamma(alpha+y)/(alpha+y-1) /
    (beta + rho*(k+1))^(alpha+y-1)

int.pi.ky(-0.5, 1, .5, 1,.1)

sum(sapply(1:100,function(x) sum(pi.ky(0:100,x,.5,1,.1))))


int.pi.ky(9.5, 1, .5, 1,.1)-int.pi.ky(999.5,1,.5,1,.1)

sum(pi.ky(10:1000,1,.5,1,.1))

all.equal(int.pi.ky(-0.5, 1, .5, 1,.1),
	    integrate(function(x) pi.ky(x,1,.5,1,.1),-0.5,Inf)$value)

all.equal(int.pi.ky(-0.5, 2, .5, 1,.1),
	    integrate(function(x) pi.ky(x,2,.5,1,.1),-0.5,Inf)$value)

all.equal(int.pi.ky(9.5, 1, .5, 1,.1),
	    integrate(function(x) pi.ky(x,1,.5,1,.1),10-0.5,Inf)$value)

## should this sum < one? No. it is not the sum over k, but an approx
int.pi.ky(-0.5, 1:10, .5, 1,.1)

## this comes close
sum(pi.ky(0,1:100,.5,1,.1))+
  sum(pi.ky(1,1:100,.5,1,.1))+
  sum(pi.ky(2,1:100,.5,1,.1))+
    sum(pi.ky(3,1:100,.5,1,.1))+
  sum(int.pi.ky(3.5, 1:100, .5, 1,.1))

log.int.pi.ky <- function(k,y,rho,alpha,beta)
  log(beta)*alpha - lgamma(alpha) + log(rho)*(y-1) - lfactorial(y) +
    lgamma(alpha+y)-log(alpha+y-1) -
    log( beta + rho*(k+1) )*(alpha+y-1)

log.int.pi.ky(-0.5, 1:10, .5, 1,.1)

## compare approx to numerical intgral via integrate
tmp <- (pi.ky(0,1:100,.5,1,.1))+
  (pi.ky(1,1:100,.5,1,.1))+
  (pi.ky(2,1:100,.5,1,.1))+
  (pi.ky(3,1:100,.5,1,.1))+
  (int.pi.ky(3.5, 1:100, .5, 1,.1))

tmp2  <- sapply(1:100,function(x) intFG(x,.5,1,.1,tol=1e-14))

## this shows breakdown in integrate for very small values - viz the
## nearly linear plot breaks at less than 1e-8. Using bigger tol moves
## the breakpoint to larger values

plot(tmp,tmp2,log='xy')

sampleParms

<<arma-headers>>
<<rmultnm>>
<<rN0GivenN1>>
<<sampleParms>>
Rcpp::sourceCpp("nobuild/sampleParms.cpp")

ept <- with(wtab, xtabs(tab[data.index,]~tmp$dataToEta))

tmp0 <- rlang::duplicate(tmp)

res <- 
  with(tmp,
	 sampleParms(wtab$tab, wtab$data.index-1L, om, dataToEta-1L, dataToLambda-1L,
		     eta, etaM, lambda, lambdaM, dprior=1.0,
		     lambdaAlpha=1.0, lambdaBeta=0.01,
		     verbose = 1L))


res0 <- res
res <- rlang::duplicate(tmp)

llk <- NULL
system.time(for (i in 1:100){
  res <- with(tmp,
		sampleParms(wtab$tab, wtab$data.index-1L, om,
			    dataToEta-1L, dataToLambda-1L,
			    res$eta, etaM, res$lambda, lambdaM, dprior=1.0,
			    lambdaAlpha=1.0, lambdaBeta=0.01, niter=20L,
			    verbose = 1L))
  llk <- c(llk,sum(ept*log(prop.table(t(with(res,eta[,1:etaM]))%*%om,1))))
  if (anyNA(res)) break
  res2 <- res
})


ols <- prop.table(pmax(solve(t(om),t(ept)),1.0),2)

print(sum(ept*t(log(prop.table(ols,2)))))
print(sum(ept*t(log(prop.table(with(tmp,eta[,1:etaM]),2)))))

plot(llk,type='l')

tmp$eta <- res$eta
tmp$lambda <- res$lambda

## rmultnm, rN0GivenN1, and tuneScan needed:
res3 <- tuneScan(wtab$tab,wtab$data.index,om,tmp)
summary(res3$logLik)

##
proc.time()
tmp  <- gibbsScan(wtab,om,nkeep=1L,alphaEta=10L,alphaLambda=0.1,
		    auxEtaM=1L,auxLambdaM=1L,verbose=TRUE)
proc.time()

tmp2  <- update(tmp,auxEtaM=1L,auxLambdaM=1L,verbose=TRUE)
proc.time()

tmp2  <- update(tmp,auxLambdaM=10L,verbose=TRUE)
proc.time()

tmp2  <- update(tmp,auxEtaM=10L,verbose=TRUE)
proc.time()



##

proc.time()
tmp  <- gibbsScan(wtab,om,nkeep=1L,alphaEta=10L,alphaLambda=0.1,
		    etaCols=150L,lambdaSize=100L,
		    verbose=TRUE)
proc.time()



res4 <- gibbsScan(wtab,om)

res5 <- gibbsScan(wtab,om,nkeep=5L,nburn=20L,alphaLambda=0.1,abLambda=c(0.1,50.0),verbose=TRUE)

with(res5[[5]],cbind(lambda,lambdaN))

with(res5[[5]],cbind(round(t(eta),3),etaN))

with(res5[[5]],plot(hclust(dist(t(eta))),labels=FALSE))

res6 <- gibbsScan(wtab,om,nkeep=5L,nthin=5L,alphaLambda=0.1,abLambda=c(0.1,50.0),verbose=TRUE)

res7 <- update(res6,auxEtaM=0L,auxLambdaM=0L)
res8 <- update(res7,auxEtaM=5L,auxLambdaM=5L)
res9 <- update(res8,auxEtaM=0L,auxLambdaM=0L)


sapply(ls(pat="^res"),function(x) sapply(get(x),function(y) y$logpost))

## this matches up with abbreviated version implemented in gibbsScan
sum(dpois(wtab$tab[wtab$data.index,],
	    with(res4[[1]],(t(eta)%*%om)[dataToEta,] * lambda[dataToLambda]),
	    log=TRUE))-
  sum(ppois(0,
	      with(res4[[1]],rowSums(t(eta)%*%om)[dataToEta]*lambda[dataToLambda]),
	      lower.tail=FALSE,log=TRUE))


ppois(0,3,lower.tail=F)
dpois(0,3)

elt <- with(res4[[1]],table(dataToEta,dataToLambda))
lpt <- xtabs(rowSums(wtab$tab)[wtab$data.index]~res4[[1]]$dataToLambda)
rtab <- with(wtab,tab[data.index,])
dev <- sum(log( res4[[1]]$lambda ) * lpt)
dev2 <- sum(t(elt) *
	      with(res4[[1]],
		   (lambda %o% rowSums(t(eta)%*%om)) +
		   log(1.0 - exp(-lambda%o% rowSums(t(eta)%*%om)))))
dev3 <- sum( rtab * with(res4[[1]],log(t(eta)%*%om)[dataToEta,]))
dev4 <- sum(with(wtab,lfactorial(tab[data.index,])))

c(dev, dev2, dev3, dev4)

dev - dev2 + dev3 - dev4
 tmp0 <- tmp


 tuneScan <- function(tab, data.index, uop, scan, nreps=50L,
		       dprior=1.0, lambdaAlpha=1.0, lambdaBeta=0.01){

   etaLambdaTab <- with(scan,table(dataToEta,dataToLambda))
   lambdaPostTab <- xtabs(rowSums(tab[data.index,])~ scan$dataToLambda)
   etaPostTab <- xtabs(tab[data.index,]~ scan$dataToEta)


   llk <- NULL
   ## update eta
   for (irep in 1:nreps){
     llk <- c(llk,sum(log(prop.table(with(scan,t(eta[,1:etaM]))%*%uop,1))*etaPostTab))
     lambdaN0 <- rep(0.0,nrow(etaLambdaTab))
     for (eta_indx in 1:nrow(etaLambdaTab)){
	eta_elt <- with(scan,eta[,eta_indx])  
	rhoplus_elt <- sum(eta_elt%*%uop)
	for (j in 1:ncol(etaLambdaTab)){
	  if (etaLambdaTab[eta_indx,j] != 0L)
	    lambdaN0[eta_indx] <- lambdaN0[eta_indx] + scan$lambda[j]*
	      (rN0GivenN1(etaLambdaTab[eta_indx,j],scan$lambda[j], rhoplus_elt) +
	       etaLambdaTab[eta_indx,j])
	}
     }

   ## etaM by J results 
     lambdaRhoComp <-
	t((1.0-rowSums(uop)) * with(scan,eta[,1:etaM])) * lambdaN0

     etaVisTab <- sapply( 1:scan$etaM, function(eta_indx){ 
	eta_elt <- with(scan,eta[,eta_indx])  
	rho_elt <- uop*eta_elt
	rowSums(sapply(1:5, function(k)
	  call_rmultnm(etaPostTab[eta_indx,k],rho_elt[,k])))
     })

     etaInvisTab <- array(rpois(lambdaRhoComp,lambdaRhoComp),dim(lambdaRhoComp))
     etaTab <- etaVisTab + t( etaInvisTab )
     etaUpdate <- prop.table(array(rgamma(etaTab,etaTab+dprior),dim(etaTab)),2)
     scan$eta[,1:scan$etaM] <- etaUpdate

     ## update lambda

     lambdaN1 <- lambdaN0 <- array(0.0,dim(etaLambdaTab))

     for (eta_indx in 1:nrow(etaLambdaTab)){
	eta_elt <- with(scan,eta[,eta_indx])  
	rhoplus_elt <- sum(eta_elt%*%uop)
	for (j in 1:ncol(etaLambdaTab)){
	  if (etaLambdaTab[eta_indx,j] != 0L)
	    lambdaN0[eta_indx,j] <- rhoplus_elt *
	      rN0GivenN1(etaLambdaTab[eta_indx,j],scan$lambda[j], rhoplus_elt)
	  lambdaN1[eta_indx,j] <- rhoplus_elt * etaLambdaTab[eta_indx,j]    
	}
     }

     lambdaUpdate <- with(scan,
			   rgamma(lambdaPostTab, lambdaAlpha+lambdaPostTab,
				  rate = lambdaBeta + colSums(lambdaN1 + lambdaN0)))

     scan$lambda[1:scan$lambdaM] <- lambdaUpdate

   }

   llk <- c(llk,sum(log(prop.table(with(scan,t(eta[,1:etaM]))%*%uop,1))*etaPostTab))

   scan$logLik <- llk
   scan
 }

 res <- tuneScan(wtab$tab,wtab$data.index,om,tmp0)
 res3 <- tuneScan(wtab$tab,wtab$data.index,om,res)
 plot(res$logLik)

 plot(llk[-(1:20)])

 diff(colMeans(matrix(res2$logLik[-1],nc=10)))

 acf(llk[-(1:40)])
 ### checks

 list(
   round(cbind(prop.table(with(tmp,t(eta[,1:etaM]))%*%om,1),
		prop.table(with(tmp0,t(eta[,1:etaM]))%*%om,1)), 3),
   round(prop.table(etaPostTab,1),3),
   round(log(prop.table(with(tmp,t(eta[,1:etaM]))%*%om,1))*etaPostTab
	  -log(prop.table(with(tmp0,t(eta[,1:etaM]))%*%om,1))*etaPostTab,1)
 )

 sum(log(prop.table(with(tmp,t(eta[,1:etaM]))%*%om,1))*etaPostTab)

 sum(log(prop.table(with(tmp0,t(eta[,1:etaM]))%*%om,1))*etaPostTab)

 ## round(cbind(lambdaUpdate,with(tmp,lambda[1:lambdaM]),lambdaPostTab,colSums(lambdaN1),colSums(lambdaN0)),3)

 ## neutrophils under counted?

cpp

Use :main no to suppress treating code as a standalone program

<<arma-headers>>
#define LOGFACTORIAL(x) 0.0 
<<rdirich>>
<<logprob_Rplus>>
<<logIntFG>>
<<logprobMulti>>
<<newIndex>>
<<rkReject>>
<<rlamGivenR>>
<<updateXX>>
<<auxGibbs>>
<<rmultnm>>
<<rN0GivenN1>>
<<sampleParms>>
Rcpp::sourceCpp("src/ct.cpp")
"done"
 source("R/gibbsDPP.R")
 load("~/projects/bushman/WAS/derep-11-17/wttabs.etc.RData")
 wtab <- wttabs[[11]]
 om <- with(param.list[[11]],diag(upsilon)%*%omega%*%diag(psi))

 tmp <- gibbsScan(wtab,om)

 keepLogLik <- 
   function(pass, log.posterior, i, nkeep){
     if ( i == nkeep )
	keep.default(pass,log.posterior,i,nkeep)
     else
	list(logLik = log.posterior)
     }
 }

 tmp2  <- gibbsScan(wtab,om,nkeep=10L,keep=keepLogLik)

 tableM <- function(){
   avgs <- list(NULL)
   function(pass,log.posterior, i , nkeep){
     avgs[[(i-1)%/%10+1]] <<-
	if (i%%10 == 1) log.posterior
     else
	avgs[[(i-1)%/%10+1]] + log.posterior
     if (i == nkeep) list(avgs = unlist(avgs)/10,
			   keep.default(pass,log.posterior,i,nkeep)
			   )
   }
 }

 tabFun <- tableM()

 tmp3  <- gibbsScan(wtab,om,nkeep=30L,keep=tabFun)

tuneup

 // assume di, dataTo[ Eta | Lambda ] are zero based indexes

 // [[Rcpp::export]]
 List sampleParms(
		   arma::imat& tab,
		   arma::ivec& di,
		   arma::mat& om,
		   arma::ivec dataToEta,
		   arma::ivec dataToLambda,
		   arma::mat eta, int etaM, 
		   arma::rowvec lambda, int lambdaM,
		   double dprior,
		   double lambdaAlpha, double lambdaBeta,
		   int niter = 5L, int verbose=0L){
   if (verbose>1L) Rprintf("starting....\n");
   int J = om.n_rows;
   int ndat = di.size();
   ivec rplus = sum( tab, 1L);
   imat etaLambdaTab(etaM, lambdaM, fill::zeros );
   ivec lambdaPostTab(lambdaM, fill::zeros );
   imat etaPostTab(etaM, J, fill::zeros );

   for (int idat = 0; idat < ndat; idat++){
     etaLambdaTab( dataToEta(idat) , dataToLambda(idat))++;
     lambdaPostTab( dataToLambda(idat) ) += rplus(di(idat));
     etaPostTab.row( dataToEta(idat) ) += tab.row(di(idat));
   }

   for (int iter = 0L; iter<niter; iter++){
     vec lambdaN0( etaM , fill::zeros);

     for (int ieta = 0L; ieta<etaM; ieta++)
	for (int ilambda = 0L; ilambda < lambdaM; ilambda++)
	  if (etaLambdaTab(ieta, ilambda) != 0L){
	    lambdaN0(ieta) +=
	      lambda(ilambda) * ((double)
				 rN0GivenN1(etaLambdaTab(ieta,ilambda),
					    lambda(ilambda),
					    sum(trans(eta.col(ieta))*om))
				 + (double) etaLambdaTab(ieta,ilambda));
	  }

     mat lambdaRhoComp = trans((1.0 - sum(om,1L)) % eta.head_cols(etaM).each_col());
     lambdaRhoComp.each_col() %= lambdaN0;

     mat etaTab( J, etaM, fill::zeros);
     for (int ieta = 0L; ieta<etaM; ieta++){
	mat XY = diagmat(eta.col(ieta)) * om;
	for (int iY = 0; iY<J; iY++){
	  ivec X(J, fill::zeros);
	  rmultnm(etaPostTab(ieta,iY), XY.colptr(iY), J, X.memptr());
	  etaTab.col(ieta) += conv_to<vec>::from(X);
	}
	for (int iX = 0L; iX<J; iX++){
	  etaTab(iX,ieta) += Rf_rpois(lambdaRhoComp(ieta, iX));
	  etaTab(iX,ieta) = Rf_rgamma(etaTab(iX,ieta)+dprior, 1.0);
	}
     }

     rowvec etaColSum = sum(etaTab, 0L);
     etaTab.each_row() /= etaColSum;

     eta.head_cols(etaM) = etaTab;

     mat lambdaN1(etaM, lambdaM, fill::zeros );

     for (int ieta = 0L; ieta < etaM; ieta++){
	double rhoplus =  sum(trans(eta.col(ieta))*om);
	for (int ilambda = 0L; ilambda <lambdaM; ilambda++)
	  if (etaLambdaTab(ieta,ilambda) != 0L)
	    lambdaN1(ieta, ilambda) =
	      rhoplus * (
			 (double) rN0GivenN1(etaLambdaTab(ieta,ilambda),
					     lambda(ilambda), rhoplus) +
			 (double) etaLambdaTab(ieta,ilambda));
     }

     rowvec lambdaUpdate(lambdaM);
     for (int ilambda = 0L; ilambda<lambdaM; ilambda++)
	lambdaUpdate(ilambda) =
	  Rf_rgamma(lambdaAlpha + (double) lambdaPostTab(ilambda),
		    1.0/(lambdaBeta + sum(lambdaN1.col(ilambda))));

     lambda.head(lambdaM) = lambdaUpdate;
   }

   double dev = as_scalar(log(lambda.head(lambdaM)) * lambdaPostTab);
   mat rho = trans(eta.head_cols(etaM))*om;
   for (int ieta = 0L; ieta<etaM; ieta++){
     double rhoplus =  sum(rho.row(ieta));
     for (int ilambda = 0L; ilambda<lambdaM; ilambda++)
	dev -=
	  etaLambdaTab(ieta, ilambda) *
	  (lambda(ilambda)*rhoplus +
	   log( 1.0 - exp( - lambda(ilambda)*rhoplus)));
     for (int j = 0L; j<J; j++) dev += etaPostTab(ieta,j)*log(rho(ieta,j));
   }

   return List::create(_["eta"] = eta,
			_["lambda"] = lambda,
			_["etaM"] = etaM,
			_["lambdaM"] = lambdaM,
			_["logLik"] = dev);
 }


deviance

scratch

library(cellTypeCompositions,lib="../cellTypeCompositions.Rcheck")
// [[Rcpp::export]]
double xlogy(ivec x, vec y){
  int n = x.size();
  double totl = 0.0;
  for (int i = 0L; i<n; i++)
    if (x(i)!=0) totl+= (double) x(i) * log(y(i));
  return totl;
}
// [[Rcpp::export]]
double xlogy2(ivec x, vec y){
  int n = x.size();
  double totl = 0.0;
  for (int i = 0L; i<n; i++)
    totl+= (double) x(i) * log(y(i));
  return totl;
}
Rcpp::sourceCpp(code=
		    '
<<arma-headers>>
<<xlogy>>
')


x <- rep(0:1,c(1600,400))
y <- rexp(2000)

microbenchmark::microbenchmark(xlogy(x,y),xlogy2(x,y))

cpp doc

//
// basic Armadillo headers:
//
<<arma-headers>>
//
// the combinatorial constant is usually not needed 
//
#define LOGFACTORIAL(x) 0.0
//
// sample from n-1 simplex with rep(d,n) as Dirichlet param
//
<<rdirich>>
//
// truncated Poisson
//
<<logprob_Rplus>>
//
// truncated Gamma-Poisson integral
//
<<logIntFG>>
//
// multinomial probability kernel
//
<<logprobMulti>>
//
// sample 1 value given log probs 
//
<<newIndex>>
//
// auxiliary varible (unseen trials)
//
<<rkReject>>
//
// sample lambda using auxiliary variable
//
<<rlamGivenR>>
//
// manage updating DPP objects with current draw
//
<<updateXX>>
//
// scan data, draw eta and lambda for each 
//
<<auxGibbs>>
//
// normalize prob, then sample multinomial
//
<<rmultnm>>
//
// negative binomial draw given lambda, rhoplus
//
<<rN0GivenN1>>
//
// update eta and lambda parameters
//
<<sampleParms>>