From 10910e2cf7514ce3ba9a10e973cd6759b30c5d67 Mon Sep 17 00:00:00 2001 From: Lilin Yin Date: Fri, 2 Aug 2024 18:08:29 +0800 Subject: [PATCH] v1.1.0 --- DESCRIPTION | 4 +- R/MVP.Data.r | 116 ++++++------- R/MVP.FarmCPU.r | 53 +++--- R/MVP.GLM.r | 12 +- R/MVP.K.VanRaden.r | 235 ++++++++++++++------------ R/MVP.MLM.r | 21 ++- R/MVP.PCA.r | 17 +- R/MVP.r | 23 +-- R/RcppExports.R | 12 +- README.md | 117 ++++++------- man/MVP.Data.Bfile2MVP.Rd | 5 +- man/MVP.Data.Hapmap2MVP.Rd | 2 +- man/MVP.Data.Kin.Rd | 10 +- man/MVP.Data.Numeric2MVP.Rd | 5 +- man/MVP.Data.PC.Rd | 10 +- man/MVP.Data.Rd | 5 +- man/MVP.Data.VCF2MVP.Rd | 2 +- man/MVP.FarmCPU.Rd | 3 + man/MVP.GLM.Rd | 4 +- man/MVP.K.VanRaden.Rd | 15 +- man/MVP.MLM.Rd | 3 + man/MVP.PCA.Rd | 5 +- man/MVP.Rd | 4 +- src/Makevars | 7 +- src/Makevars.win | 7 +- src/RcppExports.cpp | 39 +++-- src/assoc.cpp | 202 ++++------------------ src/data_converter.cpp | 159 +++++++++--------- src/impute.cpp | 8 +- src/kinship.cpp | 152 +++++++++++++---- src/rMVP.h | 322 ++++++++++++++++++++++++++++++++++++ 31 files changed, 951 insertions(+), 628 deletions(-) mode change 100755 => 100644 src/RcppExports.cpp create mode 100644 src/rMVP.h diff --git a/DESCRIPTION b/DESCRIPTION index fb7efcf..c99b4b3 100755 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -2,8 +2,8 @@ Package: rMVP Type: Package Title: Memory-Efficient, Visualize-Enhanced, Parallel-Accelerated GWAS Tool -Version: 1.0.7 -Date: 2023-08-24 +Version: 1.1.0 +Date: 2024-08-01 Authors@R: c( person("Lilin", "Yin", role = "aut"), person("Haohao", "Zhang", role = "aut"), person("Zhenshuang", "Tang", role = "aut"), diff --git a/R/MVP.Data.r b/R/MVP.Data.r index bf5d268..6bd9ae9 100755 --- a/R/MVP.Data.r +++ b/R/MVP.Data.r @@ -39,8 +39,7 @@ #' @param type.geno type parameter in bigmemory, genotype data. The default is char, it is highly recommended *NOT* to modify this parameter. #' @param pheno_cols Extract which columns of the phenotype file (including individual IDs) #' @param SNP.impute "Left", "Middle", "Right", or NULL for skip impute. -#' @param maxLine number of SNPs, only used for saving memory when calculate kinship matrix -#' @param priority "speed" or "memory" +#' @param maxLine the number of markers handled at a time, smaller value would reduce the memory cost #' @param pcs.keep how many PCs to keep #' @param verbose whether to print detail. #' @param ncpus The number of threads used, if NULL, (logical core number - 1) is automatically used @@ -68,10 +67,10 @@ MVP.Data <- function(fileMVP = NULL, fileVCF = NULL, fileHMP = NULL, fileBed = NULL, fileNum = NULL, fileMap = NULL, filePhe = NULL, fileInd = NULL, fileKin = NULL, filePC = NULL, out = "mvp", sep.num = "\t", auto_transpose = TRUE, sep.map = "\t", sep.phe = "\t", sep.kin = "\t", sep.pc = "\t", - type.geno = "char", pheno_cols = NULL, SNP.impute = "Major", maxLine = 10000, priority = "speed", + type.geno = "char", pheno_cols = NULL, SNP.impute = "Major", maxLine = 10000, pcs.keep = 5, verbose = TRUE, ncpus = NULL, ...) { - logging.initialize("MVP.Data", dirname(out)) + # logging.initialize("MVP.Data", dirname(out)) logging.log("Preparing data for MVP...\n", verbose = verbose) @@ -113,6 +112,7 @@ MVP.Data <- function(fileMVP = NULL, fileVCF = NULL, fileHMP = NULL, fileBed = N stop("Both Map and Numeric genotype files are needed!") } } + switch(flag, # fileMVP, fileVCF, fileHMP, fileBed, fileNum, fileMap # TODO: Fix it @@ -135,7 +135,6 @@ MVP.Data <- function(fileMVP = NULL, fileVCF = NULL, fileHMP = NULL, fileBed = N bfile = fileBed, out = out, maxLine = maxLine, - priority = priority, type.geno = type.geno, verbose = verbose, threads = ncpus @@ -146,7 +145,6 @@ MVP.Data <- function(fileMVP = NULL, fileVCF = NULL, fileHMP = NULL, fileBed = N map_file = fileMap, out = out, maxLine = maxLine, - priority = priority, type.geno = type.geno, auto_transpose = auto_transpose, verbose = verbose @@ -181,9 +179,9 @@ MVP.Data <- function(fileMVP = NULL, fileVCF = NULL, fileHMP = NULL, fileBed = N K <- MVP.Data.Kin( fileKin = fileKin, mvp_prefix = out, - priority = priority, + maxLine = maxLine, sep = sep.kin, - cpus = ncpus + cpu = ncpus ) } @@ -194,9 +192,9 @@ MVP.Data <- function(fileMVP = NULL, fileVCF = NULL, fileHMP = NULL, fileBed = N mvp_prefix = out, K = K[,], pcs.keep = pcs.keep, - priority = priority, + maxLine = maxLine, sep = sep.pc, - cpus = ncpus + cpu = ncpus ) } @@ -212,7 +210,7 @@ MVP.Data <- function(fileMVP = NULL, fileVCF = NULL, fileHMP = NULL, fileBed = N #' #' @param vcf_file Genotype in VCF format #' @param out the name of output file -#' @param maxLine the max number of line to write to big matrix for each loop +#' @param maxLine the number of markers handled at a time, smaller value would reduce the memory cost #' @param type.geno the type of genotype elements #' @param threads number of thread for transforming #' @param verbose whether to print the reminder @@ -242,7 +240,7 @@ MVP.Data.VCF2MVP <- function(vcf_file, out='mvp', maxLine = 1e4, type.geno='char scan <- vcf_parser_map(vcf_file = vcf_file, out = out) m <- scan$m n <- scan$n - logging.log(paste0("inds: ", n, "\tmarkers:", m, '\n'), verbose = verbose) + logging.log(paste0("inds: ", n, "\tmarkers: ", m, '\n'), verbose = verbose) # parse genotype bigmat <- filebacked.big.matrix( @@ -254,6 +252,8 @@ MVP.Data.VCF2MVP <- function(vcf_file, out='mvp', maxLine = 1e4, type.geno='char descriptorfile = descriptorfile, dimnames = c(NULL, NULL) ) + + logging.log(paste0("Loading genotype at a step of ", maxLine, '...\n'), verbose = verbose) vcf_parser_genotype(vcf_file = vcf_file, pBigMat = bigmat@address, maxLine = maxLine, threads = threads, verbose = verbose) t2 <- as.numeric(Sys.time()) logging.log("Preparation for GENOTYPE data is done within", format_time(t2 - t1), "\n", verbose = verbose) @@ -266,8 +266,7 @@ MVP.Data.VCF2MVP <- function(vcf_file, out='mvp', maxLine = 1e4, type.geno='char #' #' @param bfile Genotype in binary format (.bed, .bim, .fam) #' @param out the name of output file -#' @param maxLine the max number of line to write to big matrix for each loop -#' @param priority 'memory' or 'speed' +#' @param maxLine the number of markers handled at a time, smaller value would reduce the memory cost #' @param type.geno the type of genotype elements #' @param threads number of thread for transforming #' @param verbose whether to print the reminder @@ -284,7 +283,7 @@ MVP.Data.VCF2MVP <- function(vcf_file, out='mvp', maxLine = 1e4, type.geno='char #' MVP.Data.Bfile2MVP(bfilePath, tempfile("outfile"), threads=1) #' } #' -MVP.Data.Bfile2MVP <- function(bfile, out='mvp', maxLine=1e4, priority='speed', type.geno='char', threads=0, verbose=TRUE) { +MVP.Data.Bfile2MVP <- function(bfile, out='mvp', maxLine=1e4, type.geno='char', threads=0, verbose=TRUE) { t1 <- as.numeric(Sys.time()) bim_file <- normalizePath(paste0(bfile, '.bim'), mustWork = TRUE) fam_file <- normalizePath(paste0(bfile, '.fam'), mustWork = TRUE) @@ -303,7 +302,7 @@ MVP.Data.Bfile2MVP <- function(bfile, out='mvp', maxLine=1e4, priority='speed', n <- nrow(fam) write.table(fam[, 2], paste0( out, '.geno.ind'), row.names = FALSE, col.names = FALSE, quote = FALSE) - logging.log(paste0("inds: ", n, "\tmarkers:", m, '\n'), verbose = verbose) + logging.log(paste0("inds: ", n, "\tmarkers: ", m, '\n'), verbose = verbose) # parse genotype bigmat <- filebacked.big.matrix( @@ -316,7 +315,7 @@ MVP.Data.Bfile2MVP <- function(bfile, out='mvp', maxLine=1e4, priority='speed', dimnames = c(NULL, NULL) ) - if (priority == "speed") { maxLine <- -1 } + logging.log(paste0("Loading genotype at a step of ", maxLine, '...\n'), verbose = verbose) read_bfile(bed_file = bed_file, pBigMat = bigmat@address, maxLine = maxLine, threads = threads, verbose = verbose) t2 <- as.numeric(Sys.time()) logging.log("Preparation for GENOTYPE data is done within", format_time(t2 - t1), "\n", verbose = verbose) @@ -329,7 +328,7 @@ MVP.Data.Bfile2MVP <- function(bfile, out='mvp', maxLine=1e4, priority='speed', #' #' @param hmp_file Genotype in Hapmap format #' @param out the name of output file -#' @param maxLine the max number of line to write to big matrix for each loop +#' @param maxLine the number of markers handled at a time, smaller value would reduce the memory cost #' @param type.geno the type of genotype elements #' @param threads number of thread for transforming #' @param verbose whether to print the reminder @@ -360,7 +359,7 @@ MVP.Data.Hapmap2MVP <- function(hmp_file, out='mvp', maxLine = 1e4, type.geno='c scan <- hapmap_parser_map(hmp_file[1], out) m <- scan$m n <- scan$n - logging.log(paste0("inds: ", n, "\tmarkers:", m, '\n'), verbose = verbose) + logging.log(paste0("inds: ", n, "\tmarkers: ", m, '\n'), verbose = verbose) # parse genotype bigmat <- filebacked.big.matrix( @@ -372,6 +371,8 @@ MVP.Data.Hapmap2MVP <- function(hmp_file, out='mvp', maxLine = 1e4, type.geno='c descriptorfile = descriptorfile, dimnames = c(NULL, NULL) ) + + logging.log(paste0("Loading genotype at a step of ", maxLine, '...\n'), verbose = verbose) hapmap_parser_genotype(hmp_file = hmp_file, Major = scan$Major, pBigMat = bigmat@address, maxLine = maxLine, threads = threads, verbose = verbose) t2 <- as.numeric(Sys.time()) logging.log("Preparation for GENOTYPE data is done within", format_time(t2 - t1), "\n", verbose = verbose) @@ -385,8 +386,7 @@ MVP.Data.Hapmap2MVP <- function(hmp_file, out='mvp', maxLine = 1e4, type.geno='c #' @param num_file Genotype in Numeric format (0,1,2) #' @param map_file Genotype map file, SNP_name, Chr, Pos #' @param out the name of output file -#' @param maxLine the max number of line to write to big matrix for each loop -#' @param priority 'memory' or 'speed' +#' @param maxLine the number of markers handled at a time, smaller value would reduce the memory cost #' @param row_names whether the numeric genotype has row names #' @param col_names whether the numeric genotype has column names #' @param type.geno the type of genotype elements @@ -405,7 +405,7 @@ MVP.Data.Hapmap2MVP <- function(hmp_file, out='mvp', maxLine = 1e4, type.geno='c #' MVP.Data.Numeric2MVP(numericPath, mapPath, tempfile("outfile")) #' } #' -MVP.Data.Numeric2MVP <- function(num_file, map_file, out='mvp', maxLine=1e4, priority='speed', row_names=FALSE, col_names=FALSE, type.geno='char', auto_transpose=TRUE, verbose=TRUE) { +MVP.Data.Numeric2MVP <- function(num_file, map_file, out='mvp', maxLine=1e4, row_names=FALSE, col_names=FALSE, type.geno='char', auto_transpose=TRUE, verbose=TRUE) { t1 <- as.numeric(Sys.time()) num_file <- normalizePath(num_file, mustWork = TRUE) map_file <- normalizePath(map_file, mustWork = TRUE) @@ -426,7 +426,7 @@ MVP.Data.Numeric2MVP <- function(num_file, map_file, out='mvp', maxLine=1e4, pri transposed <- TRUE t <- n; n <- m; m <- t; } - logging.log(paste0("inds: ", n, "\tmarkers:", m, '\n'), verbose = verbose) + logging.log(paste0("inds: ", n, "\tmarkers: ", m, '\n'), verbose = verbose) # define bigmat bigmat <- filebacked.big.matrix( @@ -438,32 +438,33 @@ MVP.Data.Numeric2MVP <- function(num_file, map_file, out='mvp', maxLine=1e4, pri descriptorfile = descriptorfile, dimnames = c(NULL, NULL) ) + logging.log(paste0("Loading genotype at a step of ", maxLine, '...\n'), verbose = verbose) # convert to bigmat - speed - if (priority == "speed") { - opts <- options(bigmemory.typecast.warning = FALSE) - on.exit(options(opts)) + # if (priority == "speed") { + # opts <- options(bigmemory.typecast.warning = FALSE) + # on.exit(options(opts)) - # detecte sep - con <- file(num_file, open = 'r') - line <- readLines(con, 1) - close(con) - sep <- substr(line, 2, 2) + # # detecte sep + # con <- file(num_file, open = 'r') + # line <- readLines(con, 1) + # close(con) + # sep <- substr(line, 2, 2) - # load geno - suppressWarnings( - geno <- read.big.matrix(num_file, header = FALSE, sep = sep) - ) - if (transposed) { - bigmat[, ] <- t(geno[, ]) - } else { - bigmat[, ] <- geno[, ] - } - rm("geno") - } + # # load geno + # suppressWarnings( + # geno <- read.big.matrix(num_file, header = FALSE, sep = sep) + # ) + # if (transposed) { + # bigmat[, ] <- t(geno[, ]) + # } else { + # bigmat[, ] <- geno[, ] + # } + # rm("geno") + # } # convert to bigmat - memory - if (priority == "memory") { + # if (priority == "memory") { i <- 0 con <- file(num_file, open = 'r') if (col_names) { readLines(con, n = 1) } @@ -488,7 +489,7 @@ MVP.Data.Numeric2MVP <- function(num_file, map_file, out='mvp', maxLine=1e4, pri logging.log(paste0("Written into MVP File: ", percent, "%"), verbose = verbose) } close(con) - } + # } file.copy(map_file, paste0(out, ".geno.map")) @@ -700,9 +701,9 @@ MVP.Data.Map <- function(map, out='mvp', cols=1:5, header=TRUE, sep='\t', verbos #' @param K Kinship matrix #' @param out prefix of output file name #' @param pcs.keep how many PCs to keep -#' @param priority speed or memory +#' @param maxLine the number of markers handled at a time, smaller value would reduce the memory cost #' @param sep seperator for PC file. -#' @param cpus the number of cpu +#' @param cpu the number of cpu #' @param verbose whether to print detail. #' #' @export @@ -714,7 +715,7 @@ MVP.Data.Map <- function(map, out='mvp', cols=1:5, header=TRUE, sep='\t', verbos #' \donttest{ #' geno <- file.path(system.file("extdata", "06_mvp-impute", package = "rMVP"), "mvp.imp") #' -#' MVP.Data.PC(TRUE, mvp_prefix=geno, out=tempfile("outfile"), cpus=1) +#' MVP.Data.PC(TRUE, mvp_prefix=geno, out=tempfile("outfile"), cpu=1) #' } #' MVP.Data.PC <- function( @@ -723,9 +724,9 @@ MVP.Data.PC <- function( K=NULL, out=NULL, pcs.keep=5, - priority='speed', + maxLine=10000, sep='\t', - cpus=1, + cpu=1, verbose=TRUE ){ if (is.null(out)) out <- mvp_prefix @@ -740,9 +741,9 @@ MVP.Data.PC <- function( } else if (filePC == TRUE) { if(is.null(K)){ geno <- attach.big.matrix(paste0(mvp_prefix, ".geno.desc")) - myPC <- MVP.PCA(M=geno, pcs.keep = pcs.keep, priority=priority, cpu=cpus) + myPC <- MVP.PCA(M=geno, pcs.keep = pcs.keep, maxLine = maxLine, cpu=cpu) }else{ - myPC <- MVP.PCA(K=K, pcs.keep = pcs.keep, priority=priority, cpu=cpus) + myPC <- MVP.PCA(K=K, pcs.keep = pcs.keep, maxLine = maxLine, cpu=cpu) } } else { stop("ERROR: The value of filePC is invalid.") @@ -769,9 +770,9 @@ MVP.Data.PC <- function( #' @param fileKin Kinship that represents relationship among individuals, n * n matrix, n is sample size #' @param mvp_prefix Prefix for mvp format files #' @param out prefix of output file name -#' @param priority "speed" or "memory" +#' @param maxLine the number of markers handled at a time, smaller value would reduce the memory cost #' @param sep seperator for Kinship file. -#' @param cpus the number of cpu +#' @param cpu the number of cpu #' @param verbose whether to print detail. #' #' @export @@ -783,16 +784,16 @@ MVP.Data.PC <- function( #' \donttest{ #' geno <- file.path(system.file("extdata", "06_mvp-impute", package = "rMVP"), "mvp.imp") #' -#' MVP.Data.Kin(TRUE, mvp_prefix=geno, out=tempfile("outfile"), cpus=1) +#' MVP.Data.Kin(TRUE, mvp_prefix=geno, out=tempfile("outfile"), cpu=1) #' } #' MVP.Data.Kin <- function( fileKin=TRUE, mvp_prefix='mvp', out=NULL, - priority='speed', + maxLine=10000, sep='\t', - cpus=1, + cpu=1, verbose=TRUE ) { if (is.null(out)) out <- mvp_prefix @@ -806,8 +807,7 @@ MVP.Data.Kin <- function( myKin <- read.big.matrix(fileKin, header = FALSE, type = 'double', sep = sep) } else if (fileKin == TRUE) { geno <- attach.big.matrix(paste0(mvp_prefix, ".geno.desc")) - logging.log("Calculate KINSHIP using Vanraden method...", "\n", verbose = verbose) - myKin <- MVP.K.VanRaden(geno, priority = priority, cpu = cpus) + myKin <- MVP.K.VanRaden(geno, step = maxLine, cpu = cpu, verbose = verbose) } else { stop("ERROR: The value of fileKin is invalid.") } diff --git a/R/MVP.FarmCPU.r b/R/MVP.FarmCPU.r index 020ed36..85109e6 100755 --- a/R/MVP.FarmCPU.r +++ b/R/MVP.FarmCPU.r @@ -23,6 +23,7 @@ #' @param geno genotype, m by n matrix, m is marker size, n is sample size. This is Pure Genotype Data Matrix(GD). THERE IS NO COLUMN FOR TAXA. #' @param map SNP map information, m by 3 matrix, m is marker size, the three columns are SNP_ID, Chr, and Pos #' @param CV covariates, n by c matrix, n is sample size, c is number of covariates +#' @param geno_ind_idx the index of effective genotyped individuals #' @param P start p values for all SNPs #' @param method.sub method used in substitution process, five options: 'penalty', 'reward', 'mean', 'median', or 'onsite' #' @param method.sub.final method used in substitution process, five options: 'penalty', 'reward', 'mean', 'median', or 'onsite' @@ -62,21 +63,24 @@ #' str(farmcpu) #' } #' -`MVP.FarmCPU` <- function(phe, geno, map, CV=NULL, P=NULL, method.sub="reward", method.sub.final="reward", +`MVP.FarmCPU` <- function(phe, geno, map, CV=NULL, geno_ind_idx=NULL, P=NULL, method.sub="reward", method.sub.final="reward", method.bin=c("EMMA", "static", "FaST-LMM"), bin.size=c(5e5,5e6,5e7), bin.selection=seq(10,100,10), memo="MVP.FarmCPU", Prior=NULL, ncpus=2, maxLoop=10, threshold.output=.01, converge=1, iteration.output=FALSE, p.threshold=NA, QTN.threshold=0.01, bound=NULL, verbose=TRUE){ #print("--------------------- Welcome to FarmCPU ----------------------------") + n <- ifelse(is.null(geno_ind_idx), ncol(geno), length(geno_ind_idx)) if(!is.big.matrix(geno)) stop("genotype should be in 'big.matrix' format.") if(sum(is.na(phe[, 2])) != 0) stop("NAs are not allowed in phenotype.") - if(nrow(phe) != ncol(geno)) stop("number of individuals not match in phenotype and genotype.") + if(nrow(phe) != n) stop("number of individuals does not match in phenotype and genotype.") echo=TRUE nm=nrow(map) if(!is.null(CV)){ CV=as.matrix(CV) + if(nrow(CV) != n) stop("number of individuals does not match in phenotype and fixed effects.") + if(sum(is.na(CV)) != 0) stop("NAs are not allowed in fixed effects.") CV.index <- apply(CV, 2, function(x) length(table(x)) > 1) CV <- CV[, CV.index, drop=FALSE] npc=ncol(CV) @@ -127,9 +131,9 @@ #Step 2b: Set bins if(theLoop<=2){ - myBin=FarmCPU.BIN(Y=phe[,c(1,2)],GDP=geno,GM=map,CV=CV,P=myPrior,method=method.bin,b=bin.size,s=bin.selection,theLoop=theLoop,bound=bound,ncpus=ncpus, verbose = verbose) + myBin=FarmCPU.BIN(Y=phe[,c(1,2)],GDP=geno,GDP_index=geno_ind_idx,GM=map,CV=CV,P=myPrior,method=method.bin,b=bin.size,s=bin.selection,theLoop=theLoop,bound=bound,ncpus=ncpus, verbose = verbose) }else{ - myBin=FarmCPU.BIN(Y=phe[,c(1,2)],GDP=geno,GM=map,CV=theCV,P=myPrior,method=method.bin,b=bin.size,s=bin.selection,theLoop=theLoop,ncpus=ncpus, verbose = verbose) + myBin=FarmCPU.BIN(Y=phe[,c(1,2)],GDP=geno,GDP_index=geno_ind_idx,GM=map,CV=theCV,P=myPrior,method=method.bin,b=bin.size,s=bin.selection,theLoop=theLoop,ncpus=ncpus, verbose = verbose) } #Step 2c: Remove bin dependency @@ -190,7 +194,7 @@ } } - myRemove=FarmCPU.Remove(GDP=geno,GM=map,seqQTN=seqQTN,seqQTN.p=seqQTN.p,threshold=.7) + myRemove=FarmCPU.Remove(GDP=geno,GDP_index=geno_ind_idx,GM=map,seqQTN=seqQTN,seqQTN.p=seqQTN.p,threshold=.7) #Recoding QTNs history seqQTN=myRemove$seqQTN @@ -234,7 +238,7 @@ theCV=cbind(CV,myRemove$bin) } - myGLM=FarmCPU.LM(y=phe[,2],GDP=geno,w=theCV,ncpus=ncpus,npc=npc, verbose=verbose) + myGLM=FarmCPU.LM(y=phe[,2],GDP=geno,GDP_index=geno_ind_idx,w=theCV,ncpus=ncpus,npc=npc, verbose=verbose) if(!is.null(seqQTN)){ if(ncol(myGLM$P) != (npc + length(seqQTN) + 1)) stop("wrong dimensions.") } @@ -463,6 +467,7 @@ #' #' @param Y a n by 2 matrix, the fist column is taxa id and the second is trait #' @param GDP genotype, m by n matrix, m is marker size, n is sample size. This is Pure Genotype Data Matrix(GD). THERE IS NO COLUMN FOR TAXA +#' @param GDP_index the index of effective genotyped individuals #' @param GM SNP map information, m by 3 matrix, m is marker size, the three columns are SNP_ID, Chr, and Pos #' @param CV covariates, n by c matrix, n is sample size, c is number of covariates #' @param P start p values for all SNPs @@ -479,11 +484,12 @@ #' #' @keywords internal FarmCPU.BIN <- - function(Y=NULL, GDP=NULL, GM=NULL, CV=NULL, P=NULL, method="EMMA", b=c(5e5,5e6,5e7), s=seq(10,100,10), theLoop=NULL, bound=NULL, ncpus=2, verbose=TRUE){ + function(Y=NULL, GDP=NULL, GDP_index=NULL, GM=NULL, CV=NULL, P=NULL, method="EMMA", b=c(5e5,5e6,5e7), s=seq(10,100,10), theLoop=NULL, bound=NULL, ncpus=2, verbose=TRUE){ #print("FarmCPU.BIN Started") if(is.null(P)) return(list(bin=NULL,binmap=NULL,seqQTN=NULL)) + if(is.null(GDP_index)) GDP_index=seq(1, ncol(GDP)) #Set upper bound for bin selection to squareroot of sample size n=nrow(Y) #bound=round(sqrt(n)/log10(n)) @@ -537,7 +543,7 @@ FarmCPU.BIN <- GP=cbind(GM,P,NA,NA,NA) mySpecify=FarmCPU.Specify(GI=GM,GP=GP,bin.size=bin,inclosure.size=inc) seqQTN=which(mySpecify$index==TRUE) - GK=t(GDP[seqQTN,]) + GK=t(GDP[seqQTN, GDP_index]) myBurger=FarmCPU.Burger(Y=Y[,1:2], CV=CV, GK=GK, ncpus=ncpus, method=method) myREML=myBurger$REMLs myVG=myBurger$vg #it is unused @@ -581,7 +587,7 @@ FarmCPU.BIN <- GP=cbind(GM,P,NA,NA,NA) mySpecify=FarmCPU.Specify(GI=GM,GP=GP,bin.size=bin,inclosure.size=inc) seqQTN=which(mySpecify$index==TRUE) - GK=t(GDP[seqQTN,]) + GK=t(GDP[seqQTN,GDP_index]) myBurger=FarmCPU.Burger(Y=Y[,1:2], CV=CV, GK=GK, ncpus=ncpus, method=method) myREML=myBurger$REMLs myVG=myBurger$vg #it is unused @@ -618,7 +624,7 @@ FarmCPU.BIN <- GP=cbind(GM,P,NA,NA,NA) mySpecify=FarmCPU.Specify(GI=GM,GP=GP,bin.size=bin[ii],inclosure.size=inc[ii]) seqQTN=which(mySpecify$index==TRUE) - GK=t(GDP[seqQTN,]) + GK=t(GDP[seqQTN,GDP_index]) myBurger=FarmCPU.Burger(Y=Y[,1:2], CV=CV, GK=GK, ncpus=ncpus, method=method) myREML=myBurger$REMLs myVG=myBurger$vg #it is unused @@ -741,6 +747,7 @@ FarmCPU.Specify <- #' @param y one column matrix, dependent variable #' @param w covariates, n by c matrix, n is sample size, c is number of covariates #' @param GDP genotype, m by n matrix, m is marker size, n is sample size. This is Pure Genotype Data Matrix(GD). THERE IS NO COLUMN FOR TAXA. +#' @param GDP_index index of effective genotyped individuals #' @param ncpus number of threads used for parallele computation #' @param npc number of covariates without pseudo QTNs #' @param verbose whether to print detail. @@ -752,7 +759,7 @@ FarmCPU.Specify <- #' #' @keywords internal FarmCPU.LM <- - function(y, w=NULL, GDP, ncpus=2, npc=0, verbose=TRUE){ + function(y, w=NULL, GDP, GDP_index=NULL, ncpus=2, npc=0, verbose=TRUE){ #print("FarmCPU.LM started") if(is.null(y)) return(NULL) if(is.null(GDP)) return(NULL) @@ -828,7 +835,7 @@ FarmCPU.LM <- # if(ncol(w) == 50) write.csv(P, "P.csv") mkl_env({ - results <- glm_c(y=y, X=w, iXX = wwi, GDP@address, verbose=verbose, threads=ncpus) + results <- glm_c(y=y, X=w, iXX = wwi, GDP@address, geno_ind=GDP_index, verbose=verbose, threads=ncpus) }) return(list(P=results[ ,-c(1:3), drop=FALSE], betapred=betapred, sepred=sepred, B=results[ , 1, drop=FALSE], S=results[ , 2, drop=FALSE])) # return(list(P=P, betapred=betapred, B=as.matrix(B), S=S)) @@ -889,7 +896,7 @@ FarmCPU.Burger <- } if(method=="EMMA"){ - K <- MVP.K.VanRaden(M=as.big.matrix(t(theGK)), priority="speed",verbose=FALSE) + K <- MVP.K.VanRaden(M=as.big.matrix(t(theGK)), verbose=FALSE) myEMMAREML <- MVP.EMMA.Vg.Ve(y=matrix(Y[,-1],nrow(Y),1), X=theCV, K=K) REMLs=-2*myEMMAREML$REML delta=myEMMAREML$delta @@ -968,6 +975,7 @@ FarmCPU.SUB <- #' @author Xiaolei Liu and Zhiwu Zhang #' #' @param GDP genotype, m by n matrix, m is marker size, n is sample size. This is Pure Genotype Data Matrix(GD). THERE IS NO COLUMN FOR TAXA +#' @param GDP_index the index of effective genotyped individuals #' @param GM SNP map information, m by 3 matrix, m is marker size, the three columns are SNP_ID, Chr, and Pos #' @param seqQTN s by 1 vecter for index of QTN on GM #' @param seqQTN.p p value of each @param seqQTN @@ -980,10 +988,13 @@ FarmCPU.SUB <- #' Relationship: bin=GDP[,c(seqQTN)], binmap=GM[seqQTN,], s0<=s #' @keywords internal FarmCPU.Remove <- - function(GDP=NULL, GM=NULL, seqQTN=NULL, seqQTN.p=NULL, threshold=.99){ + function(GDP=NULL, GDP_index=NULL, GM=NULL, seqQTN=NULL, seqQTN.p=NULL, threshold=.99){ + if(is.null(seqQTN))return(list(bin=NULL,binmap=NULL,seqQTN=NULL)) seqQTN=seqQTN[order(seqQTN.p)] + if(is.null(GDP_index)) GDP_index=seq(1, ncol(GDP)) + hugeNum=10e10 n=length(seqQTN) #fielter bins by physical location @@ -1006,20 +1017,20 @@ FarmCPU.Remove <- #Set sample ratio=.1 maxNum=100000 - m=nrow(GDP) #sample size - s=ncol(GDP) #number of markers + m=nrow(GDP) + s=length(GDP_index) sampled=s - if(sampled>maxNum)sampled=maxNum + if(sampled>maxNum) sampled=maxNum #index=sample(s,sampled) - index=1:sampled + index=GDP_index[1:sampled] #This section has problem of turning big.matrix to R matrix #It is OK as x is small if(is.big.matrix(GDP)){ - x=t(as.matrix(deepcopy(GDP,rows=seqQTN,cols=index) )) + x=t(as.matrix(deepcopy(GDP,rows=seqQTN,cols=index))) }else{ x=t(GDP[seqQTN,index] ) } @@ -1043,9 +1054,9 @@ FarmCPU.Remove <- #This section has problem of turning big.matrix to R matrix if(is.big.matrix(GDP)){ - bin=t(as.matrix(deepcopy(GDP,rows=seqQTN,) )) + bin=t(as.matrix(deepcopy(GDP,rows=seqQTN,cols=GDP_index) )) }else{ - bin=t(GDP[seqQTN, , drop=FALSE] ) + bin=t(GDP[seqQTN, GDP_index, drop=FALSE] ) } binmap=GM[seqQTN, , drop=FALSE] diff --git a/R/MVP.GLM.r b/R/MVP.GLM.r index 9daf605..49a05b0 100644 --- a/R/MVP.GLM.r +++ b/R/MVP.GLM.r @@ -21,6 +21,7 @@ #' @param phe phenotype, n * 2 matrix #' @param geno Genotype in numeric format, pure 0, 1, 2 matrix; m * n, m is marker size, n is population size #' @param CV Covariance, design matrix(n * x) for the fixed effects +#' @param geno_ind_idx the index of effective genotyped individuals #' @param cpu number of cpus used for parallel computation #' @param verbose whether to print detail. #' @@ -47,21 +48,24 @@ function( phe, geno, CV=NULL, - cpu=1, + geno_ind_idx = NULL, + cpu=1, verbose=TRUE ){ - n <- ncol(geno) + n <- ifelse(is.null(geno_ind_idx), ncol(geno), length(geno_ind_idx)) m <- nrow(geno) ys <- as.numeric(as.matrix(phe[,2])) if(!is.big.matrix(geno)) stop("genotype should be in 'big.matrix' format.") if(sum(is.na(ys)) != 0) stop("NAs are not allowed in phenotype.") - if(nrow(phe) != ncol(geno)) stop("number of individuals not match in phenotype and genotype.") + if(nrow(phe) != n) stop("number of individuals does not match in phenotype and genotype.") if(is.null(CV)){ X0 <- matrix(1, n) }else{ + if(nrow(CV) != n) stop("number of individuals does not match in phenotype and fixed effects.") + if(sum(is.na(CV)) != 0) stop("NAs are not allowed in fixed effects.") CV.index <- apply(CV, 2, function(x) length(table(x)) > 1) CV <- CV[, CV.index, drop=FALSE] X0 <- cbind(matrix(1, n), CV) @@ -72,7 +76,7 @@ function( logging.log("scanning...\n", verbose = verbose) mkl_env({ - results <- glm_c(y = ys, X = X0, iXX = iX0X0, geno@address, verbose = verbose, threads = cpu) + results <- glm_c(y = ys, X = X0, iXX = iX0X0, geno@address, geno_ind = geno_ind_idx, verbose = verbose, threads = cpu) }) return(results[, c(1, 2, ncol(results))]) diff --git a/R/MVP.K.VanRaden.r b/R/MVP.K.VanRaden.r index 95e0ead..eb3264f 100755 --- a/R/MVP.K.VanRaden.r +++ b/R/MVP.K.VanRaden.r @@ -17,9 +17,11 @@ #' Last update: Dec 12, 2019 #' #' @param M Genotype, m * n, m is marker size, n is population size -#' @param priority speed or memory +#' @param step the number of markers handled at a time, smaller value would reduce the memory cost +#' @param ind_idx the index of effective genotyped individuals #' @param cpu the number of cpu #' @param verbose whether to print detail. +#' @param checkNA whether to check NA in genotype. #' #' @return K, n * n matrix #' @export @@ -36,125 +38,138 @@ MVP.K.VanRaden <- function( M, - priority=c("speed", "memory"), + step=5000, + ind_idx=NULL, cpu=1, - verbose=TRUE + verbose=TRUE, + checkNA=TRUE ){ - # R.ver <- Sys.info()[['sysname']] - # wind <- R.ver == 'Windows' - # linux <- R.ver == 'Linux' - # mac <- (!linux) & (!wind) - r.open <- eval(parse(text = "!inherits(try(Revo.version,silent=TRUE),'try-error')")) - - # if(r.open && mac){ - # Sys.setenv("VECLIB_MAXIMUM_THREADS" = "1") - # } + + R.ver <- Sys.info()[['sysname']] + wind <- R.ver == 'Windows' + linux <- R.ver == 'Linux' + mac <- (!linux) & (!wind) + HPCMathLib <- mac | grepl("mkl", sessionInfo()$LAPACK) | grepl("openblas", sessionInfo()$LAPACK) | eval(parse(text = "!inherits(try(Revo.version,silent=TRUE),'try-error')")) if (!is.big.matrix(M)) stop("Format of Genotype Data must be big.matrix") - if(hasNA(M@address)) stop("NA is not allowed in genotype, use 'MVP.Data.impute' to impute.") + if(checkNA){ + if(hasNA(M@address)) stop("NA is not allowed in genotype, use 'MVP.Data.impute' to impute.") + } + n <- ifelse(is.null(ind_idx), ncol(M), length(ind_idx)) # logging.log("Relationship matrix mode in", priority[1], "\n", verbose = verbose) # if(is.null(dim(M))) M <- t(as.matrix(M)) - switch( - match.arg(priority), - "speed" = { - # if (!is.matrix(M)) M <- as.matrix(M) - # n <- ncol(M) - # m <- nrow(M) - # Pi <- 0.5 * rowMeans(M) - # logging.log("Scale the genotype matrix", "\n", verbose = verbose) - # M <- M - 2 * Pi - # SUM <- sum(Pi * (1 - Pi)) - # logging.log("Computing Z'Z", "\n", verbose = verbose) + + logging.log(paste0("Computing GRM for ", n, " individuals with a step of ", step), "\n", verbose = verbose) + + K <- try(kin_cal(M@address, geno_ind = ind_idx, threads = cpu, step = step, verbose = verbose, mkl = HPCMathLib), silent=TRUE) + # K <- try(kin_cal_s(M@address, threads = cpu, verbose = verbose, mkl = HPCMathLib), silent=TRUE) + + if(inherits(K,"try-error")){ + logging.log("Out of memory, please set a smaller value to parameter 'maxLine' and try again.", "\n", verbose = verbose) + stop(K[[1]]) + } + + # switch( + # match.arg(priority), + # "speed" = { + # # if (!is.matrix(M)) M <- as.matrix(M) + # # n <- ncol(M) + # # m <- nrow(M) + # # Pi <- 0.5 * rowMeans(M) + # # logging.log("Scale the genotype matrix", "\n", verbose = verbose) + # # M <- M - 2 * Pi + # # SUM <- sum(Pi * (1 - Pi)) + - K <- try(kin_cal_s(M@address, threads = cpu, verbose = verbose, mkl = r.open), silent=TRUE) + # K <- try(kin_cal_s(M@address, threads = cpu, verbose = verbose, mkl = HPCMathLib), silent=TRUE) - if(inherits(K,"try-error")){ - logging.log("Out of memory, please set parameter (..., priority='memory') and try again.", "\n", verbose = verbose) - stop(K[[1]]) - } - }, + # if(inherits(K,"try-error")){ + # logging.log("Out of memory, please set parameter (..., priority='memory') and try again.", "\n", verbose = verbose) + # stop(K[[1]]) + # } + # }, - "memory" = { - K <- kin_cal_m(M@address, threads=cpu, verbose = verbose) - # n <- ncol(M) - # m <- nrow(M) - # bac <- paste0("Z", memo, ".temp.bin") - # des <- paste0("Z", memo, ".temp.desc") - # if (file.exists(bac)) file.remove(bac) - # if (file.exists(des)) file.remove(des) - # #options(bigmemory.typecast.warning=FALSE) - # Z <- big.matrix( - # nrow = m, - # ncol = n, - # type = "double", - # backingfile = bac, - # descriptorfile = des, - # init = 0.1 - # ) - # Pi <- NULL - # estimate.memory <- function(dat, integer=FALSE, raw=FALSE){ - # cells.per.gb <- 2^27 # size of double() resulting in ~1GB of memory use by R 2.15 - # dimz <- dat - # if(length(dimz) == 1) { dimz[2] <- 1 } - # if(length(dimz)>1 & length(dimz)<11 & is.numeric(dimz)) { - # total.size <- as.double(1) - # for(cc in 1:length(dimz)) { total.size <- as.double(total.size * as.double(dimz[cc])) } - # memory.estimate <- as.double(as.double(total.size)/cells.per.gb) - # memory.estimate <- memory.estimate - # if(integer) { memory.estimate <- memory.estimate/2 } else { if(raw) { memory.estimate <- memory.estimate/8 } } - # return(memory.estimate) - # } else { - # # guessing this is a vector - # if(!is.list(dimz) & is.vector(dimz)) { - # LL <- length(dimz) - # return(estimate.memory(LL, integer=integer, raw=raw)) - # } else { - # warning("tried to estimate memory for object which is neither a vector, pair of dimension sizes or a dataframe/matrix") - # } - # } - # } - # if((Sys.info()[['sysname']]) == 'Windows'){ - # max.gb <- memory.limit()/1000 - # }else{ - # max.gb <- Inf - # } - # maxLines.gb <- estimate.memory(c(maxLine, n)) - # if(maxLines.gb > max.gb) stop("Memory limited! Please reset the 'maxLine'") - # loop.index <- seq(0, m, maxLine)[-1] - # if(max(loop.index) < m) loop.index <- c(loop.index, m) - # loop.len <- length(loop.index) - # print("Z assignment...") - # for(cc in 1:loop.len){ - # if(loop.len == 1){ - # c1 <- 1 - # }else{ - # c1 <- ifelse(cc == loop.len, (loop.index[cc-1]) + 1, loop.index[cc]-maxLine + 1) - # } - # c2 <- loop.index[cc] - # means <-rowMeans(M[c1:c2, 1:n]) - # if(!is.null(weight)){ - # Z[c1:c2, 1:n] <- (M[c1:c2, 1:n]-means) * sqrt(weight[c1:c2]) - # }else{ - # Z[c1:c2, 1:n] <- M[c1:c2, 1:n]-means - # } - # Pi <- c(Pi, 0.5 * means);gc() - # } - # print("Assignment DONE!") - # if(is.null(SUM)){ - # SUM <- sum(Pi * (1-Pi)) - # } - # fl.suc <- flush(Z) - # if(!fl.suc){ stop("flush failed\n") } - # RR <- describe(Z); rm(list=c("Z", "Pi", "means")); gc() - # Z <- attach.big.matrix(RR) - # print("Computing Z'Z in big.matrix...") - # K <- 0.5 * big.crossprod(Z)/SUM - # rm(Z) - # gc() - # unlink(c(paste0("Z", memo, ".temp.bin"), paste0("Z", memo, ".temp.desc")), recursive = TRUE) - } - ) + # "memory" = { + # K <- kin_cal_m(M@address, threads=cpu, verbose = verbose) + # # n <- ncol(M) + # # m <- nrow(M) + # # bac <- paste0("Z", memo, ".temp.bin") + # # des <- paste0("Z", memo, ".temp.desc") + # # if (file.exists(bac)) file.remove(bac) + # # if (file.exists(des)) file.remove(des) + # # #options(bigmemory.typecast.warning=FALSE) + # # Z <- big.matrix( + # # nrow = m, + # # ncol = n, + # # type = "double", + # # backingfile = bac, + # # descriptorfile = des, + # # init = 0.1 + # # ) + # # Pi <- NULL + # # estimate.memory <- function(dat, integer=FALSE, raw=FALSE){ + # # cells.per.gb <- 2^27 # size of double() resulting in ~1GB of memory use by R 2.15 + # # dimz <- dat + # # if(length(dimz) == 1) { dimz[2] <- 1 } + # # if(length(dimz)>1 & length(dimz)<11 & is.numeric(dimz)) { + # # total.size <- as.double(1) + # # for(cc in 1:length(dimz)) { total.size <- as.double(total.size * as.double(dimz[cc])) } + # # memory.estimate <- as.double(as.double(total.size)/cells.per.gb) + # # memory.estimate <- memory.estimate + # # if(integer) { memory.estimate <- memory.estimate/2 } else { if(raw) { memory.estimate <- memory.estimate/8 } } + # # return(memory.estimate) + # # } else { + # # # guessing this is a vector + # # if(!is.list(dimz) & is.vector(dimz)) { + # # LL <- length(dimz) + # # return(estimate.memory(LL, integer=integer, raw=raw)) + # # } else { + # # warning("tried to estimate memory for object which is neither a vector, pair of dimension sizes or a dataframe/matrix") + # # } + # # } + # # } + # # if((Sys.info()[['sysname']]) == 'Windows'){ + # # max.gb <- memory.limit()/1000 + # # }else{ + # # max.gb <- Inf + # # } + # # maxLines.gb <- estimate.memory(c(maxLine, n)) + # # if(maxLines.gb > max.gb) stop("Memory limited! Please reset the 'maxLine'") + # # loop.index <- seq(0, m, maxLine)[-1] + # # if(max(loop.index) < m) loop.index <- c(loop.index, m) + # # loop.len <- length(loop.index) + # # print("Z assignment...") + # # for(cc in 1:loop.len){ + # # if(loop.len == 1){ + # # c1 <- 1 + # # }else{ + # # c1 <- ifelse(cc == loop.len, (loop.index[cc-1]) + 1, loop.index[cc]-maxLine + 1) + # # } + # # c2 <- loop.index[cc] + # # means <-rowMeans(M[c1:c2, 1:n]) + # # if(!is.null(weight)){ + # # Z[c1:c2, 1:n] <- (M[c1:c2, 1:n]-means) * sqrt(weight[c1:c2]) + # # }else{ + # # Z[c1:c2, 1:n] <- M[c1:c2, 1:n]-means + # # } + # # Pi <- c(Pi, 0.5 * means);gc() + # # } + # # print("Assignment DONE!") + # # if(is.null(SUM)){ + # # SUM <- sum(Pi * (1-Pi)) + # # } + # # fl.suc <- flush(Z) + # # if(!fl.suc){ stop("flush failed\n") } + # # RR <- describe(Z); rm(list=c("Z", "Pi", "means")); gc() + # # Z <- attach.big.matrix(RR) + # # print("Computing Z'Z in big.matrix...") + # # K <- 0.5 * big.crossprod(Z)/SUM + # # rm(Z) + # # gc() + # # unlink(c(paste0("Z", memo, ".temp.bin"), paste0("Z", memo, ".temp.desc")), recursive = TRUE) + # } + # ) #print("K Preparation is Done!") logging.log("Deriving relationship matrix successfully", "\n", verbose = verbose); gc() return(K) diff --git a/R/MVP.MLM.r b/R/MVP.MLM.r index 6a72557..05a425e 100644 --- a/R/MVP.MLM.r +++ b/R/MVP.MLM.r @@ -23,6 +23,7 @@ #' @param K Kinship, Covariance matrix(n * n) for random effects; must be positive semi-definite #' @param eigenK list of eigen Kinship #' @param CV covariates +#' @param geno_ind_idx the index of effective genotyped individuals #' @param REML a list that contains ve and vg #' @param cpu number of cpus used for parallel computation #' @param vc.method the methods for estimating variance component("emma" or "he" or "brent") @@ -57,36 +58,32 @@ function( K=NULL, eigenK=NULL, CV=NULL, + geno_ind_idx=NULL, REML=NULL, cpu=1, vc.method=c("BRENT", "EMMA", "HE"), verbose=TRUE ){ - # R.ver <- Sys.info()[['sysname']] - # r.open <- eval(parse(text = "!inherits(try(Revo.version,silent=TRUE),'try-error')")) - - # if (R.ver == 'Windows') cpu <- 1 - # if (r.open && cpu > 1 && R.ver == 'Darwin') { - # Sys.setenv("VECLIB_MAXIMUM_THREADS" = "1") - # } vc.method <- match.arg(vc.method) - n <- ncol(geno) + n <- ifelse(is.null(geno_ind_idx), ncol(geno), length(geno_ind_idx)) m <- nrow(geno) ys <- as.numeric(as.matrix(phe[,2])) if(!is.big.matrix(geno)) stop("genotype should be in 'big.matrix' format.") if(sum(is.na(ys)) != 0) stop("NAs are not allowed in phenotype.") - if(nrow(phe) != ncol(geno)) stop("number of individuals not match in phenotype and genotype.") + if(nrow(phe) != n) stop("number of individuals does not match in phenotype and genotype.") if(is.null(K)){ if(vc.method == "EMMA" | vc.method == "he") stop("Kinship must be provided!") if(vc.method == "BRENT"){ if(is.null(eigenK)) stop("eigenK must be provided!") + if(length(eigenK$values) != n) stop("number of individuals does not match in phenotype and eigen values.") } }else{ # convert K to base:matrix K <- K[, ] + if(nrow(K) != n) stop("number of individuals does not match in phenotype and relationship matrix.") if(is.null(eigenK)){ logging.log("Eigen Decomposition on GRM", "\n", verbose = verbose) eigenK <- eigen(K, symmetric=TRUE) @@ -95,7 +92,9 @@ function( if (is.null(CV)) { X0 <- matrix(1, n) - } else { + }else{ + if(nrow(CV) != n) stop("number of individuals does not match in phenotype and fixed effects.") + if(sum(is.na(CV)) != 0) stop("NAs are not allowed in fixed effects.") CV.index <- apply(CV, 2, function(x) length(table(x)) > 1) CV <- CV[, CV.index, drop=FALSE] X0 <- cbind(matrix(1, n), CV) @@ -121,7 +120,7 @@ function( logging.log("scanning...\n", verbose = verbose) mkl_env({ - results <- mlm_c(y = ys, X = X0, U = U, vgs = vgs, geno@address, verbose = verbose, threads = cpu) + results <- mlm_c(y = ys, X = X0, U = U, vgs = vgs, geno@address, geno_ind_idx, verbose = verbose, threads = cpu) }) return(results) diff --git a/R/MVP.PCA.r b/R/MVP.PCA.r index cccdf51..90602a4 100755 --- a/R/MVP.PCA.r +++ b/R/MVP.PCA.r @@ -19,7 +19,7 @@ #' #' @param M Genotype in numeric format, pure 0, 1, 2 matrix; m * n, m is marker size, n is population size #' @param K kinship matrix -#' @param priority speed or memory +#' @param maxLine the number of markers handled at a time, smaller value would reduce the memory cost #' @param pcs.keep maximum number of PCs for output #' @param cpu the number of cpu #' @param verbose whether to print detail. @@ -40,23 +40,13 @@ #' } #' MVP.PCA <- -function(M=NULL, K=NULL, priority=c("speed", "memory"), pcs.keep=5, cpu=1, verbose=TRUE){ - # R.ver <- Sys.info()[['sysname']] - # wind <- R.ver == 'Windows' - # linux <- R.ver == 'Linux' - # mac <- (!linux) & (!wind) - # r.open <- eval(parse(text = "!inherits(try(Revo.version,silent=TRUE),'try-error')")) +function(M=NULL, K=NULL, maxLine, ind_idx=NULL, pcs.keep=5, cpu=1, verbose=TRUE){ - # if (r.open && mac) { - # Sys.setenv("VECLIB_MAXIMUM_THREADS" = "1") - # } #Data Check if (is.null(M) & is.null(K)) { stop("There is no genotype data or relationship matrix!") } - priority <- match.arg(priority) - if (pcs.keep < 0) { pcs.keep = 0 } @@ -66,11 +56,10 @@ function(M=NULL, K=NULL, priority=c("speed", "memory"), pcs.keep=5, cpu=1, verbo } if(is.null(K)){ - K <- MVP.K.VanRaden(M=M, priority=priority, cpu=cpu) + K <- MVP.K.VanRaden(M=M, ind_idx = ind_idx, step = maxLine, cpu = cpu, verbose = verbose) } logging.log("Eigen Decomposition on GRM", "\n", verbose = verbose) - # if(r.open) eval(parse(text = "try(setMKLthreads(cpu), silent=TRUE)")) PCs <- eigen(K, symmetric=TRUE)$vectors[, 1:pcs.keep] logging.log("Deriving PCs successfully", "\n", verbose = verbose) diff --git a/R/MVP.r b/R/MVP.r index 584ce62..dbe298d 100755 --- a/R/MVP.r +++ b/R/MVP.r @@ -34,7 +34,7 @@ #' @param CV.MLM covariates added in MLM #' @param CV.FarmCPU covariates added in FarmCPU #' @param REML a list contains ve and vg -#' @param priority speed or memory +#' @param maxLine the number of markers handled at a time, smaller value would reduce the memory cost #' @param ncpus number of cpus used for parallel #' @param vc.method methods for estimating variance component("EMMA" or "HE" or "BRENT") #' @param method the GWAS model, "GLM", "MLM", and "FarmCPU", models can be selected simutaneously, i.e. c("GLM", "MLM", "FarmCPU") @@ -83,7 +83,7 @@ #' } MVP <- function(phe, geno, map, K=NULL, nPC.GLM=NULL, nPC.MLM=NULL, nPC.FarmCPU=NULL, - CV.GLM=NULL, CV.MLM=NULL, CV.FarmCPU=NULL, REML=NULL, priority="speed", + CV.GLM=NULL, CV.MLM=NULL, CV.FarmCPU=NULL, REML=NULL, maxLine=10000, ncpus=detectCores(logical = FALSE), vc.method=c("BRENT", "EMMA", "HE"), method=c("GLM", "MLM", "FarmCPU"), p.threshold=NA, QTN.threshold=0.01, method.bin="static", bin.size=c(5e5,5e6,5e7), @@ -162,10 +162,6 @@ function(phe, geno, map, K=NULL, nPC.GLM=NULL, nPC.MLM=NULL, nPC.FarmCPU=NULL, if (length(na.index) != 0) seqTaxa <- intersect(seqTaxa, c(1:n)[-na.index]) if (length(seqTaxa) != n) { logging.log("Total", n - length(seqTaxa), "individuals removed due to missings", "\n", verbose = verbose) - geno = deepcopy( - x = geno, - cols = seqTaxa - ) phe = phe[seqTaxa,] if (!is.null(K)) { K = K[seqTaxa, seqTaxa] } if (!is.null(CV.GLM)) { CV.GLM = CV.GLM[seqTaxa,] } @@ -173,9 +169,6 @@ function(phe, geno, map, K=NULL, nPC.GLM=NULL, nPC.MLM=NULL, nPC.FarmCPU=NULL, if (!is.null(CV.FarmCPU)) { CV.FarmCPU = CV.FarmCPU[seqTaxa,] } } - m <- nrow(geno) - n <- ncol(geno) - #initial results glm.results <- NULL mlm.results <- NULL @@ -199,9 +192,11 @@ function(phe, geno, map, K=NULL, nPC.GLM=NULL, nPC.MLM=NULL, nPC.FarmCPU=NULL, if (is.null(K)) { K <- MVP.K.VanRaden( M = geno, - priority = priority, + ind_idx = seqTaxa, + step = maxLine, cpu = ncpus, - verbose = verbose + verbose = verbose, + checkNA = FALSE ) } logging.log("Eigen Decomposition on GRM", "\n", verbose = verbose) @@ -278,7 +273,7 @@ function(phe, geno, map, K=NULL, nPC.GLM=NULL, nPC.MLM=NULL, nPC.FarmCPU=NULL, logging.log("-------------------------GWAS Start-------------------------", "\n", verbose = verbose) if (glm.run) { logging.log("General Linear Model (GLM) Start...", "\n", verbose = verbose) - glm.results <- MVP.GLM(phe=phe, geno=geno, CV=CV.GLM, cpu=ncpus, verbose = verbose);gc() + glm.results <- MVP.GLM(phe=phe, geno=geno, CV=CV.GLM, geno_ind_idx=seqTaxa, cpu=ncpus, verbose = verbose);gc() colnames(glm.results) <- c("Effect", "SE", paste(colnames(phe)[2],"GLM",sep=".")) z = glm.results[, 1]/glm.results[, 2] lambda = median(z^2, na.rm=TRUE)/qchisq(1/2, df = 1,lower.tail=FALSE) @@ -293,7 +288,7 @@ function(phe, geno, map, K=NULL, nPC.GLM=NULL, nPC.MLM=NULL, nPC.FarmCPU=NULL, if (mlm.run) { logging.log("Mixed Linear Model (MLM) Start...", "\n", verbose = verbose) - mlm.results <- MVP.MLM(phe=phe, geno=geno, K=K, eigenK=eigenK, CV=CV.MLM, cpu=ncpus, vc.method=vc.method, verbose = verbose);gc() + mlm.results <- MVP.MLM(phe=phe, geno=geno, K=K, eigenK=eigenK, CV=CV.MLM, geno_ind_idx=seqTaxa, cpu=ncpus, vc.method=vc.method, verbose = verbose);gc() colnames(mlm.results) <- c("Effect", "SE", paste(colnames(phe)[2],"MLM",sep=".")) z = mlm.results[, 1]/mlm.results[, 2] lambda = median(z^2, na.rm=TRUE)/qchisq(1/2, df = 1,lower.tail=FALSE) @@ -308,7 +303,7 @@ function(phe, geno, map, K=NULL, nPC.GLM=NULL, nPC.MLM=NULL, nPC.FarmCPU=NULL, if (farmcpu.run) { logging.log("FarmCPU Start...", "\n", verbose = verbose) - farmcpu.results <- MVP.FarmCPU(phe=phe, geno=geno, map=map[,1:3], CV=CV.FarmCPU, ncpus=ncpus, memo="MVP.FarmCPU", p.threshold=p.threshold, QTN.threshold=QTN.threshold, method.bin=method.bin, bin.size=bin.size, bin.selection=bin.selection, maxLoop=maxLoop, verbose = verbose) + farmcpu.results <- MVP.FarmCPU(phe=phe, geno=geno, map=map[,1:3], CV=CV.FarmCPU, geno_ind_idx=seqTaxa, ncpus=ncpus, memo="MVP.FarmCPU", p.threshold=p.threshold, QTN.threshold=QTN.threshold, method.bin=method.bin, bin.size=bin.size, bin.selection=bin.selection, maxLoop=maxLoop, verbose = verbose) colnames(farmcpu.results) <- c("Effect", "SE", paste(colnames(phe)[2],"FarmCPU",sep=".")) z = farmcpu.results[, 1]/farmcpu.results[, 2] lambda = median(z^2, na.rm=TRUE)/qchisq(1/2, df = 1,lower.tail=FALSE) diff --git a/R/RcppExports.R b/R/RcppExports.R index 1790a81..30a366c 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -5,12 +5,12 @@ getRow <- function(pBigMat, row) { .Call(`_rMVP_getRow`, pBigMat, row) } -glm_c <- function(y, X, iXX, pBigMat, verbose = TRUE, threads = 0L) { - .Call(`_rMVP_glm_c`, y, X, iXX, pBigMat, verbose, threads) +glm_c <- function(y, X, iXX, pBigMat, geno_ind = NULL, verbose = TRUE, threads = 0L) { + .Call(`_rMVP_glm_c`, y, X, iXX, pBigMat, geno_ind, verbose, threads) } -mlm_c <- function(y, X, U, vgs, pBigMat, verbose = TRUE, threads = 0L) { - .Call(`_rMVP_mlm_c`, y, X, U, vgs, pBigMat, verbose, threads) +mlm_c <- function(y, X, U, vgs, pBigMat, geno_ind = NULL, verbose = TRUE, threads = 0L) { + .Call(`_rMVP_mlm_c`, y, X, U, vgs, pBigMat, geno_ind, verbose, threads) } vcf_parser_map <- function(vcf_file, out) { @@ -69,3 +69,7 @@ kin_cal_s <- function(pBigMat, threads = 0L, mkl = FALSE, verbose = TRUE) { .Call(`_rMVP_kin_cal_s`, pBigMat, threads, mkl, verbose) } +kin_cal <- function(pBigMat, geno_ind = NULL, threads = 0L, step = 10000L, mkl = FALSE, verbose = TRUE) { + .Call(`_rMVP_kin_cal`, pBigMat, geno_ind, threads, step, mkl, verbose) +} + diff --git a/README.md b/README.md index c671a10..e7494ab 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# MVP [![](https://img.shields.io/badge/Issues-%2B-brightgreen.svg)](https://github.com/XiaoleiLiuBio/rMVP/issues/new) [![](http://www.r-pkg.org/badges/version/rMVP?color=red)](https://cran.r-project.org/package=rMVP) [![](https://img.shields.io/badge/GitHub-1.0.6-blueviolet.svg)]() ![](http://cranlogs.r-pkg.org/badges/grand-total/rMVP?color=green) [![](https://cranlogs.r-pkg.org/badges/rMVP)](https://cran.r-project.org/package=rMVP) +# rMVP [![](https://img.shields.io/badge/Issues-%2B-brightgreen.svg)](https://github.com/XiaoleiLiuBio/rMVP/issues/new) [![](http://www.r-pkg.org/badges/version/rMVP?color=red)](https://cran.r-project.org/package=rMVP) [![CRAN Version](https://www.r-pkg.org/badges/version/rMVP?color=yellow)](https://CRAN.R-project.org/package=rMVP) [![](https://img.shields.io/badge/GitHub-1.1.0-blueviolet.svg)]() ![](http://cranlogs.r-pkg.org/badges/grand-total/rMVP?color=green) [![](https://cranlogs.r-pkg.org/badges/rMVP)](https://cran.r-project.org/package=rMVP) -## A [M](https://)emory-efficient, [V](https://)isualization-enhanced, and [P](https://)arallel-accelerated Tool for Genome-Wide Association Study +## An [r](https://) package for [M](https://)emory-efficient, [V](https://)isualization-enhanced, and [P](https://)arallel-accelerated Genome-Wide Association Study

@@ -9,7 +9,7 @@

-### #-----------------------***rMVP [v1.0.0]() is coming, and better again!***------------------------# +### #-----------------------***rMVP [v1.1.0]() is coming, and stronger again!***------------------------# ### Repos: **Github:** https://github.com/xiaolei-lab/rMVP @@ -37,39 +37,39 @@ Questions, suggestions, and bug reports are welcome and appreciated: [xiaoleiliu 📊 CMplot: A drawing tool for genetic analyses. - + ### Contents - - - [Installation](#installation) - [Data Preparation](#data-preparation) - - [Phenotype](#phenotype) - - [PLINK binary](#plink-binary)/[VCF](#vcf)/[Hapmap](#hapmap)/[Numeric](#numeric) - - [Kinship](#kinship)/[Principal Components](#principal-components) + - [Phenotype](#phenotype) + - [Genotype](#genotype) + - [PLINK binary](#plink-binary) + - [VCF](#vcf) + - [Hapmap](#hapmap) + - [Numeric](#numeric) + - [Genomic relationship matrix (Optional)](#genomic-relationship-matrix-optional) + - [Principal Components (Optional)](#principal-components-optional) - [Data Input](#data-input) - - [Basic](#basic) - - [Advanced](#advanced) + - [Basic](#basic) + - [Advanced](#advanced) - [Start GWAS](#start-gwas) - [Output](#output) - - [Phenotype distribution](#phenotype-distribution) - - [SNP-density plot](#snp-density-plot) - - [PCA plot](#pca-plot) - - [Manhattan plot in Circular fashion](#manhattan-plot-in-circular-fashion) - - [Manhattan plot in Rectangular fashion for single trait or method](#manhattan-plot-in-rectangular-fashion-for-single-trait-or-method) + - [Phenotype distribution](#phenotype-distribution) + - [SNP-density plot](#snp-density-plot) + - [PCA plot](#pca-plot) + - [Manhattan plot in Circular fashion](#manhattan-plot-in-circular-fashion) + - [Manhattan plot in Rectangular fashion for single trait or method](#manhattan-plot-in-rectangular-fashion-for-single-trait-or-method) - [Manhattan plot in Rectangular fashion for multiple traits or methods](#manhattan-plot-in-rectangular-fashion-for-multiple-traits-or-methods) - [Q-Q plot for single trait or method](#q-q-plot-for-single-trait-or-method) - [Q-Q plot for multiple traits or methods](#q-q-plot-for-multiple-traits-or-methods) - [Citation](#citation) - [FAQ and Hints](#faq-and-hints) - - - --- # Installation **[back to top](#contents)** -~~**WE STRONGLY RECOMMEND TO INSTALL MVP ON Microsoft R Open(https://mran.microsoft.com/download/)**~~ (deprecated) +WE STRONGLY RECOMMEND TO link MKL or OpenBLAS with R to accelerate parallel computing. To install **rMVP** in R: * The stable version: ```R @@ -81,11 +81,11 @@ install.packages("rMVP") devtools::install_github("xiaolei-lab/rMVP") ``` -After installed successfully, **MVP** can be loaded by typing +After installed successfully, **rMVP** can be loaded by typing ```r library(rMVP) ``` -Typing ```?MVP``` could get the details of all parameters. +Typing ```?rMVP``` could get the details of all parameters. **For more help on Windows installation, see the [wiki page](https://github.com/XiaoleiLiuBio/rMVP/wiki/R-%E8%AF%AD%E8%A8%80%E7%8E%AF%E5%A2%83%E9%85%8D%E7%BD%AE%E6%8C%87%E5%8D%97%EF%BC%88Windows%EF%BC%89) (Chinese)** @@ -94,7 +94,7 @@ Typing ```?MVP``` could get the details of all parameters. # Data Preparation ## Phenotype **[back to top](#contents)** -We suggest to provide the phenotype file, users needn't to manually pre-treat the order of phenotype and genotype individuals, rMVP could automatically adjust the order of genotype file to be consistent with phenotype file. **Note that if the phenotype is provided in data conversion, rMVP will generate a new phenotype file, please remember to load it for analysis rather than original one**. +We suggest to provide the phenotype file, because users needn't to manually pre-treat the order of phenotype and genotype individuals, rMVP could automatically adjust the order of phenotype file to be consistent with genotype file. **Note that if the phenotype is provided in data conversion, rMVP will generate a new phenotype file, please remember to load it for analysis rather than original one**. | Taxa | trait1 | trait2 | trait3 | | :---: | :---: |:---: |:---: | @@ -106,7 +106,8 @@ We suggest to provide the phenotype file, users needn't to manually pre-treat th |A214N| 95.13|0.87|0| |A239 |100.2|-0.16|1| -## PLINK binary +## Genotype +### PLINK binary **[back to top](#contents)** If you have genotype data in **PLINK Binary** format (details see http://zzz.bwh.harvard.edu/plink/data.shtml#bed):   @@ -114,24 +115,22 @@ If you have genotype data in **PLINK Binary** format (details see http://zzz.bwh **fileKin**, TRUE or FALSE, if TRUE, kinship matrix represents relationship among individuals will be calculated **filePC**, TRUE or FALSE, if TRUE, principal component analysis will be performed **out**, prefix of output file -**priority**, "speed" or "memory", the "speed" mode is faster but uses more memory while "memory" is slower but uses less memory -**maxLine**, number, if **priority = "memory"**, it is the number of markers read into memory +**maxLine**, number, the number of markers read into memory ```r # Full-featured function (Recommended) MVP.Data(fileBed="plink", filePhe=NULL, fileKin=FALSE, filePC=FALSE, - #priority="speed", #maxLine=10000, out="mvp.plink" ) # Only convert genotypes -MVP.Data.Bfile2MVP(bfile="plink", out='mvp', maxLine=1e4, priority='speed') # the genotype data should be fully imputed before using this function +MVP.Data.Bfile2MVP(bfile="plink", out='mvp', maxLine=1e4) # the genotype data should be fully imputed before using this function ``` -## VCF +### VCF **[back to top](#contents)** If you have genotype data in **VCF** format: **fileVCF**, name of genotype data in VCF format @@ -169,7 +168,7 @@ MVP.Data(fileVCF="myVCF.vcf", MVP.Data.VCF2MVP("myVCF.vcf", out='mvp') # the genotype data should be fully imputed before using this function ``` -## Hapmap +### Hapmap **[back to top](#contents)** If you have genotype data in **Hapmap** format: @@ -178,9 +177,8 @@ If you have genotype data in **Hapmap** format: **sep.phe**, separator of phenotype file **fileKin**, TRUE or FALSE, if TRUE, kinship matrix represents relationship among individuals will be calculated **filePC**, TRUE or FALSE, if TRUE, principal component analysis will be performed -**out**, the prefix of output file -**priority**, "speed" or "memory", the 'speed' mode is faster but uses more memory while 'memory' is slower but uses less memory -**maxLine**, number, if **priority = "memory"**, it is the number of markers read into memory +**out**, the prefix of output file +**maxLine**, number, the number of markers read into memory > `hapmap.txt` @@ -202,7 +200,6 @@ MVP.Data(fileHMP="hapmap.txt", SNP.effect="Add", fileKin=FALSE, filePC=FALSE, - #priority="memory", #maxLine=10000, out="mvp.hmp" ) @@ -222,14 +219,13 @@ MVP.Data(fileHMP=c("hmp.chr1.txt", "hmp.chr2.txt", "hmp.chr3.txt", "hmp.chr4.txt SNP.effect="Add", fileKin=FALSE, filePC=FALSE, - #priority="memory", #maxLine=10000, out="mvp.hmp" ) ``` --> -## Numeric +### Numeric **[back to top](#contents)** If you have genotype data in **Numeric** (m * n, m rows and n columns, m is the number of SNPs, n is the number of individuals) format:   @@ -241,9 +237,8 @@ If you have genotype data in **Numeric** (m * n, m rows and n columns, m is the **type.geno**, the type of data in Numeric file, "char", "integer", or "double" **fileKin**, TRUE or FALSE, if TRUE, kinship matrix represents relationship among individuals will be calculated **filePC**, TRUE or FALSE, if TRUE, principal component analysis will be performed -**out**, the prefix of output file -**priority**, "speed" or "memory", the "speed" mode is faster but uses more memory while "memory" is slower but uses less memory -**maxLine**, number, if **priority = "memory"**, it is the number of markers read into memory +**out**, the prefix of output file +**maxLine**, number, the number of markers read into memory **auto_transpose**, bool, if **auto_transpose = TRUE**, it is automatically transposed to ensure that the number of rows (markers) is greater than the number of columns (individuals). @@ -352,16 +347,15 @@ MVP.Data(fileNum="Numeric.txt", sep.phe="\t", fileKin=FALSE, filePC=FALSE, - #priority="memory", #maxLine=10000, out="mvp.num" ) # Only convert genotypes -MVP.Data.Numeric2MVP("Numeric.txt", out='mvp', maxLine=1e4, priority='speed', auto_transpose=T) # the genotype data should be fully imputed before using this function +MVP.Data.Numeric2MVP("Numeric.txt", out='mvp', maxLine=1e4, auto_transpose=T) # the genotype data should be fully imputed before using this function ``` -## Kinship +## Genomic relationship matrix (Optional) **[back to top](#contents)** If you have Kinship matrix data that represents the relationship among individuals @@ -438,13 +432,13 @@ If you have Kinship matrix data that represents the relationship among individua ```r # read from file -MVP.Data.Kin("mvp.kin.txt", out="mvp", maxLine=1e4, priority='memory', sep='\t') +MVP.Data.Kin("mvp.kin.txt", out="mvp", maxLine=1e4, sep='\t') # calculate from mvp_geno_file MVP.Data.Kin(TRUE, mvp_prefix='mvp.vcf', out='mvp') ``` -## Principal Components +## Principal Components (Optional) **[back to top](#contents)** If you have Principal Components data @@ -513,7 +507,7 @@ map <- read.table("mvp.geno.map" , head = TRUE) ## Advanced **[back to top](#contents)** -You can give MVP the prepared Kinship matrix and Covariates data generated by **'MVP.Data'** function +You can load the prepared Kinship matrix and Covariates data generated by **'MVP.Data'** function **Kinship**, Kinship matrix, the dimension of Kinship matrix is n * n (n is sample size), no taxa names included **Covariates**, Covariates matrix, the dimension of Covariates matrix is n * nCV (n is sample size, nCV is number of covariates, no taxa names and header row included ***NOTE***: **If pcs have been added in covariate files, PLEASE DO NOT assign value to nPC.GLM, nPC.MLM, nPC.FarmCPU.** @@ -522,15 +516,16 @@ You can give MVP the prepared Kinship matrix and Covariates data generated by ** Kinship <- attach.big.matrix("mvp.kin.desc") Covariates_PC <- bigmemory::as.matrix(attach.big.matrix("mvp.pc.desc")) ``` -If you have additional fixed effects (breed, sex) or covariates (weight), please use it as following: +If you have additional environmental fixed effects (e.g., breed, sex) or covariates (weight), please use it as following: ``` -Covariates <- model.matrix(~as.factor(breed)+as.factor(sex)+as.numeric(weight), data=yourdata) +Covariates <- model.matrix.lm(~as.factor(breed)+as.factor(sex)+as.numeric(weight), data=yourdata, na.action = "na.pass") +# NA is acceptable in 'Covariates' # if you are supposed to take PC to covariate Covariates <- cbind(Covariates, Covariates_PC) ``` -***NOTE***: **rMVP has no function of adjusting the order of individuals in covariates. PLEASE make sure the order of individuals in covariates file must be consistent with the output phenotype file of 'MVP.Data'.** +***NOTE***: **rMVP has no function of adjusting the order of individuals in covariates. PLEASE make sure the order of individuals in covariates file must be consistent with that in genotype file.** If you have prepared Kinship matrix and Covariates data generated by other software packages, see **[Kinship](#kinship)** and **[Principal Components](#principal-components)** @@ -555,7 +550,7 @@ Three models are included in MVP package: General Linear Model (GLM), Mixed Line **nPC.MLM**, number of first columns of Principal Components added in MLM **please attention that if nPC.FarmCPU > 0, no PCs should be added in CV.FarmCPU** **nPC.FarmCPU**, number of first columns of Principal Components added in FarmCPU -**priority**, **"speed"** or **"memory" when calculating the genomic relationship matrix** +**maxLine**, the number of markers handled at a time, smaller value would reduce the memory cost **ncpus**, number of CPUs used for parallel computation, If not set, all CPUs will be used by default **vc.method**, methods of variance components analysis, three methods are avaiblable, "BRENT", "EMMA", and "HE" **maxLoop**, a parameter for FarmCPU only, the maximum iterations allowed in FarmCPU @@ -568,23 +563,20 @@ Three models are included in MVP package: General Linear Model (GLM), Mixed Line ```r imMVP <- MVP( - phe=phenotype, + phe=phenotype, #NA is acceptable in phenotype geno=genotype, map=map, - #K=Kinship, - #CV.GLM=Covariates, ##if you have additional covariates, please keep there open. + #K=Kinship, #if you have pre-computed GRM, please keep there open, otherwise rMVP will compute it automatically + #CV.GLM=Covariates, #if you have environmental covariates, please keep all 'CV.*' open #CV.MLM=Covariates, #CV.FarmCPU=Covariates, - nPC.GLM=5, ##if you have added PCs into covariates, please keep there closed. - nPC.MLM=3, ##if you don't want to add PCs as covariates, please comment out the parameters instead of setting the nPC to 0. + nPC.GLM=5, #if you have added PCs into covariates, please keep there closed + nPC.MLM=3, #if you don't want to add PCs as covariates, please comment out the parameter instead of setting it to 0. nPC.FarmCPU=3, - priority="speed", ##for Kinship construction + maxLine=10000, #smaller value would reduce the memory cost #ncpus=10, - vc.method="BRENT", ##only works for MLM - maxLoop=10, - method.bin="static", ## "FaST-LMM", "static" (#only works for FarmCPU) - #permutation.threshold=TRUE, - #permutation.rep=100, + vc.method="BRENT", #only works for MLM + method.bin="static", # "FaST-LMM", "static" (#only works for FarmCPU) threshold=0.05, method=c("GLM", "MLM", "FarmCPU"), file.output=c("pmap", "pmap.signal", "plot", "log") @@ -606,13 +598,10 @@ for(i in 2:ncol(phenotype)){ nPC.GLM=5, nPC.MLM=3, nPC.FarmCPU=3, - priority="speed", + maxLine=10000, #ncpus=10, vc.method="BRENT", - maxLoop=10, method.bin="static", - #permutation.threshold=TRUE, - #permutation.rep=100, threshold=0.05, method=c("GLM", "MLM", "FarmCPU"), file.output=c("pmap", "pmap.signal", "plot", "log") diff --git a/man/MVP.Data.Bfile2MVP.Rd b/man/MVP.Data.Bfile2MVP.Rd index 8cfbe73..750038c 100644 --- a/man/MVP.Data.Bfile2MVP.Rd +++ b/man/MVP.Data.Bfile2MVP.Rd @@ -10,7 +10,6 @@ MVP.Data.Bfile2MVP( bfile, out = "mvp", maxLine = 10000, - priority = "speed", type.geno = "char", threads = 0, verbose = TRUE @@ -21,9 +20,7 @@ MVP.Data.Bfile2MVP( \item{out}{the name of output file} -\item{maxLine}{the max number of line to write to big matrix for each loop} - -\item{priority}{'memory' or 'speed'} +\item{maxLine}{the number of markers handled at a time, smaller value would reduce the memory cost} \item{type.geno}{the type of genotype elements} diff --git a/man/MVP.Data.Hapmap2MVP.Rd b/man/MVP.Data.Hapmap2MVP.Rd index c95a6f9..0909e2a 100644 --- a/man/MVP.Data.Hapmap2MVP.Rd +++ b/man/MVP.Data.Hapmap2MVP.Rd @@ -20,7 +20,7 @@ MVP.Data.Hapmap2MVP( \item{out}{the name of output file} -\item{maxLine}{the max number of line to write to big matrix for each loop} +\item{maxLine}{the number of markers handled at a time, smaller value would reduce the memory cost} \item{type.geno}{the type of genotype elements} diff --git a/man/MVP.Data.Kin.Rd b/man/MVP.Data.Kin.Rd index b272922..bb4172b 100644 --- a/man/MVP.Data.Kin.Rd +++ b/man/MVP.Data.Kin.Rd @@ -8,9 +8,9 @@ MVP.Data.Kin( fileKin = TRUE, mvp_prefix = "mvp", out = NULL, - priority = "speed", + maxLine = 10000, sep = "\\t", - cpus = 1, + cpu = 1, verbose = TRUE ) } @@ -21,11 +21,11 @@ MVP.Data.Kin( \item{out}{prefix of output file name} -\item{priority}{"speed" or "memory"} +\item{maxLine}{the number of markers handled at a time, smaller value would reduce the memory cost} \item{sep}{seperator for Kinship file.} -\item{cpus}{the number of cpu} +\item{cpu}{the number of cpu} \item{verbose}{whether to print detail.} } @@ -41,7 +41,7 @@ Kinship \donttest{ geno <- file.path(system.file("extdata", "06_mvp-impute", package = "rMVP"), "mvp.imp") -MVP.Data.Kin(TRUE, mvp_prefix=geno, out=tempfile("outfile"), cpus=1) +MVP.Data.Kin(TRUE, mvp_prefix=geno, out=tempfile("outfile"), cpu=1) } } diff --git a/man/MVP.Data.Numeric2MVP.Rd b/man/MVP.Data.Numeric2MVP.Rd index ca87b9a..954f0d0 100644 --- a/man/MVP.Data.Numeric2MVP.Rd +++ b/man/MVP.Data.Numeric2MVP.Rd @@ -11,7 +11,6 @@ MVP.Data.Numeric2MVP( map_file, out = "mvp", maxLine = 10000, - priority = "speed", row_names = FALSE, col_names = FALSE, type.geno = "char", @@ -26,9 +25,7 @@ MVP.Data.Numeric2MVP( \item{out}{the name of output file} -\item{maxLine}{the max number of line to write to big matrix for each loop} - -\item{priority}{'memory' or 'speed'} +\item{maxLine}{the number of markers handled at a time, smaller value would reduce the memory cost} \item{row_names}{whether the numeric genotype has row names} diff --git a/man/MVP.Data.PC.Rd b/man/MVP.Data.PC.Rd index 926d1ee..603d7cf 100644 --- a/man/MVP.Data.PC.Rd +++ b/man/MVP.Data.PC.Rd @@ -10,9 +10,9 @@ MVP.Data.PC( K = NULL, out = NULL, pcs.keep = 5, - priority = "speed", + maxLine = 10000, sep = "\\t", - cpus = 1, + cpu = 1, verbose = TRUE ) } @@ -27,11 +27,11 @@ MVP.Data.PC( \item{pcs.keep}{how many PCs to keep} -\item{priority}{speed or memory} +\item{maxLine}{the number of markers handled at a time, smaller value would reduce the memory cost} \item{sep}{seperator for PC file.} -\item{cpus}{the number of cpu} +\item{cpu}{the number of cpu} \item{verbose}{whether to print detail.} } @@ -47,7 +47,7 @@ Principal component analysis \donttest{ geno <- file.path(system.file("extdata", "06_mvp-impute", package = "rMVP"), "mvp.imp") -MVP.Data.PC(TRUE, mvp_prefix=geno, out=tempfile("outfile"), cpus=1) +MVP.Data.PC(TRUE, mvp_prefix=geno, out=tempfile("outfile"), cpu=1) } } diff --git a/man/MVP.Data.Rd b/man/MVP.Data.Rd index 7d193b2..c2f722c 100644 --- a/man/MVP.Data.Rd +++ b/man/MVP.Data.Rd @@ -29,7 +29,6 @@ MVP.Data( pheno_cols = NULL, SNP.impute = "Major", maxLine = 10000, - priority = "speed", pcs.keep = 5, verbose = TRUE, ncpus = NULL, @@ -80,9 +79,7 @@ the column represents the individual.} \item{SNP.impute}{"Left", "Middle", "Right", or NULL for skip impute.} -\item{maxLine}{number of SNPs, only used for saving memory when calculate kinship matrix} - -\item{priority}{"speed" or "memory"} +\item{maxLine}{the number of markers handled at a time, smaller value would reduce the memory cost} \item{pcs.keep}{how many PCs to keep} diff --git a/man/MVP.Data.VCF2MVP.Rd b/man/MVP.Data.VCF2MVP.Rd index 7453ad6..72f88d2 100644 --- a/man/MVP.Data.VCF2MVP.Rd +++ b/man/MVP.Data.VCF2MVP.Rd @@ -20,7 +20,7 @@ MVP.Data.VCF2MVP( \item{out}{the name of output file} -\item{maxLine}{the max number of line to write to big matrix for each loop} +\item{maxLine}{the number of markers handled at a time, smaller value would reduce the memory cost} \item{type.geno}{the type of genotype elements} diff --git a/man/MVP.FarmCPU.Rd b/man/MVP.FarmCPU.Rd index b85b8de..7cbeca6 100755 --- a/man/MVP.FarmCPU.Rd +++ b/man/MVP.FarmCPU.Rd @@ -9,6 +9,7 @@ MVP.FarmCPU( geno, map, CV = NULL, + geno_ind_idx = NULL, P = NULL, method.sub = "reward", method.sub.final = "reward", @@ -37,6 +38,8 @@ MVP.FarmCPU( \item{CV}{covariates, n by c matrix, n is sample size, c is number of covariates} +\item{geno_ind_idx}{the index of effective genotyped individuals} + \item{P}{start p values for all SNPs} \item{method.sub}{method used in substitution process, five options: 'penalty', 'reward', 'mean', 'median', or 'onsite'} diff --git a/man/MVP.GLM.Rd b/man/MVP.GLM.Rd index c063d85..e0d42c5 100755 --- a/man/MVP.GLM.Rd +++ b/man/MVP.GLM.Rd @@ -4,7 +4,7 @@ \alias{MVP.GLM} \title{To perform GWAS with GLM and MLM model and get the P value of SNPs} \usage{ -MVP.GLM(phe, geno, CV = NULL, cpu = 1, verbose = TRUE) +MVP.GLM(phe, geno, CV = NULL, geno_ind_idx = NULL, cpu = 1, verbose = TRUE) } \arguments{ \item{phe}{phenotype, n * 2 matrix} @@ -13,6 +13,8 @@ MVP.GLM(phe, geno, CV = NULL, cpu = 1, verbose = TRUE) \item{CV}{Covariance, design matrix(n * x) for the fixed effects} +\item{geno_ind_idx}{the index of effective genotyped individuals} + \item{cpu}{number of cpus used for parallel computation} \item{verbose}{whether to print detail.} diff --git a/man/MVP.K.VanRaden.Rd b/man/MVP.K.VanRaden.Rd index 129e4cb..4ba350c 100755 --- a/man/MVP.K.VanRaden.Rd +++ b/man/MVP.K.VanRaden.Rd @@ -4,16 +4,27 @@ \alias{MVP.K.VanRaden} \title{Calculate Kinship matrix by VanRaden method} \usage{ -MVP.K.VanRaden(M, priority = c("speed", "memory"), cpu = 1, verbose = TRUE) +MVP.K.VanRaden( + M, + step = 5000, + ind_idx = NULL, + cpu = 1, + verbose = TRUE, + checkNA = TRUE +) } \arguments{ \item{M}{Genotype, m * n, m is marker size, n is population size} -\item{priority}{speed or memory} +\item{step}{the number of markers handled at a time, smaller value would reduce the memory cost} + +\item{ind_idx}{the index of effective genotyped individuals} \item{cpu}{the number of cpu} \item{verbose}{whether to print detail.} + +\item{checkNA}{whether to check NA in genotype.} } \value{ K, n * n matrix diff --git a/man/MVP.MLM.Rd b/man/MVP.MLM.Rd index aecbbc6..c0378df 100755 --- a/man/MVP.MLM.Rd +++ b/man/MVP.MLM.Rd @@ -10,6 +10,7 @@ MVP.MLM( K = NULL, eigenK = NULL, CV = NULL, + geno_ind_idx = NULL, REML = NULL, cpu = 1, vc.method = c("BRENT", "EMMA", "HE"), @@ -27,6 +28,8 @@ MVP.MLM( \item{CV}{covariates} +\item{geno_ind_idx}{the index of effective genotyped individuals} + \item{REML}{a list that contains ve and vg} \item{cpu}{number of cpus used for parallel computation} diff --git a/man/MVP.PCA.Rd b/man/MVP.PCA.Rd index 1a0feb2..5257742 100755 --- a/man/MVP.PCA.Rd +++ b/man/MVP.PCA.Rd @@ -7,7 +7,8 @@ MVP.PCA( M = NULL, K = NULL, - priority = c("speed", "memory"), + maxLine, + ind_idx = NULL, pcs.keep = 5, cpu = 1, verbose = TRUE @@ -18,7 +19,7 @@ MVP.PCA( \item{K}{kinship matrix} -\item{priority}{speed or memory} +\item{maxLine}{the number of markers handled at a time, smaller value would reduce the memory cost} \item{pcs.keep}{maximum number of PCs for output} diff --git a/man/MVP.Rd b/man/MVP.Rd index b644bea..df2f07d 100755 --- a/man/MVP.Rd +++ b/man/MVP.Rd @@ -16,7 +16,7 @@ MVP( CV.MLM = NULL, CV.FarmCPU = NULL, REML = NULL, - priority = "speed", + maxLine = 10000, ncpus = detectCores(logical = FALSE), vc.method = c("BRENT", "EMMA", "HE"), method = c("GLM", "MLM", "FarmCPU"), @@ -62,7 +62,7 @@ MVP( \item{REML}{a list contains ve and vg} -\item{priority}{speed or memory} +\item{maxLine}{the number of markers handled at a time, smaller value would reduce the memory cost} \item{ncpus}{number of cpus used for parallel} diff --git a/src/Makevars b/src/Makevars index 48f15f8..9b030a3 100755 --- a/src/Makevars +++ b/src/Makevars @@ -13,7 +13,8 @@ ## _In general_ we should no longer need to set a standard as any recent R ## installation will do the right thing. Should you need it, uncomment it and ## set the appropriate value, possibly CXX17. -#CXX_STD = CXX11 -PKG_CXXFLAGS = $(SHLIB_OPENMP_CXXFLAGS) -PKG_LIBS = $(SHLIB_OPENMP_CXXFLAGS) $(LAPACK_LIBS) $(BLAS_LIBS) $(FLIBS) \ No newline at end of file +## CXX_STD = CXX11 + +PKG_CXXFLAGS = -DARMA_64BIT_WORD=1 $(SHLIB_OPENMP_CXXFLAGS) +PKG_LIBS = $(SHLIB_OPENMP_CXXFLAGS) $(LAPACK_LIBS) $(BLAS_LIBS) $(FLIBS) diff --git a/src/Makevars.win b/src/Makevars.win index 48f15f8..9b030a3 100755 --- a/src/Makevars.win +++ b/src/Makevars.win @@ -13,7 +13,8 @@ ## _In general_ we should no longer need to set a standard as any recent R ## installation will do the right thing. Should you need it, uncomment it and ## set the appropriate value, possibly CXX17. -#CXX_STD = CXX11 -PKG_CXXFLAGS = $(SHLIB_OPENMP_CXXFLAGS) -PKG_LIBS = $(SHLIB_OPENMP_CXXFLAGS) $(LAPACK_LIBS) $(BLAS_LIBS) $(FLIBS) \ No newline at end of file +## CXX_STD = CXX11 + +PKG_CXXFLAGS = -DARMA_64BIT_WORD=1 $(SHLIB_OPENMP_CXXFLAGS) +PKG_LIBS = $(SHLIB_OPENMP_CXXFLAGS) $(LAPACK_LIBS) $(BLAS_LIBS) $(FLIBS) diff --git a/src/RcppExports.cpp b/src/RcppExports.cpp old mode 100755 new mode 100644 index bcc2d9a..fc92566 --- a/src/RcppExports.cpp +++ b/src/RcppExports.cpp @@ -25,8 +25,8 @@ BEGIN_RCPP END_RCPP } // glm_c -SEXP glm_c(const arma::vec& y, const arma::mat& X, const arma::mat& iXX, SEXP pBigMat, const bool verbose, const int threads); -RcppExport SEXP _rMVP_glm_c(SEXP ySEXP, SEXP XSEXP, SEXP iXXSEXP, SEXP pBigMatSEXP, SEXP verboseSEXP, SEXP threadsSEXP) { +SEXP glm_c(const arma::vec& y, const arma::mat& X, const arma::mat& iXX, SEXP pBigMat, const Nullable geno_ind, const bool verbose, const int threads); +RcppExport SEXP _rMVP_glm_c(SEXP ySEXP, SEXP XSEXP, SEXP iXXSEXP, SEXP pBigMatSEXP, SEXP geno_indSEXP, SEXP verboseSEXP, SEXP threadsSEXP) { BEGIN_RCPP Rcpp::RObject rcpp_result_gen; Rcpp::RNGScope rcpp_rngScope_gen; @@ -34,15 +34,16 @@ BEGIN_RCPP Rcpp::traits::input_parameter< const arma::mat& >::type X(XSEXP); Rcpp::traits::input_parameter< const arma::mat& >::type iXX(iXXSEXP); Rcpp::traits::input_parameter< SEXP >::type pBigMat(pBigMatSEXP); + Rcpp::traits::input_parameter< const Nullable >::type geno_ind(geno_indSEXP); Rcpp::traits::input_parameter< const bool >::type verbose(verboseSEXP); Rcpp::traits::input_parameter< const int >::type threads(threadsSEXP); - rcpp_result_gen = Rcpp::wrap(glm_c(y, X, iXX, pBigMat, verbose, threads)); + rcpp_result_gen = Rcpp::wrap(glm_c(y, X, iXX, pBigMat, geno_ind, verbose, threads)); return rcpp_result_gen; END_RCPP } // mlm_c -SEXP mlm_c(const arma::vec& y, const arma::mat& X, const arma::mat& U, const double vgs, SEXP pBigMat, const bool verbose, const int threads); -RcppExport SEXP _rMVP_mlm_c(SEXP ySEXP, SEXP XSEXP, SEXP USEXP, SEXP vgsSEXP, SEXP pBigMatSEXP, SEXP verboseSEXP, SEXP threadsSEXP) { +SEXP mlm_c(const arma::vec& y, const arma::mat& X, const arma::mat& U, const double vgs, SEXP pBigMat, const Nullable geno_ind, const bool verbose, const int threads); +RcppExport SEXP _rMVP_mlm_c(SEXP ySEXP, SEXP XSEXP, SEXP USEXP, SEXP vgsSEXP, SEXP pBigMatSEXP, SEXP geno_indSEXP, SEXP verboseSEXP, SEXP threadsSEXP) { BEGIN_RCPP Rcpp::RObject rcpp_result_gen; Rcpp::RNGScope rcpp_rngScope_gen; @@ -51,9 +52,10 @@ BEGIN_RCPP Rcpp::traits::input_parameter< const arma::mat& >::type U(USEXP); Rcpp::traits::input_parameter< const double >::type vgs(vgsSEXP); Rcpp::traits::input_parameter< SEXP >::type pBigMat(pBigMatSEXP); + Rcpp::traits::input_parameter< const Nullable >::type geno_ind(geno_indSEXP); Rcpp::traits::input_parameter< const bool >::type verbose(verboseSEXP); Rcpp::traits::input_parameter< const int >::type threads(threadsSEXP); - rcpp_result_gen = Rcpp::wrap(mlm_c(y, X, U, vgs, pBigMat, verbose, threads)); + rcpp_result_gen = Rcpp::wrap(mlm_c(y, X, U, vgs, pBigMat, geno_ind, verbose, threads)); return rcpp_result_gen; END_RCPP } @@ -84,12 +86,12 @@ BEGIN_RCPP END_RCPP } // hapmap_parser_map -List hapmap_parser_map(Rcpp::StringVector hmp_file, std::string out); +List hapmap_parser_map(std::string hmp_file, std::string out); RcppExport SEXP _rMVP_hapmap_parser_map(SEXP hmp_fileSEXP, SEXP outSEXP) { BEGIN_RCPP Rcpp::RObject rcpp_result_gen; Rcpp::RNGScope rcpp_rngScope_gen; - Rcpp::traits::input_parameter< Rcpp::StringVector >::type hmp_file(hmp_fileSEXP); + Rcpp::traits::input_parameter< std::string >::type hmp_file(hmp_fileSEXP); Rcpp::traits::input_parameter< std::string >::type out(outSEXP); rcpp_result_gen = Rcpp::wrap(hapmap_parser_map(hmp_file, out)); return rcpp_result_gen; @@ -240,11 +242,27 @@ BEGIN_RCPP return rcpp_result_gen; END_RCPP } +// kin_cal +SEXP kin_cal(SEXP pBigMat, const Nullable geno_ind, int threads, size_t step, bool mkl, bool verbose); +RcppExport SEXP _rMVP_kin_cal(SEXP pBigMatSEXP, SEXP geno_indSEXP, SEXP threadsSEXP, SEXP stepSEXP, SEXP mklSEXP, SEXP verboseSEXP) { +BEGIN_RCPP + Rcpp::RObject rcpp_result_gen; + Rcpp::RNGScope rcpp_rngScope_gen; + Rcpp::traits::input_parameter< SEXP >::type pBigMat(pBigMatSEXP); + Rcpp::traits::input_parameter< const Nullable >::type geno_ind(geno_indSEXP); + Rcpp::traits::input_parameter< int >::type threads(threadsSEXP); + Rcpp::traits::input_parameter< size_t >::type step(stepSEXP); + Rcpp::traits::input_parameter< bool >::type mkl(mklSEXP); + Rcpp::traits::input_parameter< bool >::type verbose(verboseSEXP); + rcpp_result_gen = Rcpp::wrap(kin_cal(pBigMat, geno_ind, threads, step, mkl, verbose)); + return rcpp_result_gen; +END_RCPP +} static const R_CallMethodDef CallEntries[] = { {"_rMVP_getRow", (DL_FUNC) &_rMVP_getRow, 2}, - {"_rMVP_glm_c", (DL_FUNC) &_rMVP_glm_c, 6}, - {"_rMVP_mlm_c", (DL_FUNC) &_rMVP_mlm_c, 7}, + {"_rMVP_glm_c", (DL_FUNC) &_rMVP_glm_c, 7}, + {"_rMVP_mlm_c", (DL_FUNC) &_rMVP_mlm_c, 8}, {"_rMVP_vcf_parser_map", (DL_FUNC) &_rMVP_vcf_parser_map, 2}, {"_rMVP_vcf_parser_genotype", (DL_FUNC) &_rMVP_vcf_parser_genotype, 5}, {"_rMVP_hapmap_parser_map", (DL_FUNC) &_rMVP_hapmap_parser_map, 2}, @@ -259,6 +277,7 @@ static const R_CallMethodDef CallEntries[] = { {"_rMVP_hasNA", (DL_FUNC) &_rMVP_hasNA, 2}, {"_rMVP_kin_cal_m", (DL_FUNC) &_rMVP_kin_cal_m, 3}, {"_rMVP_kin_cal_s", (DL_FUNC) &_rMVP_kin_cal_s, 4}, + {"_rMVP_kin_cal", (DL_FUNC) &_rMVP_kin_cal, 6}, {NULL, NULL, 0} }; diff --git a/src/assoc.cpp b/src/assoc.cpp index ee471a7..1e35c6b 100755 --- a/src/assoc.cpp +++ b/src/assoc.cpp @@ -1,11 +1,4 @@ -#include -#include "mvp_omp.h" -#include -#include -#include -#include -#include -#include "progress_bar.hpp" +#include "rMVP.h" // [[Rcpp::depends(bigmemory, BH)]] // [[Rcpp::depends(RcppArmadillo)]] @@ -15,140 +8,6 @@ using namespace std; using namespace Rcpp; using namespace arma; -class MinimalProgressBar_plus: public ProgressBar{ - public: - MinimalProgressBar_plus() { - _finalized = false; - } - - ~MinimalProgressBar_plus() {} - - std::string _time_to_string(double seconds) { - - int time = (int) seconds; - - int hour = 0; - int min = 0; - int sec = 0; - - hour = time / 3600; - time = time % 3600; - min = time / 60; - time = time % 60; - sec = time; - - std::stringstream time_strs; - time_strs << "TimeLeft: "; - if (hour != 0) time_strs << hour << "h"; - if (hour != 0 || min != 0) time_strs << min << "m"; - time_strs << sec << "s"; - std::string time_str = time_strs.str(); - - return time_str; - } - - int _compute_nb_ticks(float progress) { - return int(progress * _max_ticks); - } - - std::string _construct_ticks_display_string(int nb) { - std::stringstream ticks_strs; - for (int i = 1; i <= _max_ticks; ++i) { - if(i < 4){ - ticks_strs << ">"; - } else if (i < nb) { - ticks_strs << "-"; - } else if(i == nb) { - ticks_strs << ">"; - } else { - ticks_strs << " "; - } - } - std::string tick_space_string = ticks_strs.str(); - return tick_space_string; - } - - void end_display() { - update(1); - } - - void _finalize_display() { - if (_finalized) return; - REprintf("\n"); - _finalized = true; - } - - void display() { - REprintf("\r"); - } - - void update(float progress) { - - // stop if already finalized - if (_finalized) return; - - // start time measurement when update() is called the first time - if (_timer_flag) { - _timer_flag = false; - // measure start time - time(&start); - } else { - - int nb_ticks = _compute_nb_ticks(progress); - int delta = nb_ticks - _ticks_displayed; - if (delta > 0) { - _ticks_displayed = nb_ticks; - std::string cur_display = _construct_ticks_display_string(nb_ticks); - - // measure current time - time(&end); - - // calculate passed time and remaining time (in seconds) - double pas_time = std::difftime(end, start); - double rem_time = (pas_time / progress) * (1 - progress); - if(rem_time < 1 && rem_time > 0.5) rem_time = 1; - - // convert seconds to time string - std::string time_string = _time_to_string(rem_time); - - // ensure overwriting of old time info - int empty_length = time_string.length(); - - std::string empty_space; - - // merge progress bar and time string - std::stringstream strs; - if(empty_length_p && abs(empty_length - empty_length_p)){ - empty_space = std::string(abs(empty_length - empty_length_p), ' '); - strs << "[" << cur_display << "] " << time_string << empty_space; - }else{ - strs << "[" << cur_display << "] " << time_string; - } - empty_length_p = empty_length; - // strs << "[" << cur_display << "]"; - - std::string temp_str = strs.str(); - char const* char_type = temp_str.c_str(); - - // print: remove old display and replace with new - REprintf("\r"); - REprintf("%s", char_type); - - } - if (_ticks_displayed >= _max_ticks) - // end_display(); - _finalize_display(); - } - } - private: - int empty_length_p = 0; - int _max_ticks = 45; - bool _finalized; - bool _timer_flag = true; - time_t start, end; - int _ticks_displayed = 0; -}; - arma::mat GInv(const arma::mat A){ arma::mat ginv; @@ -208,19 +67,24 @@ NumericVector getRow(SEXP pBigMat, const int row){ } template -SEXP glm_c(const arma::vec &y, const arma::mat &X, const arma::mat & iXX, XPtr pMat, const bool verbose = true, const int threads = 0){ +SEXP glm_c(const arma::vec &y, const arma::mat &X, const arma::mat & iXX, XPtr pMat, const Nullable geno_ind = R_NilValue, const bool verbose = true, const int threads = 0){ omp_setup(threads); MatrixAccessor genomat = MatrixAccessor(*pMat); - int ind = pMat->ncol(); + uvec _geno_ind; + if(geno_ind.isNotNull()){ + _geno_ind = as(geno_ind) - 1; + }else{ + _geno_ind = regspace(0, pMat->ncol() - 1); + } + + int ind = _geno_ind.n_elem; int mkr = pMat->nrow(); int q0 = X.n_cols; - int y_len = y.n_elem; - if(y_len != ind) - throw Rcpp::exception("number of individuals not match.!"); + if(y.n_elem != ind) throw Rcpp::exception("number of individuals not match!"); MinimalProgressBar_plus pb; Progress progress(mkr, verbose, pb); @@ -235,8 +99,8 @@ SEXP glm_c(const arma::vec &y, const arma::mat &X, const arma::mat & iXX, XPtr geno_ind = R_NilValue, const bool verbose = true, const int threads = 0){ XPtr xpMat(pBigMat); switch(xpMat->matrix_type()){ case 1: - return glm_c(y, X, iXX, xpMat, verbose, threads); + return glm_c(y, X, iXX, xpMat, geno_ind, verbose, threads); case 2: - return glm_c(y, X, iXX, xpMat, verbose, threads); + return glm_c(y, X, iXX, xpMat, geno_ind, verbose, threads); case 4: - return glm_c(y, X, iXX, xpMat, verbose, threads); + return glm_c(y, X, iXX, xpMat, geno_ind, verbose, threads); case 8: - return glm_c(y, X, iXX, xpMat, verbose, threads); + return glm_c(y, X, iXX, xpMat, geno_ind, verbose, threads); default: throw Rcpp::exception("unknown type detected for big.matrix object!"); } } template -SEXP mlm_c(const arma::vec & y, const arma::mat & X, const arma::mat & U, const double vgs, XPtr pMat, const bool verbose = true, const int threads = 0){ +SEXP mlm_c(const arma::vec & y, const arma::mat & X, const arma::mat & U, const double vgs, XPtr pMat, const Nullable geno_ind = R_NilValue, const bool verbose = true, const int threads = 0){ omp_setup(threads); MatrixAccessor genomat = MatrixAccessor(*pMat); - int ind = pMat->ncol(); + uvec _geno_ind; + if(geno_ind.isNotNull()){ + _geno_ind = as(geno_ind) - 1; + }else{ + _geno_ind = regspace(0, pMat->ncol() - 1); + } + + int ind = _geno_ind.n_elem; int mkr = pMat->nrow(); int q0 = X.n_cols; - int y_len = y.n_elem; - if(y_len != ind) - throw Rcpp::exception("number of individuals not match.!"); + if(y.n_elem != ind) throw Rcpp::exception("number of individuals not match.!"); MinimalProgressBar_plus pb; Progress progress(mkr, verbose, pb); @@ -340,9 +209,10 @@ SEXP mlm_c(const arma::vec & y, const arma::mat & X, const arma::mat & U, const #pragma omp parallel for schedule(dynamic) firstprivate(snp, iXXs) for(int i = 0; i < mkr; i++){ - for(int ii = 0; ii < ind; ii++){ - snp[ii] = genomat[ii][i]; + for(uword ii = 0; ii < ind; ii++){ + snp[ii] = genomat[_geno_ind[ii]][i]; } + arma::mat Us = U.t() * snp; arma::mat UXUs = UX.t() * Us; @@ -375,19 +245,19 @@ SEXP mlm_c(const arma::vec & y, const arma::mat & X, const arma::mat & U, const } // [[Rcpp::export]] -SEXP mlm_c(const arma::vec & y, const arma::mat & X, const arma::mat & U, const double vgs, SEXP pBigMat, const bool verbose = true, const int threads = 0){ +SEXP mlm_c(const arma::vec & y, const arma::mat & X, const arma::mat & U, const double vgs, SEXP pBigMat, const Nullable geno_ind = R_NilValue, const bool verbose = true, const int threads = 0){ XPtr xpMat(pBigMat); switch(xpMat->matrix_type()){ case 1: - return mlm_c(y, X, U, vgs, xpMat, verbose, threads); + return mlm_c(y, X, U, vgs, xpMat, geno_ind, verbose, threads); case 2: - return mlm_c(y, X, U, vgs, xpMat, verbose, threads); + return mlm_c(y, X, U, vgs, xpMat, geno_ind, verbose, threads); case 4: - return mlm_c(y, X, U, vgs, xpMat, verbose, threads); + return mlm_c(y, X, U, vgs, xpMat, geno_ind, verbose, threads); case 8: - return mlm_c(y, X, U, vgs, xpMat, verbose, threads); + return mlm_c(y, X, U, vgs, xpMat, geno_ind, verbose, threads); default: throw Rcpp::exception("unknown type detected for big.matrix object!"); } diff --git a/src/data_converter.cpp b/src/data_converter.cpp index 3ac8950..4153b44 100755 --- a/src/data_converter.cpp +++ b/src/data_converter.cpp @@ -14,15 +14,14 @@ // See the License for the specific language governing permissions and // limitations under the License. - -#include "mvp_omp.h" +#include "rMVP.h" #include #include -#include #include using namespace std; using namespace Rcpp; +using namespace arma; // [[Rcpp::depends(BH, bigmemory)]] // [[Rcpp::plugins(openmp)]] @@ -31,6 +30,24 @@ using namespace Rcpp; #include #include +vector split_line(const string& str, const string& whitespace = " ,\t\r\n") +{ + vector result; + size_t col_end = 0; + while (true) { + size_t col_begin = str.find_first_not_of(whitespace, col_end); + // LOG_TRACE_FUNC << "col_begin: " << col_begin << endl; + if (col_begin == std::string::npos) + return result; + col_end = str.find_first_of(whitespace, col_begin); + // LOG_TRACE_FUNC << "col_end: " << col_end << endl; + if (col_end == std::string::npos) + col_end = str.size(); + string stri = str.substr(col_begin, col_end - col_begin); + stri.erase(stri.find_last_not_of(" \n\r\t") + 1); + result.push_back(stri); + } +} // ***** VCF ***** @@ -58,11 +75,12 @@ List vcf_parser_map(std::string vcf_file, std::string out) { } } if (!have_header) { - Rcpp::stop("ERROR: Wrong VCF file, no line begin with \"#CHROM\"."); + Rcpp::stop("wrong VCF file, no line begin with \"#CHROM\"."); } // Write inds to file. - boost::split(ind, line, boost::is_any_of("\t")); + ind = split_line(line); + // boost::split(ind, line, boost::is_any_of("\t")); vector(ind.begin() + 9, ind.end()).swap(ind); // delete first 9 columns int indn = ind.size(); for (int i = 0; i < indn; i++) { @@ -82,7 +100,8 @@ List vcf_parser_map(std::string vcf_file, std::string out) { m = 0; while (getline(file, line)) { string tmp = line.substr(0, MAP_INFO_N); - boost::split(l, tmp, boost::is_any_of("\t")); + l = split_line(tmp); + // boost::split(l, tmp, boost::is_any_of("\t")); if (l[2] == ".") { // snp name missing l[2] = l[0] + '-' + l[1]; @@ -119,7 +138,8 @@ void vcf_parser_genotype(std::string vcf_file, XPtr pMat, long maxLin MatrixAccessor mat = MatrixAccessor(*pMat); // progress bar - Progress progress(pMat->nrow(), verbose); + MinimalProgressBar_perc pb; + Progress progress(pMat->nrow(), verbose, pb); // Skip Header string prefix("#CHROM"); @@ -132,7 +152,7 @@ void vcf_parser_genotype(std::string vcf_file, XPtr pMat, long maxLin } } if (!have_header) { - Rcpp::stop("ERROR: Wrong VCF file, no line begin with \"#CHROM\"."); + Rcpp::stop("wrong VCF file, no line begin with \"#CHROM\"."); } // parser genotype @@ -147,7 +167,8 @@ void vcf_parser_genotype(std::string vcf_file, XPtr pMat, long maxLin } #pragma omp parallel for private(l, markers) for (std::size_t i = 0; i < buffer.size(); i++) { - boost::split(l, buffer[i], boost::is_any_of("\t")); + l = split_line(buffer[i]); + // boost::split(l, buffer[i], boost::is_any_of("\t")); markers.clear(); // There is only one char in REF and ALT // if (l[3].length() == 1 && l[4].length() == 1) { @@ -165,8 +186,8 @@ void vcf_parser_genotype(std::string vcf_file, XPtr pMat, long maxLin for (std::size_t j = 0; j < markers.size(); j++) { mat[j][m + i] = markers[j]; } - progress.increment(); } + progress.increment(buffer.size()); m += buffer.size(); } } @@ -193,9 +214,9 @@ void vcf_parser_genotype(std::string vcf_file, SEXP pBigMat, long maxLine, int t // ***** HAPMAP ***** // [[Rcpp::export]] -List hapmap_parser_map(Rcpp::StringVector hmp_file, std::string out) { +List hapmap_parser_map(std::string hmp_file, std::string out) { // Define - const int MAP_INFO_N = 10000; // max length of "SNP, POS and CHROM" + const int MAP_INFO_N = 1000; // max length of "SNP, POS and CHROM" ofstream map(out + ".geno.map"); ofstream indfile(out + ".geno.ind"); @@ -207,8 +228,8 @@ List hapmap_parser_map(Rcpp::StringVector hmp_file, std::string out) { size_t n; size_t m; - for (int i = 0; i < hmp_file.size(); i++) { // TODO(haohao): Support mulit hmp file. - ifstream file(hmp_file(i)); + // for (int i = 0; i < hmp_file.size(); i++) { // TODO(haohao): Support mulit hmp file. + ifstream file(hmp_file); // Skip Header string prefix("rs#"); @@ -221,11 +242,12 @@ List hapmap_parser_map(Rcpp::StringVector hmp_file, std::string out) { } } if (!have_header) { - Rcpp::stop("ERROR: Wrong HAPMAP file, no line begin with \"rs#\"."); + Rcpp::stop("wrong HAPMAP file, no line begin with \"rs#\"."); } // Write inds to file. - boost::split(ind, line, boost::is_any_of(" \t")); + ind = split_line(line); + // boost::split(ind, line, boost::is_any_of(" \t")); vector(ind.begin() + 11, ind.end()).swap(ind); // delete first 11 columns for (size_t i = 0; i < ind.size(); i++) { indfile << ind[i] << endl; @@ -238,7 +260,8 @@ List hapmap_parser_map(Rcpp::StringVector hmp_file, std::string out) { m = 0; while (getline(file, line)) { string tmp = line.substr(0, MAP_INFO_N); - boost::split(l, tmp, boost::is_any_of(" \t")); + l = split_line(tmp); + // boost::split(l, tmp, boost::is_any_of(" \t")); if (l[0] == ".") { // snp name missing l[0] = l[2] + '-' + l[3]; @@ -246,33 +269,30 @@ List hapmap_parser_map(Rcpp::StringVector hmp_file, std::string out) { vector alleles1; vector alleles; - vector atcg{"A", "T", "C", "G"}; + // vector atcg{"A", "T", "C", "G"}; + std::unordered_set atcg{"A", "T", "C", "G"}; - boost::split(alleles1, l[1], boost::is_any_of("/")); + alleles1 = split_line(l[1], "/"); + // boost::split(alleles1, l[1], boost::is_any_of("/")); for(size_t ii = 0; ii < alleles1.size(); ii++){ if (alleles1[ii].length() != 1) { - Rcpp::stop(("ERROR: unknown variants [" + l[1] + "] at " + to_string(m+2) + "th row of second column in HAPMAP file.").c_str()); + Rcpp::stop(("unknown variants [" + l[1] + "] at " + to_string(m+2) + "th row of second column in HAPMAP file.").c_str()); } - for(int jj = 0; jj < 4; jj++){ - if(atcg[jj] == alleles1[ii]){ - alleles.push_back(alleles1[ii]); - break; - } + if (atcg.find(alleles1[ii]) != atcg.end()) { + alleles.push_back(alleles1[ii]); } } - if (alleles.size() == 0) { - Rcpp::stop(("ERROR: unknown variants at " + to_string(m+2) + "th row of second column in HAPMAP file.").c_str()); + Rcpp::stop(("unknown variants at " + to_string(m+2) + "th row of second column in HAPMAP file.").c_str()); } - if (alleles.size() == 1) { alleles.push_back(alleles[0]); } - if (alleles.size() > 2) { - Rcpp::stop(("ERROR: variants with more than 2 alleles at " + to_string(m+2) + "th row in HAPMAP file.").c_str()); + Rcpp::stop(("variants with more than 2 alleles at " + to_string(m+2) + "th row in HAPMAP file.").c_str()); } + Major.push_back(alleles[0]); map << l[0] << '\t' << l[2] << '\t' << l[3] << '\t' << alleles[0] << @@ -281,20 +301,18 @@ List hapmap_parser_map(Rcpp::StringVector hmp_file, std::string out) { } map.close(); file.close(); - } - + // } return List::create(_["n"] = n, _["m"] = m, _["Major"] = Major); } -template -T hapmap_marker_parser(string m, char major, double NA_C) { +double hapmap_marker_parser(string m, char major, double NA_C) { if (m.length() == 1) { // Hapmap // Rcout << "major: " << major << '\t' << "now: " << m[0] << endl; if (m[0] == '+' || m[0] == '0' || m[0] == '-' || m[0] == 'N') { - return static_cast(NA_C); + return NA_C; } else if (m[0] == major) { return 0; } else if (m[0] == 'R' || m[0] == 'Y' || m[0] == 'S' || m[0] == 'W' || m[0] == 'K' || m[0] == 'M') { @@ -305,14 +323,12 @@ T hapmap_marker_parser(string m, char major, double NA_C) { } else if (m.length() == 2) { // Hapmap Diploid if ((m[0] != 'A' && m[0] != 'T' && m[0] != 'G' && m[0] != 'C') || (m[1] != 'A' && m[1] != 'T' && m[1] != 'G' && m[1] != 'C')) { - return static_cast(NA_C); + return NA_C; } else { - return static_cast( - ((m[0] == major)?0:1) + ((m[1] == major)?0:1) - ); + return ((m[0] == major) ? 0 : 1) + ((m[1] == major) ? 0 : 1); } } - return static_cast(NA_C); + return NA_C; } template @@ -324,12 +340,11 @@ void hapmap_parser_genotype(std::string hmp_file, std::vector Major string line; char major; vector l; - vector markers; - size_t m; MatrixAccessor mat = MatrixAccessor(*pMat); // progress bar - Progress progress(pMat->nrow(), verbose); + MinimalProgressBar_perc pb; + Progress progress(pMat->nrow(), verbose, pb); // Skip Header string prefix("rs#"); @@ -343,46 +358,37 @@ void hapmap_parser_genotype(std::string hmp_file, std::vector Major } if (!have_header) { - Rcpp::stop("ERROR: Wrong HAPMAP file, no line begin with \"rs#\"."); + Rcpp::stop("wrong HAPMAP file, no line begin with \"rs#\"."); } + + l = split_line(line); + size_t n_col = l.size(); // parser genotype - m = 0; + size_t m = 0; vector buffer; - int idx1 = 0; - int idx2 = 0; while (file) { buffer.clear(); - idx2 = idx1; for (int i = 0; (maxLine <= 0 || i < maxLine) && getline(file, line); i++) { // Rcout << i << endl << line << endl; if (line.length() > 1) { // Handling the blank line at the end of the file. buffer.push_back(line); - idx1++; } } - // Rcout << "buffer.size()\t" << buffer.size() << endl; - #pragma omp parallel for private(l, markers, major) - for (std::size_t i = 0; i < buffer.size(); i++) { - boost::split(l, buffer[i], boost::is_any_of(" \t")); - major = Major[idx2 + i][0]; - vector(l.begin() + 11, l.end()).swap(l); - markers.clear(); - transform( - l.begin(), - l.end(), - back_inserter(markers), - boost::bind(&hapmap_marker_parser, _1, major, NA_C) - ); - // Rcout << m << '\t' << i << endl; - for (std::size_t j = 0; j < markers.size(); j++) { - // Rcout << int(markers[j]) << '\t'; - mat[j][m + i] = markers[j]; + size_t n_marker = buffer.size(); + #pragma omp parallel for private(l, major) + for (size_t i = 0; i < n_marker; i++) { + l = split_line(buffer[i]); + // boost::split(l, buffer[i], boost::is_any_of(" \t")); + if(l.size() != n_col) + Rcpp::stop(("line " + to_string(m+i+2) + " does not have " + to_string(n_col) + " elements in HAPMAP file.").c_str()); + major = Major[m + i][0]; + for(size_t j = 0; j < (l.size() - 11); j++) { + mat[j][m + i] = static_cast(hapmap_marker_parser(l[j + 11], major, NA_C)); } - // Rcout << endl; - progress.increment(); } - m += buffer.size(); + progress.increment(n_marker); + m += n_marker; } } @@ -418,7 +424,8 @@ List numeric_scan(std::string num_file) { ifstream file(num_file); getline(file, line); - boost::split(l, line, boost::is_any_of("\t ,")); + l = split_line(line); + // boost::split(l, line, boost::is_any_of("\t ,")); n = l.size(); m = 1; @@ -456,7 +463,8 @@ void write_bfile(XPtr pMat, std::string bed_file, double NA_C, int th fout = fopen(bed_file.c_str(), "wb"); // progress bar - Progress progress(m, verbose); + MinimalProgressBar_perc pb; + Progress progress(pMat->nrow(), verbose, pb); // magic number of bfile const unsigned char magic_bytes[] = { 0x6c, 0x1b, 0x01 }; @@ -523,7 +531,7 @@ void read_bfile(std::string bed_file, XPtr pMat, long maxLine, double MatrixAccessor mat = MatrixAccessor(*pMat); // map - std::map code; + std::map code; code[3] = 0; code[2] = 1; code[1] = static_cast(NA_C); @@ -542,7 +550,8 @@ void read_bfile(std::string bed_file, XPtr pMat, long maxLine, double // progress bar int n_block = (length - 3) / buffer_size; if ((length - 3) % buffer_size != 0) { n_block++; } - Progress progress(n_block, verbose); + MinimalProgressBar_perc pb; + Progress progress(n_block, verbose, pb); // magic number of bfile buffer = new char [3]; @@ -561,8 +570,8 @@ void read_bfile(std::string bed_file, XPtr pMat, long maxLine, double #pragma omp parallel for schedule(static) for (size_t j = 0; j < cond; j++) { // bit -> item in matrix - size_t r = (block_start + j) / n; - size_t c = (block_start + j) % n * 4; + size_t r = j / n + i * maxLine; + size_t c = j % n * 4; uint8_t p = buffer[j]; for (size_t x = 0; x < 4 && (c + x) < ind; x++) { diff --git a/src/impute.cpp b/src/impute.cpp index 71ba288..d428d9c 100755 --- a/src/impute.cpp +++ b/src/impute.cpp @@ -12,10 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. - -#include "mvp_omp.h" -#include -#include +#include "rMVP.h" using namespace Rcpp; @@ -86,7 +83,8 @@ bool hasNA(XPtr pMat, double NA_C, const int threads=0) { bool HasNA = false; MatrixAccessor mat = MatrixAccessor(*pMat); - #pragma omp parallel for schedule(dynamic) shared(HasNA) + + #pragma omp parallel for shared(HasNA) for (size_t j = 0; j < n; j++) { if(HasNA) continue; for (size_t i = 0; i < m; i++) { diff --git a/src/kinship.cpp b/src/kinship.cpp index ad4ef06..4815cb1 100755 --- a/src/kinship.cpp +++ b/src/kinship.cpp @@ -1,11 +1,9 @@ -#include -#include "mvp_omp.h" -#include -#include -#include +#if !defined(ARMA_64BIT_WORD) +#define ARMA_64BIT_WORD 1 +#endif + +#include "rMVP.h" #include -#include -#include "progress_bar.hpp" // [[Rcpp::depends(RcppArmadillo)]] // [[Rcpp::depends(bigmemory, BH)]] @@ -15,32 +13,6 @@ using namespace std; using namespace Rcpp; using namespace arma; - -class MinimalProgressBar: public ProgressBar{ - public: - MinimalProgressBar() { - _finalized = false; - } - ~MinimalProgressBar() {} - void display() {} - void update(float progress) { - if (_finalized) return; - REprintf("\r"); - REprintf("Calculating in process...(finished %.2f%%)", progress * 100); - } - void end_display() { - if (_finalized) return; - REprintf("\r"); - - REprintf("Calculating in process...(finished 100.00%%)"); - REprintf("\n"); - _finalized = true; - } - private: - bool _finalized; -}; - - template arma::vec BigRowMean(XPtr pMat, int threads = 0){ @@ -236,3 +208,117 @@ SEXP kin_cal_s(SEXP pBigMat, int threads = 0, bool mkl = false, bool verbose = t throw Rcpp::exception("unknown type detected for big.matrix object!"); } } + + +template +SEXP kin_cal(XPtr pMat, const Nullable geno_ind = R_NilValue, int threads = 0, size_t step = 5000, bool mkl = false, bool verbose = true){ + + omp_setup(threads); + + MatrixAccessor bigm = MatrixAccessor(*pMat); + + #ifdef _OPENMP + #else + if(!mkl) + mkl = true; + #endif + if(threads == 1) + mkl = true; + + uvec _geno_ind; + if(geno_ind.isNotNull()){ + _geno_ind = as(geno_ind) - 1; + }else{ + _geno_ind = regspace(0, pMat->ncol() - 1); + } + + int n = _geno_ind.n_elem; + int m = pMat->nrow(); + + arma::vec means(m); + #pragma omp parallel for + for (size_t j = 0; j < m; j++){ + double p1 = 0.0; + for(size_t k = 0; k < n; k++){ + p1 += bigm[_geno_ind[k]][j]; + } + means[j] = p1 / n; + } + + arma::mat kin = zeros(n, n); + arma::mat Z_buffer(step, n, fill::none); + + size_t i = 0, j = 0; + size_t i_marker = 0; + MinimalProgressBar_plus pb; + Progress progress(m, verbose, pb); + for (;i < m;) { + + int cnt = 0; + for (; j < m && cnt < step; j++) + { + cnt++; + } + + if (cnt != step) { + Z_buffer.set_size(cnt, n); + } + + #pragma omp parallel for + for(size_t k = 0; k < n; k++){ + for(size_t l = 0; l < cnt; l++){ + Z_buffer(l, k) = bigm[_geno_ind[k]][i_marker + l] - means[i_marker + l]; + } + } + + if(mkl){ + double alp = 1.0; + double beta = 1.0; + char uplo = 'L'; + dsyrk_(&uplo, "T", &n, &cnt, &alp, Z_buffer.memptr(), &cnt, &beta, kin.memptr(), &n); + }else{ + arma::colvec coli; + #pragma omp parallel for schedule(dynamic) private(coli) + for(size_t k = 0; k < n; k++){ + coli = Z_buffer.col(k); + for(size_t l = k; l < n; l++){ + kin(l, k) += sum(coli % Z_buffer.col(l)); + } + } + } + i = j; + i_marker += cnt; + + if(!Progress::check_abort()) progress.increment(cnt); + } + Z_buffer.reset(); + + #pragma omp parallel for schedule(dynamic) + for (uword j = 0; j < kin.n_cols; j++) { + for (uword i = (j + 1); i < kin.n_cols; i++) { + kin(j, i) = kin(i, j); + } + } + kin /= arma::mean(kin.diag()); + + return Rcpp::wrap(kin); +} + +// [[Rcpp::export]] +SEXP kin_cal(SEXP pBigMat, const Nullable geno_ind = R_NilValue, int threads = 0, size_t step = 10000, bool mkl = false, bool verbose = true){ + + XPtr xpMat(pBigMat); + + switch(xpMat->matrix_type()){ + case 1: + return kin_cal(xpMat, geno_ind, threads, step, mkl, verbose); + case 2: + return kin_cal(xpMat, geno_ind, threads, step, mkl, verbose); + case 4: + return kin_cal(xpMat, geno_ind, threads, step, mkl, verbose); + case 8: + return kin_cal(xpMat, geno_ind, threads, step, mkl, verbose); + default: + throw Rcpp::exception("unknown type detected for big.matrix object!"); + } +} diff --git a/src/rMVP.h b/src/rMVP.h new file mode 100644 index 0000000..53ac0cd --- /dev/null +++ b/src/rMVP.h @@ -0,0 +1,322 @@ +#ifndef RMVP_ +#define RMVP_ +#endif + +#if !defined(ARMA_64BIT_WORD) +#define ARMA_64BIT_WORD 1 +#endif + +#define ARMA_DONT_USE_FORTRAN_HIDDEN_ARGS 1 + +#include +#include +#include +#include +#include +#include +#include "progress_bar.hpp" +#include "mvp_omp.h" + +class MinimalProgressBar_plus: public ProgressBar{ + public: + MinimalProgressBar_plus() { + _finalized = false; + } + + ~MinimalProgressBar_plus() {} + + std::string _time_to_string(double seconds, float progress) { + + int time = (int) seconds; + + int hour = 0; + int min = 0; + int sec = 0; + + hour = time / 3600; + time = time % 3600; + min = time / 60; + time = time % 60; + sec = time; + + std::stringstream time_strs; + time_strs << (progress < 1.0 ? "TimeLeft: " : "RunTime: "); + if (hour != 0) time_strs << hour << "h"; + if (hour != 0 || min != 0) time_strs << min << "m"; + time_strs << sec << "s"; + std::string time_str = time_strs.str(); + + return time_str; + } + + int _compute_nb_ticks(float progress) { + return int(progress * _max_ticks); + } + + std::string _construct_ticks_display_string(int nb) { + std::stringstream ticks_strs; + for (int i = 1; i <= _max_ticks; ++i) { + if(i < 4){ + ticks_strs << ">"; + } else if (i < nb) { + ticks_strs << "-"; + } else if(i == nb) { + ticks_strs << ">"; + } else { + ticks_strs << " "; + } + } + std::string tick_space_string = ticks_strs.str(); + return tick_space_string; + } + + void end_display() { + update(1); + } + + void _finalize_display() { + if (_finalized) return; + REprintf("\n"); + _finalized = true; + } + + void display() { + flush_console(); + } + + void update(float progress) { + + // stop if already finalized + if (_finalized) return; + + // start time measurement when update() is called the first time + if (_timer_flag) { + _timer_flag = false; + // measure start time + time(&start); + } else { + + int nb_ticks = _compute_nb_ticks(progress); + int delta = nb_ticks - _ticks_displayed; + if (delta > 0) { + _ticks_displayed = nb_ticks; + std::string cur_display = _construct_ticks_display_string(nb_ticks); + + // measure current time + time(&end); + + // calculate passed time and remaining time (in seconds) + double pas_time = std::difftime(end, start); + double rem_time = (progress < 1.0 ? (pas_time / progress) * (1 - progress) : pas_time); + if(rem_time < 1 && rem_time > 0.5) rem_time = 1; + + // convert seconds to time string + std::string time_string = _time_to_string(rem_time, progress); + + // ensure overwriting of old time info + int empty_length = time_string.length(); + + std::string empty_space; + + // merge progress bar and time string + std::stringstream strs; + if(empty_length_p && abs(empty_length - empty_length_p)){ + empty_space = std::string(abs(empty_length - empty_length_p), ' '); + strs << "[" << cur_display << "] " << time_string << empty_space; + }else{ + strs << "[" << cur_display << "] " << time_string; + } + empty_length_p = empty_length; + // strs << "[" << cur_display << "]"; + + std::string temp_str = strs.str(); + char const* char_type = temp_str.c_str(); + + // print: remove old display and replace with new + REprintf("\r"); + REprintf("%s", char_type); + + } + if (_ticks_displayed >= _max_ticks) + // end_display(); + _finalize_display(); + } + } + + void flush_console() { + #if !defined(WIN32) && !defined(__WIN32) && !defined(__WIN32__) + R_FlushConsole(); + #endif + } + + private: + int empty_length_p = 0; + int _max_ticks = 45; + bool _finalized; + bool _timer_flag = true; + time_t start, end; + int _ticks_displayed = 0; +}; + +class MinimalProgressBar_perc: public ProgressBar{ + public: + MinimalProgressBar_perc() { + _finalized = false; + } + + ~MinimalProgressBar_perc() {} + + std::string _time_to_string(double seconds, float progress) { + + int time = (int) seconds; + + int hour = 0; + int min = 0; + int sec = 0; + + hour = time / 3600; + time = time % 3600; + min = time / 60; + time = time % 60; + sec = time; + + std::stringstream time_strs; + time_strs << (progress < 1.0 ? "TimeLeft: " : "RunTime: "); + if (hour != 0) time_strs << hour << "h"; + if (hour != 0 || min != 0) time_strs << min << "m"; + time_strs << sec << "s"; + std::string time_str = time_strs.str(); + + return time_str; + } + + int _compute_nb_ticks(float progress) { + return int(progress * _max_ticks); + } + + std::string _construct_ticks_display_string(int nb) { + std::stringstream ticks_strs; + for (int i = 1; i <= _max_ticks; ++i) { + if (i <= nb) { + ticks_strs << "*"; + } else { + ticks_strs << " "; + } + } + std::string tick_space_string = ticks_strs.str(); + return tick_space_string; + } + + void end_display() { + update(1); + } + + void _finalize_display() { + if (_finalized) return; + REprintf("\n"); + _finalized = true; + } + + void display() { + REprintf("0%% 10 20 30 40 50 60 70 80 90 100%%\n"); + REprintf("[----|----|----|----|----|----|----|----|----|----|\n"); + flush_console(); + } + + void update(float progress) { + + // stop if already finalized + if (_finalized) return; + + // start time measurement when update() is called the first time + if (_timer_flag) { + _timer_flag = false; + // measure start time + time(&start); + } else { + + int nb_ticks = _compute_nb_ticks(progress); + int delta = nb_ticks - _ticks_displayed; + if (delta > 0) { + _ticks_displayed = nb_ticks; + std::string cur_display = _construct_ticks_display_string(nb_ticks); + + // measure current time + time(&end); + + // calculate passed time and remaining time (in seconds) + double pas_time = std::difftime(end, start); + double rem_time = (progress < 1.0 ? (pas_time / progress) * (1 - progress) : pas_time); + if(rem_time < 1 && rem_time > 0.5) rem_time = 1; + + // convert seconds to time string + std::string time_string = _time_to_string(rem_time, progress); + + // ensure overwriting of old time info + int empty_length = time_string.length(); + + std::string empty_space; + + // merge progress bar and time string + std::stringstream strs; + if(empty_length_p && abs(empty_length - empty_length_p)){ + empty_space = std::string(abs(empty_length - empty_length_p), ' '); + strs << "[" << cur_display << "] " << time_string << empty_space; + }else{ + strs << "[" << cur_display << "] " << time_string; + } + empty_length_p = empty_length; + // strs << "[" << cur_display << "]"; + + std::string temp_str = strs.str(); + char const* char_type = temp_str.c_str(); + + // print: remove old display and replace with new + REprintf("\r"); + REprintf("%s", char_type); + + } + if (_ticks_displayed >= _max_ticks) + // end_display(); + _finalize_display(); + } + } + + void flush_console() { + #if !defined(WIN32) && !defined(__WIN32) && !defined(__WIN32__) + R_FlushConsole(); + #endif + } + + private: + int empty_length_p = 0; + int _max_ticks = 49; + bool _finalized; + bool _timer_flag = true; + time_t start, end; + int _ticks_displayed = 0; +}; + +class MinimalProgressBar: public ProgressBar{ + public: + MinimalProgressBar() { + _finalized = false; + } + ~MinimalProgressBar() {} + void display() {} + void update(float progress) { + if (_finalized) return; + REprintf("\r"); + REprintf("Calculating in process...(finished %.2f%%)", progress * 100); + } + void end_display() { + if (_finalized) return; + REprintf("\r"); + + REprintf("Calculating in process...(finished 100.00%%)"); + REprintf("\n"); + _finalized = true; + } + private: + bool _finalized; +};