diff --git a/flow/tool/flow.dnld.dp0p.hdf5.R b/flow/tool/flow.dnld.dp0p.hdf5.R new file mode 100644 index 00000000..50bc649b --- /dev/null +++ b/flow/tool/flow.dnld.dp0p.hdf5.R @@ -0,0 +1,58 @@ +############################################################################################## +#' @title Workflow for downloading dp0p data from S3 + +#' @author +#' David Durden \email{eddy4R.info@gmail.com} + +#' @description +#' Workflow. Downloading dp0p data from S3. + +#' @param Currently none + +#' @return Currently none + +#' @references + +#' @keywords eddy-covariance, NEON + +#' @examples Currently none + +#' @seealso Currently none + +# changelog and author contributions / copyrights +# David (2020-01-25) +# original creation +############################################################################################## + +#site to download data for +site <- "STEI" +#domain +dom <- "D05" +#SAE system (ecte vs. ecse) +sys <- "ecte" + +#Create download folder, create if it doesn't exist +DirDnld <- paste0("~/eddy/data/turbTow/inpRefe/",site) +if(!dir.exists(DirDnld)) dir.create(DirDnld, recursive = TRUE) + +#Create data download string +DateBgn <- as.Date("2019-09-11") +DateEnd <- as.Date("2019-09-21") +DateSeq <- seq.Date(from = DateBgn,to = DateEnd, by = "day") +PrdWndwDnld <- base::as.character(DateSeq) + + + +#Filename base +fileInBase <- paste0("NEON.",dom,".",site,".IP0.00200.001.",sys,".") + +#Create URL for data files +urlDnld <- paste0("https://storage.cloud.google.com/neon-sae-files/ods/dataproducts/IP0/",PrdWndwDnld,"/",site,"/",fileInBase,PrdWndwDnld,".l0p.h5") + +#Download filename (full path) +fileDnld <- paste0(DirDnld,"/",base::toupper(sys),"_dp0p_",site,"_",PrdWndwDnld,".h5") + +#Download files +sapply(seq_along(urlDnld), function(x){ +download.file(url = urlDnld[x], destfile = fileDnld[x]) +}) diff --git a/pack/eddy4R.base/DESCRIPTION b/pack/eddy4R.base/DESCRIPTION index a09c94b2..66328832 100644 --- a/pack/eddy4R.base/DESCRIPTION +++ b/pack/eddy4R.base/DESCRIPTION @@ -1,6 +1,6 @@ Package: eddy4R.base Title: Eddy-covariance calculation for R: Base package -Version: 0.2.20 +Version: 0.2.24 Authors@R: c( person("Stefan", "Metzger", email = "eddy4R.info@gmail.com", role = c("aut", "cre")), person("David", "Durden", email = "ddurden@battelleecology.org", role = c("aut")), person("Natchaya", "Pingintha-Durden", email = "ndurden@battelleecology.org", role = c("aut")), @@ -28,4 +28,4 @@ Suggests: NEONprocIS.base License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 LazyData: true -RoxygenNote: 6.1.1 +RoxygenNote: 7.1.1 diff --git a/pack/eddy4R.base/NAMESPACE b/pack/eddy4R.base/NAMESPACE index e338e271..5865c02f 100644 --- a/pack/eddy4R.base/NAMESPACE +++ b/pack/eddy4R.base/NAMESPACE @@ -30,6 +30,7 @@ export(def.idx.agr) export(def.idx.diff) export(def.inst.depe) export(def.irga.vali.cor) +export(def.irga.vali.thsh) export(def.lag) export(def.mean.med.mode) export(def.med.mad) diff --git a/pack/eddy4R.base/R/def.hdf5.copy.para.R b/pack/eddy4R.base/R/def.hdf5.copy.para.R index 12835a5a..b41eb2a3 100755 --- a/pack/eddy4R.base/R/def.hdf5.copy.para.R +++ b/pack/eddy4R.base/R/def.hdf5.copy.para.R @@ -39,6 +39,10 @@ # applied term name convention; replaced FileIn by FileInp # Natchaya P-Durden (2018-05-22) # rename function from def.para.hdf5.dp01() to def.hdf5.copy.para() +# Dave Durden (2021-08-17) +# Failsafe to remove rhdf5 attribute from list of attributes written out +# Dave Durden (2021-10-12) +# Copy global attributes by adding file level to listGrp ############################################################################################################## #Start of function call to read metadata from one file and write to another ############################################################################################################## @@ -63,6 +67,8 @@ listPara <- rhdf5::h5ls(FileInp, datasetinfo = FALSE) #listPara <- listPara[listPara$otype == "H5I_GROUP",] #Used to grab metadata if it is only attached to the group level listGrp <- base::paste(listPara$group, listPara$name, sep = "/") # Combining output +#Append global attribute level +listGrp <- append("/", listGrp) # read attributes from input file listAttr <- base::lapply(listGrp, rhdf5::h5readAttributes, file = FileInp) @@ -74,6 +80,13 @@ base::names(listAttr) <- listGrp #Remove all empty lists listAttr <- listAttr[!base::sapply(listAttr, function(x) base::length(x) == 0)] +#Failsafe to remove rhdf5 attribute +lapply(names(listAttr), function(x){ + if(length(names(listAttr[[x]])) == 1 && grepl(pattern = "rhdf5", x = names(listAttr[[x]]))){ + #Remove attribute if rhdf5 attribute is the only one written + listAttr[[x]] <<- NULL + }#end if logical for single rhdf5 attribute +})#end failsafe for rhdf5 attribute #Open the output file HDF5 link idFile <- rhdf5::H5Fopen(FileOut) diff --git a/pack/eddy4R.base/R/def.hdf5.crte.R b/pack/eddy4R.base/R/def.hdf5.crte.R index 90cc1967..5dfa3d42 100755 --- a/pack/eddy4R.base/R/def.hdf5.crte.R +++ b/pack/eddy4R.base/R/def.hdf5.crte.R @@ -73,6 +73,8 @@ # Adding irga validation system sensors for ECTE (pressure sensors) # Dave Durden (2019-07-14) # Adding irga validation system sensors for ECTE (valve and pump) +# Dave Durden (2020-02-09) +# Grabbing objDesc and Readme from S3, removing dropbox link ############################################################################################################## #Start of function call to generate NEON HDF5 files ############################################################################################################## @@ -102,8 +104,7 @@ def.hdf5.crte <- function( DirTmp <- tempdir() #Download file description readme and object list - eddy4R.base::def.dld.zip(Inp = list(Url = "https://www.dropbox.com/s/dqq3j7epiy98y29/fileDesc.zip?dl=1", - Dir = DirTmp)) + eddy4R.base::def.dld.zip(Inp = list(Url = "https://storage.googleapis.com/neon-ec-goldfiles/EC-turbulence-processing/fileDesc.zip", Dir = DirTmp)) #Store the path to the readme file FileNameReadMe <- base::list.files( path = base::paste0(DirTmp,"/fileDesc"), pattern = ".txt", full.names = TRUE) diff --git a/pack/eddy4R.base/R/def.hdf5.extr.R b/pack/eddy4R.base/R/def.hdf5.extr.R index e4e144ca..5a1c398a 100755 --- a/pack/eddy4R.base/R/def.hdf5.extr.R +++ b/pack/eddy4R.base/R/def.hdf5.extr.R @@ -52,6 +52,10 @@ # applied term name convention; replaced FileIn by FileInp # Natchaya P-Durden (2018-05-11) # rename function from def.extr.hdf5() to def.hdf5.extr() +# Dave Durden (2018-03-12) +# Adding failsafe for rhdf5 metadata attribute on individual dp0p arrays +# Dave Durden (2021-10-12) +# Copy global attributes by adding file level to listGrp ############################################################################################################## #Start of function call to extract data from one file and write to another ############################################################################################################## @@ -88,6 +92,9 @@ if(base::is.null(rpt)) { #List of all object names listObjName <- base::paste(listObj$group, listObj$name, sep = "/") + + #Append global attribute level + listObjName <- append("/", listObjName) # Groups for HDF5 group structure @@ -191,10 +198,20 @@ if(!is.null(FileOut)) { # determine if attributes should be written to output HDF5 if(MethExtrAttr == TRUE){ + #Failsafe to remove rhdf5 attribute + lapply(names(rpt$listAttr), function(x){ + if(length(names(rpt$listAttr[[x]])) == 1 && grepl(pattern = "rhdf5", x = names(rpt$listAttr[[x]]))){ + #Remove attribute if rhdf5 attribute is the only one written + rpt$listAttr[[x]] <<- NULL + }#end if logical for single rhdf5 attribute + })#end failsafe for rhdf5 attribute + #Write attributes to the output HDF5 file lapply(names(rpt$listAttr), function(x){ + #print(x) idData <- rhdf5::H5Oopen(idFile, x) base::lapply(names(rpt$listAttr[[x]]), function(y){ + #print(y) #y <- names(rpt$listAttr[[x]])[1] rhdf5::h5writeAttribute(attr = rpt$listAttr[[x]][[y]], h5obj = idData, name = y)}) }) diff --git a/pack/eddy4R.base/R/def.hdf5.pack.R b/pack/eddy4R.base/R/def.hdf5.pack.R index 47e31d57..9b5e40af 100644 --- a/pack/eddy4R.base/R/def.hdf5.pack.R +++ b/pack/eddy4R.base/R/def.hdf5.pack.R @@ -112,6 +112,7 @@ if(MethMeas %in% "ecse"){ if (Dp %in% c("Dp01", "Dp02")) { for(idxDp in names(inpList)) { + #idxDp <- "co2Stor" #for testing tmp00[[idxDp]] <- list() tmp01[[idxDp]] <- list() @@ -121,14 +122,14 @@ if(MethMeas %in% "ecse"){ #idxLvLReso <- names(inpList[[idxDp]])[1] #Check if qm is part of the input list if(exists('qm', where = inpList[[idxDp]][[idxLvLReso]][[1]]) == TRUE){ - + # Add the qm's to tmp list tmp00[[idxDp]][[idxLvLReso]]$qm <- lapply(names(inpList[[idxDp]][[idxLvLReso]][[1]]$qm), function(idxStat) # second call to lapply, targeting the observations to be combined into the result data.frames do.call(rbind, lapply(1:length(inpList[[idxDp]][[idxLvLReso]]), function(idxt) { inpList[[idxDp]][[idxLvLReso]][[idxt]]$qm[[idxStat]] - } ) - )) + } ) + )) # assign names to data.frames names(tmp00[[idxDp]][[idxLvLReso]]$qm) <- names(inpList[[idxDp]][[idxLvLReso]][[1]]$qm) } @@ -138,7 +139,7 @@ if(MethMeas %in% "ecse"){ tmp01[[idxDp]][[idxLvLReso]] <- lapply(names(inpList[[idxDp]][[idxLvLReso]][[1]]), function(idxStat) # second call to lapply, targeting the observations to be combined into the result data.frames - do.call(rbind, lapply(1:length(inpList[[idxDp]][[idxLvLReso]]), function(idxt) inpList[[idxDp]][[idxLvLReso]][[idxt]][[idxStat]] )) + base::as.data.frame(dplyr::bind_rows(lapply(1:length(inpList[[idxDp]][[idxLvLReso]]), function(idxt) inpList[[idxDp]][[idxLvLReso]][[idxt]][[idxStat]] ))) ) # assign names to data.frames diff --git a/pack/eddy4R.base/R/def.hdf5.read.qfqm.R b/pack/eddy4R.base/R/def.hdf5.read.qfqm.R index 93d3ec88..4a0057c6 100644 --- a/pack/eddy4R.base/R/def.hdf5.read.qfqm.R +++ b/pack/eddy4R.base/R/def.hdf5.read.qfqm.R @@ -12,10 +12,11 @@ #' @param VarLoca Character: Which instrument to read data from. #' @param LvlTowr The tower level that the sensor data is being collected in NEON data product convention (HOR_VER) #' @param FreqLoca Integer: Measurement frequency. +#' @param DataType Character: Specify between data and qfqm for read in. #' @param MethMeas A vector of class "character" containing the name of measurement method (eddy-covariance turbulent exchange or storage exchange), MethMeas = c("ecte", "ecse"). Defaults to "ecte". #' @return -#' Named list \code{qfqm} containing time-series of quality flags. +#' Named list \code{rpt} containing time-series of quality flags. #' @references #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007. @@ -42,6 +43,8 @@ # applied term name convention; replaced Levl by Lvl # Natchaya P-Durden (2018-05-22) # rename function from def.neon.read.hdf5.qfqm() to def.hdf5.read.qfqm() +# David Durden (2020-06-22) +# extending function to work with data and qfqm, as well as adding metadata ############################################################################################## def.hdf5.read.qfqm <- function( @@ -51,32 +54,77 @@ def.hdf5.read.qfqm <- function( VarLoca, LvlTowr = c("000_040", "000_050", "000_060")[3], FreqLoca, + DataType = c("data","qfqm")[1], MethMeas = c("ecte", "ecse")[1] ){ #Read in the flags from the HDF5 file if (MethMeas == "ecte") { -qfqm <- rhdf5::h5read(file = base::paste0(DirInpLoca, "/ECTE_dp0p_", SiteLoca, "_", DateLoca, ".h5"), - name = base::paste0("/", SiteLoca, "/dp0p/qfqm/", VarLoca, "/",LvlTowr), read.attributes = TRUE) +rpt <- rhdf5::h5read(file = base::paste0(DirInpLoca, "/ECTE_dp0p_", SiteLoca, "_", DateLoca, ".h5"), + name = base::paste0("/", SiteLoca, "/dp0p/",DataType,"/", VarLoca, "/",LvlTowr), read.attributes = TRUE) } if (MethMeas == "ecse") { -qfqm <- rhdf5::h5read(file = base::paste0(DirInpLoca, "/ECSE_dp0p_", SiteLoca, "_", DateLoca, ".h5"), - name = base::paste0("/", SiteLoca, "/dp0p/qfqm/", VarLoca, "/",LvlTowr), read.attributes = TRUE) +rpt <- rhdf5::h5read(file = base::paste0(DirInpLoca, "/ECSE_dp0p_", SiteLoca, "_", DateLoca, ".h5"), + name = base::paste0("/", SiteLoca, "/dp0p/",DataType,"/", VarLoca, "/",LvlTowr), read.attributes = TRUE) } +# print message to screen +msg <- paste0("dataset ", DateLoca, ": ", VarLoca, " hdf5 read-in complete") +tryCatch({rlog$debug(msg)}, error=function(cond){print(msg)}) + #Convert each flag to a vector from a 1D array -for(idx in base::names(qfqm)) qfqm[[idx]] <- base::as.vector(qfqm[[idx]]); base::rm(idx) +for(idx in base::names(rpt)) rpt[[idx]] <- base::as.vector(rpt[[idx]]); base::rm(idx) + +#Cache attributes before coverting to data.frame +attr <- attributes(rpt) +#Convert each flag to a vector from a 1D array +for(idx in base::names(attr)) attr[[idx]] <- base::as.vector(attr[[idx]]); base::rm(idx) + +# convert list to data.frame +rpt <- base::as.data.frame(rpt, stringsAsFactors = FALSE) + +#Reapply attributes to reported data.frame +attributes(rpt)$unit <- attr$Unit +if(is.null(attr(rpt,"unit")) & DataType == "qfqm") attributes(rpt)$unit <- rep(NA, length(rpt)) + +# convert type of variable time +if("time" %in% colnames(rpt)){ +rpt$time <- base::as.POSIXct(rpt$time, format="%Y-%m-%dT%H:%M:%OSZ", tz="UTC") + 0.0001 +} + +# perform unit conversion +if(DataType == "data"){ +rpt <- base::suppressWarnings(eddy4R.base::def.unit.conv(data = rpt, + unitFrom = attributes(rpt)$unit, + unitTo = "intl")) +} + +#Reapply attributes to reported data.frame +lapply(grep("Unit", names(attr), value = TRUE, invert = TRUE), function(x){ + attributes(rpt)[x] <<- attr[x] + }) + + +# sd assign attribute to gasRefe +if (VarLoca == "gasRefe"){ + names(attr(rpt,"Sd")) <- attr(rpt,"Name") + names(attributes(rpt))[which(names(attributes(rpt))=="Sd")] <- "sd" #Change to lower case to keep format + names(attr(rpt,"DfSd")) <- attr(rpt,"Name") + #base::attributes(rpt)$sd <- attr$Sd[base::names(rpt)] + #base::attributes(rpt)$DfSd <- attr$DfSd[base::names(rpt)] +} #Apply units to each flag -lapply(seq_len(length(qfqm)), function(x){ +lapply(seq_len(length(rpt)), function(x){ tryCatch({rlog$debug(x)}, error=function(cond){print(x)}) - attributes(qfqm[[x]])$Unit <<- attributes(qfqm)$Unit[[x]] + attributes(rpt[[x]])$unit <<- attributes(rpt)$unit[[x]] + attributes(rpt[[x]])$`Dspk$Br86$MaxReso` <<- attributes(rpt)$`Dspk$Br86$MaxReso`[[x]] + attributes(rpt[[x]])$`Dspk$Br86$NumBin` <<- attributes(rpt)$`Dspk$Br86$NumBin`[[x]] + attributes(rpt[[x]])$`Dspk$Br86$NumWndw` <<- attributes(rpt)$`Dspk$Br86$NumWndw`[[x]] }) -# convert list to data.frame -qfqm <- base::as.data.frame(qfqm, stringsAsFactors = FALSE) -return(qfqm) +return(rpt) } diff --git a/pack/eddy4R.base/R/def.hdf5.wrte.dp01.R b/pack/eddy4R.base/R/def.hdf5.wrte.dp01.R index f33bee13..f260b556 100755 --- a/pack/eddy4R.base/R/def.hdf5.wrte.dp01.R +++ b/pack/eddy4R.base/R/def.hdf5.wrte.dp01.R @@ -48,6 +48,12 @@ # adding linear regression coefficients and its se to the attribute of rtioMoleDryCo2Vali # Natchaya P-Durden (2019-02-21) # adding results from MLF (scale) to the attribute of rtioMoleDryCo2Vali +# Natchaya P-Durden (2019-01-08) +# removed rtioMoleDryH2oCor and rtioMoleDryH2oRaw data and ucrt from the basic file +# Natchaya P-Durden (2020-02-23) +# removed unnecessary qfqm unit attributes when writing the basic file +# Chris Florian (2021-08-09) +# adding qfValiThsh to rtioMoleDryCo2Vali attributes ############################################################################################## @@ -70,6 +76,8 @@ MethExpd <- grepl(pattern = "expanded", x = FileOut) if(MethExpd == FALSE){ inpList$data$co2Turb$rtioMoleDryCo2Cor <- NULL inpList$data$co2Turb$rtioMoleDryCo2Raw <- NULL + inpList$data$h2oTurb$rtioMoleDryH2oCor <- NULL + inpList$data$h2oTurb$rtioMoleDryH2oRaw <- NULL } #Create HDF5 connection to the output file @@ -113,6 +121,8 @@ if(MethSubAgr == TRUE){ if(MethExpd == FALSE){ inpList$dp01AgrSub$data$co2Turb$rtioMoleDryCo2Cor <- NULL inpList$dp01AgrSub$data$co2Turb$rtioMoleDryCo2Raw <- NULL + inpList$dp01AgrSub$data$h2oTurb$rtioMoleDryH2oCor <- NULL + inpList$dp01AgrSub$data$h2oTurb$rtioMoleDryH2oRaw <- NULL } #Writing sub-aggregated (e.g.1-min) data to output HDF5 file lapply(names(inpList$dp01AgrSub$data[[Dp01]]), function(x) rhdf5::h5writeDataset.data.frame(obj = inpList$dp01AgrSub$data[[Dp01]][[x]], h5loc = idData01, name = x, DataFrameAsCompound = TRUE)) @@ -127,9 +137,14 @@ if(MethSubAgr == TRUE){ if (Dp01 == "co2Turb"){ if (!is.null(attributes(inpList$dp01AgrSub$data$co2Turb$rtioMoleDryCo2Vali)$coef) == TRUE){ dgid <- rhdf5::H5Dopen(idData01, "rtioMoleDryCo2Vali") - rhdf5::h5writeAttribute(attributes(inpList$dp01AgrSub$data$co2Turb$rtioMoleDryCo2Vali)$coef, h5obj = dgid, name = "coef") - rhdf5::h5writeAttribute(attributes(inpList$dp01AgrSub$data$co2Turb$rtioMoleDryCo2Vali)$coefSe, h5obj = dgid, name = "coefSe") - rhdf5::h5writeAttribute(attributes(inpList$dp01AgrSub$data$co2Turb$rtioMoleDryCo2Vali)$scal, h5obj = dgid, name = "scal") + rhdf5::h5writeAttribute(attributes(inpList$dp01AgrSub$data$co2Turb$rtioMoleDryCo2Vali)$coef, h5obj = dgid, name = "valiCoef") + rhdf5::h5writeAttribute(attributes(inpList$dp01AgrSub$data$co2Turb$rtioMoleDryCo2Vali)$coefSe, h5obj = dgid, name = "valiCoefSe") + rhdf5::h5writeAttribute(attributes(inpList$dp01AgrSub$data$co2Turb$rtioMoleDryCo2Vali)$scal, h5obj = dgid, name = "valiScal") + rhdf5::h5writeAttribute(attributes(inpList$dp01AgrSub$data$co2Turb$rtioMoleDryCo2Vali)$qfEvalThsh, h5obj = dgid, name = "qfEvalThsh") + rhdf5::h5writeAttribute(attributes(inpList$dp01AgrSub$data$co2Turb$rtioMoleDryCo2Vali)$evalCoef, h5obj = dgid, name = "evalCoef") + rhdf5::h5writeAttribute(attributes(inpList$dp01AgrSub$data$co2Turb$rtioMoleDryCo2Vali)$evalCoefSe, h5obj = dgid, name = "evalCoefSe") + rhdf5::h5writeAttribute(attributes(inpList$dp01AgrSub$data$co2Turb$rtioMoleDryCo2Vali)$evalSlpThsh, h5obj = dgid, name = "evalSlpThsh") + rhdf5::h5writeAttribute(attributes(inpList$dp01AgrSub$data$co2Turb$rtioMoleDryCo2Vali)$evalOfstThsh, h5obj = dgid, name = "evalOfstThsh") }} } ########################################################################################## @@ -184,7 +199,12 @@ if(MethSubAgr == TRUE){ lapply(names(inpList$qfqm[[Dp01]]), function(x) { if (!is.null(attributes(inpList$qfqm[[Dp01]][[x]])$unit) == TRUE){ dgid <- rhdf5::H5Dopen(idQfqm30, x) + if(MethExpd == FALSE){ + tmpAttr <- attributes(inpList$qfqm[[Dp01]][[x]])$unit[which(names(attributes(inpList$qfqm[[Dp01]][[x]])$unit) %in% c("qfFinl","timeBgn","timeEnd"))] + rhdf5::h5writeAttribute(tmpAttr, h5obj = dgid, name = "unit") + }else{ rhdf5::h5writeAttribute(attributes(inpList$qfqm[[Dp01]][[x]])$unit, h5obj = dgid, name = "unit") + } }}) if(MethSubAgr == TRUE){ @@ -192,7 +212,12 @@ if(MethSubAgr == TRUE){ lapply(names(inpList$dp01AgrSub$qfqm[[Dp01]]), function(x) { if (!is.null(attributes(inpList$dp01AgrSub$qfqm[[Dp01]][[x]])$unit) == TRUE){ dgid <- rhdf5::H5Dopen(idQfqm01, x) + if(MethExpd == FALSE){ + tmpAttr <- attributes(inpList$dp01AgrSub$qfqm[[Dp01]][[x]])$unit[which(names(attributes(inpList$dp01AgrSub$qfqm[[Dp01]][[x]])$unit) %in% c("qfFinl","timeBgn","timeEnd"))] + rhdf5::h5writeAttribute(tmpAttr, h5obj = dgid, name = "unit") + }else{ rhdf5::h5writeAttribute(attributes(inpList$dp01AgrSub$qfqm[[Dp01]][[x]])$unit, h5obj = dgid, name = "unit") + } }}) } ########################################################################################## @@ -203,6 +228,8 @@ if(MethUcrt == TRUE){ if(MethExpd == FALSE){ inpList$ucrt$co2Turb$rtioMoleDryCo2Cor <- NULL inpList$ucrt$co2Turb$rtioMoleDryCo2Raw <- NULL + inpList$ucrt$h2oTurb$rtioMoleDryH2oCor <- NULL + inpList$ucrt$h2oTurb$rtioMoleDryH2oRaw <- NULL } #Writing 30-min ucrt to output HDF5 file lapply(names(inpList$ucrt[[Dp01]]), function(x) rhdf5::h5writeDataset.data.frame(obj = inpList$ucrt[[Dp01]][[x]], h5loc = idUcrt30, name = x, DataFrameAsCompound = TRUE)) @@ -219,6 +246,8 @@ if(MethSubAgr == TRUE){ if(MethExpd == FALSE){ inpList$dp01AgrSub$ucrt$co2Turb$rtioMoleDryCo2Cor <- NULL inpList$dp01AgrSub$ucrt$co2Turb$rtioMoleDryCo2Raw <- NULL + inpList$dp01AgrSub$ucrt$h2oTurb$rtioMoleDryH2oCor <- NULL + inpList$dp01AgrSub$ucrt$h2oTurb$rtioMoleDryH2oRaw <- NULL } #Writing sub-aggregated (e.g.1-min) ucrt to output HDF5 file lapply(names(inpList$dp01AgrSub$ucrt[[Dp01]]), function(x) rhdf5::h5writeDataset.data.frame(obj = inpList$dp01AgrSub$ucrt[[Dp01]][[x]], h5loc = idUcrt01, name = x, DataFrameAsCompound = TRUE)) diff --git a/pack/eddy4R.base/R/def.hdf5.wrte.dp01.api.R b/pack/eddy4R.base/R/def.hdf5.wrte.dp01.api.R index aeb55a15..2ba18901 100755 --- a/pack/eddy4R.base/R/def.hdf5.wrte.dp01.api.R +++ b/pack/eddy4R.base/R/def.hdf5.wrte.dp01.api.R @@ -47,6 +47,10 @@ # Natchaya P-Durden (2019-09-12) # get information of existing dp01 hor and ver from dp0p hdf5 file # convert qmBeta and qmAlph to fraction +# David Durden(2020-07-02) +# updating function to check physical locations of reingest sensors exist before pulling from API +# Chris Florian (2022-05-09) +# updating to use SOM API function due to Noble package bug that replicated data from the highest horver in a missing horver ############################################################################################## def.hdf5.wrte.dp01.api <- function( @@ -72,7 +76,7 @@ date <- lubridate::as_datetime(date) #get only year and month yearMnth <- as.character.Date(date, format = "%Y-%m") -timeBgn <- date - lubridate::seconds(1) +timeBgn <- date timeEnd <- date + lubridate::days(1) @@ -115,10 +119,74 @@ if(DpName == "presBaro") TblName <- c("presCor", "presAtm") #Grab 30 minute data to be written msg <- paste0("downloading ", TimeAgr, " min data from the portal") tryCatch({rlog$debug(msg)}, error=function(cond){print(msg)}) -data <- try(expr = Noble::pull.date(site = SiteLoca, dpID = DpNum, bgn.date = timeBgn, end.date = timeEnd, package = "expanded", time.agr = TimeAgr), silent = TRUE) #Currently requires to subtract 1 minute otherwise (1 index will be cut from the beginning) +#old Noble data pull +#data <- try(expr = Noble::pull.date(site = SiteLoca, dpID = DpNum, bgn.date = timeBgn, end.date = timeEnd, package = "expanded", time.agr = TimeAgr), silent = TRUE) #Currently requires to subtract 1 minute otherwise (1 index will be cut from the beginning) + +data <- list() + +for(idxLvl in LvlTowr[[DpName]]){ + #idxLvl <- LvlTowr[[DpName]][2] + #extract hor and ver for each data product as a character string of three values e.g. "000" and "010" + locHor <- substr(idxLvl, start = 1, stop = 3) + locVer <- substr(idxLvl, start = 5, stop = 7) + #pad TimeAgr with zero/s to fit the expected three digit format for wndwAgr + if(nchar(TimeAgr) == 1){ + wndwAgr <- paste0("00", TimeAgr) + } else if(nchar(TimeAgr) == 2){ + wndwAgr <- paste0("0", TimeAgr) + } + + data[[idxLvl]] <- try(expr = som::def.neon.api.get.data(site = SiteLoca, idDpMain = DpNum, locHor = locHor, locVer = locVer, wndwAgr = wndwAgr, year = lubridate::year(timeBgn), mnth = lubridate::month(timeBgn), Pack = "expanded"), silent = TRUE) #need to generalize TimeAgr to wndwAgr, Time Agr is 1,30 and wndwAgr needs "001" and "030" + +} + +#compile noble-like data format if not all the datasets have a try-error class +if(!all(sapply(data, class) == "try-error")){ + #fill in the try error datasets and paste horver to names + for(idxLvl in 1:length(data)){ + if(class(data[[idxLvl]]) == "try-error"){ + #find the first dataset that wasn't a try error + whrData <- which(lapply(data,class) == "data.frame")[1] + #get timestamps + timeBgnOut <- data[[whrData]][which(grepl(x = names(data[[whrData]]), pattern = "startdatetime", ignore.case = T))] + timeEndOut <- data[[whrData]][which(grepl(x = names(data[[whrData]]), pattern = "enddatetime", ignore.case = T))] + colNamesOut <- colnames(data[[whrData]]) + #create dataframe of NaNs to fill with + dataFill <- data.frame(matrix(NaN, nrow=nrow(timeBgnOut), ncol=length(colnames(data[[whrData]])[which(!grepl(x = names(data[[whrData]]), pattern = "time", ignore.case = T))]))) + #create placeholder data.frame + data[[idxLvl]] <- data.frame(timeBgnOut, timeEndOut, dataFill) + #apply names + colnames(data[[idxLvl]]) <- colNamesOut + #remove data fill + rm(dataFill) + #fill QFQM + data[[idxLvl]]$alphaQM <- 0.0 + data[[idxLvl]]$betaQM <- 1.0 + data[[idxLvl]]$finalQF <- 1L + data[[idxLvl]]$finalQFSciRvw <- 0L + } + #idxLvl <- 1 + LocMeas <- gsub("\\_", ".", names(data[idxLvl])) + #append horver to names of each ML's dataset + colnames(data[[idxLvl]])[which(!grepl(x = names(data[[idxLvl]]), pattern = "time", ignore.case = T))] <- + paste0(colnames(data[[idxLvl]][which(!grepl(x = names(data[[idxLvl]]), pattern = "time", ignore.case = T))]), ".", LocMeas) + } + + #join list of data into wide format + + data <- data.frame(base::Reduce(function(x, y) merge(x, y, all=TRUE), data)) + + #restrict API data to the processing date, TODO: is there any way to grab data from the API for just one day? + + data <- data[grep(pattern = date, x = data$startDateTime),] + +} + + + #Failsafe test if API pull produced an error -if(class(data) == "try-error"){ +if(all(sapply(data, class) == "try-error")){ #Initialize lists rpt <- list(data = list(), qfqm = list(), ucrt = list()) #get sensor HOR and VER @@ -131,10 +199,10 @@ if(class(data) == "try-error"){ names(LvlMeasOut) <- LvlMeas #Create the timeBgn vector for aggregation period specified (1, 30 minutes) - timeBgnOut <- seq(from = lubridate::ymd_hms(timeBgn) + lubridate::seconds(1), to = base::as.POSIXlt(timeEnd) - lubridate::minutes(TimeAgr), by = paste(TimeAgr, "mins", sep = " ")) + timeBgnOut <- seq(from = timeBgn + lubridate::seconds(1), to = base::as.POSIXlt(timeEnd) - lubridate::minutes(TimeAgr), by = paste(TimeAgr, "mins", sep = " ")) #Create the timeEnd vector for aggregation period specified (1, 30 minutes) - timeEndOut <- seq(from = lubridate::ymd_hms(timeBgn) + lubridate::minutes(TimeAgr)+ lubridate::seconds(1), to = base::as.POSIXlt(timeEnd), by = paste(TimeAgr, "mins", sep = " ")) + timeEndOut <- seq(from = timeBgn + lubridate::minutes(TimeAgr)+ lubridate::seconds(1), to = base::as.POSIXlt(timeEnd), by = paste(TimeAgr, "mins", sep = " ")) #Creating a vector of NaN's to fill data.frames dataNa <- rep(x = NaN, length = length(timeBgnOut)) @@ -188,7 +256,7 @@ if(class(data) == "try-error"){ } else { #Convert times to POSIXct -data$startDateTime <- as.POSIXct(data$startDateTime) +data$startDateTime <- as.POSIXct(data$startDateTime, format = "%Y-%m-%dT%H:%M:%OSZ", tz = "UTC") data$endDateTime <- as.POSIXct(data$endDateTime, format = "%Y-%m-%dT%H:%M:%OSZ", tz = "UTC") @@ -265,10 +333,17 @@ LvlMeasOut <- LocMeas #Name for HDF5 output names(LvlMeasOut) <- LvlMeas +#Check measurement levels returned from API +LvlExis <- unique(unlist(regmatches(names(data),gregexpr(pattern = "[0-9][0-9][0-9].[)0-9][0-9][0-9]", names(data))))) + +#Check against the measurement location +LvlMeasOut <- LvlMeasOut[LvlMeasOut %in% LvlExis] + ##################################################################################### #Sort output data and apply eddy4R naming conventions tmp$data <- lapply(LvlMeasOut, function(x){ + #print(x) #Grab just the columns to be output tmp <- data[,grep(pattern = paste(nameVar$DataOut, collapse = "|"), x = names(data))] if(DpName %in% "presBaro"){ diff --git a/pack/eddy4R.base/R/def.irga.vali.cor.R b/pack/eddy4R.base/R/def.irga.vali.cor.R index 28d3de79..47b5d6ca 100644 --- a/pack/eddy4R.base/R/def.irga.vali.cor.R +++ b/pack/eddy4R.base/R/def.irga.vali.cor.R @@ -10,9 +10,10 @@ #' @param DateProc A vector of class "character" containing the processing date. #' @param valiData List consisting of descriptive statistics (mean, min, max, vari, numSamp, se) of CO2 dry mole concentration during performing validation and CO2 dry mole concentration of reference gases. #' @param coef List consists of linear regression coefficients (slope and offset) for DateProc - 1, DateProc, and DateProc + 1. -#' @param valiCrit A logical stating if there are more than one validation occurred within DateProc. Defaut to FALSE. -#' @param ScalMax Maximum scale value (resulted from maximum-likelihood fitting of a functional relationship (MLFR)). Defaults to 20. -#' @param FracSlpMax Maximum fraction of slope value (resulted from maximum-likelihood fitting of a functional relationship (MLFR)). Defaults to 0.1. +#' @param valiCrit A logical stating if there are more than one validation occurred within DateProc. +#' @param ScalMax Maximum scale value. The validation correction will not apply if scale (resulted from maximum-likelihood fitting of a functional relationship (MLFR)) is greater than ScalMax or ScalMax = FALSE. Defaults to FALSE. +#' @param FracSlp Upper and lower bounds of slope values. The validation correction will not apply if slope (resulted from regression fitting) is greater/lower than the FracSlp maximum or minimum value or FracSlp = FALSE. Defaults to FALSE. +#' @param OfstMax Maximum offset value. The validation correction will not apply if slope (resulted from regression fitting) is greater than the OfstMax (unit in mol mol-1) or OfstMax = FALSE. Defaults to FALSE. #' @param Freq Measurement frequency. Defaults to 20. [Hz] #' @return @@ -43,6 +44,23 @@ # apply ff object to dataframe to save the memory # Natchaya P-Durden (2019-03-15) # added ScalMax, FracSlpMax, and Freq into input function parameters +# Natchaya P-Durden (2020-01-14) +# added time when the real validation begin +# Natchaya P-Durden (2020-01-16) +# generated NA for rtioMoleDryH2oCor +# Natchaya P-Durden (2020-01-14) +# added 5 min after the validation end +# Natchaya P-Durden (2020-02-28) +# added logical statement to not apply filters (slope, offset, and scale) if they are equal to FALSE +# Natchaya P-Durden (2020-04-14) +# update the way to determine time begin and end to be able to +# work when the validation do not have a full set of gas tanks +# Natchaya P-Durden (2020-04-15) +# adding logical to handle the period that falling into the last day and first day of year +# David Durden (2020-05-26) +# Failsafe for when the valve is switched, but no validation occurs +# Chris Florian (2022-03-02) +# Updating the slope filter to allow for values not evenly centered around 1 ############################################################################################## def.irga.vali.cor <- function( data, @@ -50,8 +68,9 @@ def.irga.vali.cor <- function( coef, valiData, valiCrit = FALSE, - ScalMax = 20, - FracSlpMax = 0.1, + ScalMax = FALSE, + FracSlp = FALSE, + OfstMax = FALSE, Freq = 20 ){ #adding library @@ -65,27 +84,64 @@ def.irga.vali.cor <- function( Date <- c(base::as.Date(DateProc) - 1, base::as.Date(DateProc), base::as.Date(DateProc) + 1) Date <- as.character(Date) Freq <- Freq #measurement frequency (20 Hz) - #threshold - #minimum and maximum slope - minSlp <- 1 - FracSlpMax - maxSlp <- 1 + FracSlpMax - #scale - ScalMax <- ScalMax - #check if the slope and scale are meet the criteria if not replace them with NA - #default slope less than or equal to +/-10% (0.9 <= slope <=1.10) + + #create a list to keep all filters + filt <- list() + filt$ScalMax <- ScalMax + filt$FracSlp <- FracSlp + filt$OfstMax <- OfstMax + + + #check if filter will apply. for (idxDate in Date){ #idxDate <- Date[1] for (idxData in names(coef[[idxDate]])){ #idxData <- names(coef[[idxDate]])[1] - if (!is.na(coef[[idxDate]][[idxData]]$coef[2]) & !is.na(coef[[idxDate]][[idxData]]$scal[1]) & coef[[idxDate]][[idxData]]$coef[2] >= minSlp & coef[[idxDate]][[idxData]]$coef[2] <= maxSlp & coef[[idxDate]][[idxData]]$scal[1] <= ScalMax){ - coef[[idxDate]][[idxData]] <- coef[[idxDate]][[idxData]] - }else{ - coef[[idxDate]][[idxData]]$coef <- NA - coef[[idxDate]][[idxData]]$se <- NA - coef[[idxDate]][[idxData]]$scal <- NA - } - } - } + for (idxFilt in names(filt)){ + #idxFilt <- names(filt)[1] + if (filt[[idxFilt]] == FALSE){ + coef[[idxDate]][[idxData]] <- coef[[idxDate]][[idxData]] + } else{#testing the filters + if (idxFilt == "ScalMax"){ + if (!is.na(coef[[idxDate]][[idxData]]$scal[1]) & coef[[idxDate]][[idxData]]$scal[1] <= ScalMax){ + coef[[idxDate]][[idxData]] <- coef[[idxDate]][[idxData]] + } else { + coef[[idxDate]][[idxData]]$coef <- NA + coef[[idxDate]][[idxData]]$se <- NA + coef[[idxDate]][[idxData]]$scal <- NA + } + }#end ScalMax + + if (idxFilt == "FracSlp"){ + #calculate minimum and maximum slope + minSlp <- base::min(FracSlp) + maxSlp <- base::max(FracSlp) + if (!is.na(coef[[idxDate]][[idxData]]$coef[2]) & (coef[[idxDate]][[idxData]]$coef[2] >= minSlp & coef[[idxDate]][[idxData]]$coef[2] <= maxSlp)){ + coef[[idxDate]][[idxData]] <- coef[[idxDate]][[idxData]] + } else { + coef[[idxDate]][[idxData]]$coef <- NA + coef[[idxDate]][[idxData]]$se <- NA + coef[[idxDate]][[idxData]]$scal <- NA + } + }#end of FracSlp + + if (idxFilt == "OfstMax"){ + if (!is.na(coef[[idxDate]][[idxData]]$coef[1]) & (abs(coef[[idxDate]][[idxData]]$coef[1]) <= OfstMax)){ + coef[[idxDate]][[idxData]] <- coef[[idxDate]][[idxData]] + } + else { + coef[[idxDate]][[idxData]]$coef <- NA + coef[[idxDate]][[idxData]]$se <- NA + coef[[idxDate]][[idxData]]$scal <- NA + } + }#end OfstMax + }#end of testing filter + + }#end of idxFilt + + + }#end of idxData + }#end idxDate #organize input coefficient table if (valiCrit == TRUE){ @@ -106,21 +162,36 @@ def.irga.vali.cor <- function( numDate <- 0 for (idx in 1:length(dateBgn)){ numDate <- numDate + 1 + tmpTimeBgn <- as.POSIXlt(paste(dateBgn[idx], " ", "23:59:59.950", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC") + tmpTimeEnd <- as.POSIXlt(paste(dateEnd[idx], " ", "", sep="00:00:00.000"), format="%Y-%m-%d %H:%M:%OS", tz="UTC") #time begin and time End to apply coefficient - #time when performing of high gas is done - if (length(valiData[[dateBgn[idx]]][[coefBgn[idx]]]$timeEnd[which(valiData[[dateBgn[idx]]][[coefBgn[idx]]]$gasType == "qfIrgaTurbValiGas05")]) == 0){ + #time when performing of last gas is done + if (all(valiData[[dateBgn[idx]]][[coefBgn[idx]]]$timeEnd == tmpTimeBgn) & + all(is.na(valiData[[dateBgn[idx]]][[coefBgn[idx]]]$numSamp))){ timeBgn <- as.POSIXlt(paste(dateBgn[idx], " ", "23:59:59.950", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC") } else { - timeBgn <- as.POSIXlt(valiData[[dateBgn[idx]]][[coefBgn[idx]]]$timeEnd[which(valiData[[dateBgn[idx]]][[coefBgn[idx]]]$gasType == "qfIrgaTurbValiGas05")]) + #identify which row that timeEnd not = "23:59:59.950" + tmpEndRow <- which(valiData[[dateBgn[idx]]][[coefBgn[idx]]]$timeEnd != tmpTimeBgn & !is.na(valiData[[dateBgn[idx]]][[coefBgn[idx]]]$numSamp)) + timeBgn <- as.POSIXlt(valiData[[dateBgn[idx]]][[coefBgn[idx]]]$timeEnd[tmpEndRow[length(tmpEndRow)]]+(60*5.0)) } - #time when performing of zero gas is started - if (length(valiData[[dateEnd[idx]]][[coefEnd[idx]]]$timeBgn[which(valiData[[dateEnd[idx]]][[coefEnd[idx]]]$gasType == "qfIrgaTurbValiGas02")]) == 0){ + #time when performing of first gas is started + if (all(valiData[[dateEnd[idx]]][[coefEnd[idx]]]$timeBgn == tmpTimeEnd) & + all(is.na(valiData[[dateEnd[idx]]][[coefEnd[idx]]]$numSamp))){ timeEnd <- as.POSIXlt(paste(dateEnd[idx], " ", "", sep="00:00:00.000"), format="%Y-%m-%d %H:%M:%OS", tz="UTC") } else { - timeEnd <- as.POSIXlt(valiData[[dateEnd[idx]]][[coefEnd[idx]]]$timeBgn[which(valiData[[dateEnd[idx]]][[coefEnd[idx]]]$gasType == "qfIrgaTurbValiGas02")]) + #identify which row that timeEnd not = "00:00:00.000" + tmpBgnRow <- which(valiData[[dateEnd[idx]]][[coefEnd[idx]]]$timeBgn != tmpTimeEnd & !is.na(valiData[[dateEnd[idx]]][[coefEnd[idx]]]$numSamp)) + timeEnd <- as.POSIXlt(valiData[[dateEnd[idx]]][[coefEnd[idx]]]$timeBgn[tmpBgnRow[1]]-(60*3.5)) } + # #fail safe to make sure timeBgn less than timeEnd + # if (difftime(timeBgn, timeEnd) >= 0){ + # timeBgn <- as.POSIXlt(paste(dateBgn[idx], " ", "23:59:59.950", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC") + # } else { + # timeBgn <- timeBgn + # } + #output time timeOut <- as.POSIXlt(seq.POSIXt( from = as.POSIXlt(timeBgn, format="%Y-%m-%d %H:%M:%OS", tz="UTC"), @@ -130,6 +201,12 @@ def.irga.vali.cor <- function( #fractional timeFracOut <- timeOut$hour + timeOut$min / 60 + timeOut$sec / 3600 + #adding logical to handle the period that falling into the last day and first day of year + if (format(as.Date(timeBgn, format="%d/%m/%Y"),"%Y") != format(as.Date(timeEnd, format="%d/%m/%Y"),"%Y")){ + #replace those fist day (equal to 0 to max(timeOut$yday)+1) + timeOut$yday[timeOut$yday != max(timeOut$yday)] <- max(timeOut$yday)+1 + } + #calculate doy timeDoy <- timeOut$yday + 1 + timeFracOut / 24 @@ -170,8 +247,11 @@ def.irga.vali.cor <- function( if (all(is.na(ofstLin)) & all(is.na(slpLin))){ #subData$rtioMoleDryCo2Cor <- as.numeric(subData$rtioMoleDryCo2) subData$rtioMoleDryCo2Cor <- NA + subData$rtioMoleDryH2oCor <- NA } else { subData$rtioMoleDryCo2Cor <- as.numeric(ofstLin + subData$rtioMoleDryCo2*slpLin) + #place holder for future h2o correction + subData$rtioMoleDryH2oCor <- NA } #outSub[[idx]] <- subData @@ -185,7 +265,8 @@ def.irga.vali.cor <- function( } #append dataframe #outTmp00 <- do.call(rbind,outSub) - +#change row.name in allSubData + row.names(allSubData) <- make.names(1:length(allSubData$time), unique = TRUE) #return data only the processing date #report time options(digits.secs=3) @@ -211,16 +292,15 @@ rpt <- eddy4R.base::def.rglr(timeMeas = base::as.POSIXlt(outTmp01$time, format=" #replace time to regularize time rpt$time <- timeRglr #check if rtioMoleDryCo2Cor in rpt if not add them with all NA -if (length(rpt$rtioMoleDryCo2Cor) == 0){ - rpt$rtioMoleDryCo2Cor <- NA -} +if (length(rpt$rtioMoleDryCo2Cor) == 0) {rpt$rtioMoleDryCo2Cor <- NA} +if (length(rpt$rtioMoleDryH2oCor) == 0) {rpt$rtioMoleDryH2oCor <- NA} #Creating the index to organize the variables in alphabetical order idxIrga <- order(names(rpt)) #Changing the order of the variables to alphabetical order using the index rpt <- rpt[,idxIrga] #adding unit attributes attrUnit <- c("-", "-", "molCo2 m-3", "molH2o m-3", "NA", "V", "W", "W", "W", "W", "Pa", "Pa", "Pa", "molCo2 mol-1Dry", "molCo2 mol-1Dry", "molH2o mol-1Dry", - "-", "-", "K", "K", "K", "K", "NA") + "molH2o mol-1Dry", "-", "-", "K", "K", "K", "K", "NA") for(idxVar in 1:length(attrUnit)) { diff --git a/pack/eddy4R.base/R/def.irga.vali.thsh.R b/pack/eddy4R.base/R/def.irga.vali.thsh.R new file mode 100644 index 00000000..5206d96e --- /dev/null +++ b/pack/eddy4R.base/R/def.irga.vali.thsh.R @@ -0,0 +1,123 @@ +############################################################################################## +#' @title Definition function: Threshold IRGA validation data based on benchmarking regression + +#' @author +#' Chris Florian \email{eddy4R.info@gmail.com} + +#' @description Wrapper function to apply IRGA validation. + +#' @param data List of validation data as a report from eddy4R.base::wrap.irga.vali() +#' @param gasRefe List containing the values of the reference gases. [mol mol-1] +#' @param DateProc A vector of class "character" containing the processing date. +#' @param evalSlpMax Maximum acceptable slope of the benchmarking regression +#' @param evalSlpMin Minimum acceptable slope of the benchmarking regression +#' @param evalOfstMax Maximum acceptable offset of the benchmarking regression +#' @param evalOfstMin Minimum acceptable offset of the benchmarking regression + + +#' @return pass/fail criteria for a day's validation + + +#' @references +#' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007. + +#' @keywords derived, irgaTurb, post-processing, pre-processing, validation + +#' @examples +#' Currently none. + +#' @seealso Currently none. + +#' @export + +# changelog and author contributions / copyrights +# Chris Florian (2021-08-03) +# original creation +# Chris Florian (2021-08-09) +# adding logic to prevent errors if data are missing +# Chris Florian (2021-08-18) +# updating terms to replace bnch with eval, adding additional coef outputs and adding offset criteria +# Chris Florian (2021-02-15) +# updating logic for failsafe to prevent lm() error due to missing values +############################################################################################## + +def.irga.vali.thsh <- function( + data, + DateProc, + evalSlpMax, + evalSlpMin, + evalOfstMax, + evalOfstMin) { + + #get reference gas values for the processing date (in mol mol-1) + + zeroRefe <- data$rtioMoleDryCo2Vali$rtioMoleDryCo2Refe[2] + lowRefe <- data$rtioMoleDryCo2Vali$rtioMoleDryCo2Refe[3] + midRefe <- data$rtioMoleDryCo2Vali$rtioMoleDryCo2Refe[4] + highRefe <- data$rtioMoleDryCo2Vali$rtioMoleDryCo2Refe[5] + + refeVals <- c(zeroRefe, lowRefe, midRefe, highRefe) + + #get the mean measured values of the reference gas for the processing date + zeroMeas <- data$rtioMoleDryCo2Vali$mean[2] + lowMeas <- data$rtioMoleDryCo2Vali$mean[3] + midMeas <- data$rtioMoleDryCo2Vali$mean[4] + highMeas <- data$rtioMoleDryCo2Vali$mean[5] + + #correct measured LI7200 validation gas data based on calibration coefficients + + meanZeroCor <- zeroMeas*data$rtioMoleDryCo2Mlf$coef[2] + data$rtioMoleDryCo2Mlf$coef[1] + meanLowCor <- lowMeas*data$rtioMoleDryCo2Mlf$coef[2] + data$rtioMoleDryCo2Mlf$coef[1] + meanMidCor <- midMeas*data$rtioMoleDryCo2Mlf$coef[2] + data$rtioMoleDryCo2Mlf$coef[1] + meanHighCor <- highMeas*data$rtioMoleDryCo2Mlf$coef[2] + data$rtioMoleDryCo2Mlf$coef[1] + + meanCor <- c(meanZeroCor, meanLowCor, meanMidCor, meanHighCor) + + + #run benchmarking least squares regression on corrected mean values from the reference gasses vs. the reference values + #adding logic to avoid an error when one of the lists passed into lm() is entirely NA + + if(sum(!is.na(refeVals)) > 1 & sum(!is.na(meanCor)) > 0){ #lm() will fail if one list is entirely NA, or if both lists have only one value. + valiEval <- stats::lm(meanCor ~ refeVals) + valiEvalSe <- sqrt(diag(vcov(valiEval))) + valiEvalSlp <- valiEval$coefficient[[2]] + valiEvalOfst <- valiEval$coefficient[[1]] + } else { + msg <- paste0("valiEval coefficients set to NA beacuse of insufficent refe or measured values") + tryCatch({rlog$debug(msg)}, error=function(cond){print(msg)}) + valiEval <- NA + valiEvalSe <- NA + valiEvalSlp <- NA + valiEvalOfst <- NA + } + + #determine if slope passes + + evalSlpPass <- valiEvalSlp >= evalSlpMin & valiEvalSlp <= evalSlpMax + + #determine if offset passes + + evalOfstPass <- valiEvalOfst >= evalOfstMin & valiEvalOfst <= evalOfstMax + + #set valiPass flag, 0 for good validation, 1 for bad, -1 for missing values + if(!is.na(valiEvalSlp)){ + if (evalSlpPass == TRUE & evalOfstPass == TRUE){ + valiEvalPass <- TRUE + } else { + valiEvalPass <- FALSE + } + } else { + valiEvalPass <- -1 + } + + #compile report including validation pass status and the corrected reference files to add to the vali table + rpt <- list() + rpt$valiEvalPass <- valiEvalPass + rpt$meanCor <- meanCor + rpt$evalCoef <- c(valiEvalOfst, valiEvalSlp) + rpt$evalCoefSe <- valiEvalSe + rpt$evalSlpThsh <- c(evalSlpMin, evalSlpMax) + rpt$evalOfstThsh <- c(evalOfstMin, evalOfstMax) + + return(rpt) +} diff --git a/pack/eddy4R.base/R/def.mtch.out.refe.R b/pack/eddy4R.base/R/def.mtch.out.refe.R index 5f5fd4fd..f62b7a00 100644 --- a/pack/eddy4R.base/R/def.mtch.out.refe.R +++ b/pack/eddy4R.base/R/def.mtch.out.refe.R @@ -39,6 +39,8 @@ # rename to def.mtch.out.refe() # Natchaya P-Durden (2018-04-03) # update @param format +# David Durden (2020-07-16) +# add some information when it falls ############################################################################################## def.mtch.out.refe <- function( @@ -55,7 +57,7 @@ def.mtch.out.refe <- function( #Compare the first numLine lines of the data between the output and reference if(!isTRUE(base::all.equal(dataOut[1:NumLine,],dataRefe[1:NumLine,]))){ - base::stop("Bummer! The current output DOES NOT MATCH the reference output :(") + base::stop(cat("Summary output: \n dataOut & dataRefe \n", str(dataOut[1:NumLine,]),"\n\n", str(dataRefe[1:NumLine,]),"Bummer! The current output DOES NOT MATCH the reference output :( ==> diferences are: \n", base::all.equal(dataOut[1:NumLine,],dataRefe[1:NumLine,]), "\n\n" )) } else { tryCatch({rlog$debug("Yay! The current output MATCHES the reference output :)")}, error=function(cond){ diff --git a/pack/eddy4R.base/R/def.para.flow.R b/pack/eddy4R.base/R/def.para.flow.R index f7c10ab5..1ed0366a 100644 --- a/pack/eddy4R.base/R/def.para.flow.R +++ b/pack/eddy4R.base/R/def.para.flow.R @@ -20,10 +20,12 @@ #' @param VersDp is the data product level that will be output #' @param VersEddy is the version of the eddy4R docker that is being used to perform the processing #' @param MethParaFlow is the method used to specify workflow parameters, "EnvVar" will grab ParaFlow parameters from environmental variable and "DfltInp" will use whatever is specified in the function call. +#' @param MethDp01Api is a logical (TRUE/FALSE) determining if Dp01 reingest data for storage exchange workflow should be gathered from the API. #' @param UrlInpRefe A single-entry vector of class "character" containing the web address of the reference input data zip file to be downloaded. #' @param UrlOutRefe A single-entry vector of class "character" containing the web address of the reference output data zip file to be downloaded. +#' @param MethCh4Conc is a logical (TRUE/FALSE) determining if Dp01 methane concentration from the Picarro G2131 should be run -#' @return \code{ParaFlow} is a list returned that indicates the workflow control parameters, including \code{ParaFlow$DirFilePara},\code{ParaFlow$DirInp}, \code{ParaFlow$DirMnt}, \code{ParaFlow$DirOut}, \code{ParaFlow$DirTmp}, \code{ParaFlow$DirWrk}, \code{ParaFlow$DateOut}, \code{ParaFlow$FileOutBase}, \code{ParaFlow$Read}, \code{ParaFlow$VersDp}, \code{ParaFlow$VersEddy}. +#' @return \code{ParaFlow} is a list returned that indicates the workflow control parameters, including \code{ParaFlow$DirFilePara},\code{ParaFlow$DirInp}, \code{ParaFlow$DirMnt}, \code{ParaFlow$DirOut}, \code{ParaFlow$DirTmp}, \code{ParaFlow$DirWrk}, \code{ParaFlow$DateOut}, \code{ParaFlow$FileOutBase}, \code{ParaFlow$Read}, \code{ParaFlow$VersDp}, \code{ParaFlow$VersEddy}, \code{ParaFlow$MethDp01Api}. #' @references #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007. \cr @@ -51,6 +53,10 @@ # superseed parameter MethMeas with the ability to directly provide reference data urls as arguments UrlInpRefe and UrlOutRefe # Stefan Metzger (2017-09-29) # fixing construction of ParaFlow$DirFilePara for batch-processing (when not using gold file) +# Dave Durden (2020-07-06) +# adding MethDp01Api parameter +# Chris Florian (2021-09-21) +# adding MethCh4Conc parameter ############################################################################################################## #Start of function call to determine workflow parameters @@ -70,12 +76,14 @@ def.para.flow <- function( VersDp = c("001","004")[1], VersEddy = "latest", MethParaFlow = c("DfltInp","EnvVar")[1], + MethDp01Api = TRUE, UrlInpRefe, UrlOutRefe, + MethCh4Conc = FALSE, ... ){ - ParaFlow <- list(Deve = Deve, DirFilePara = DirFilePara,DirInp = DirInp,DirMnt = DirMnt,DirOut = DirOut,DirTmp = DirTmp,DirWrk = DirWrk, DateOut = DateOut, FileOutBase = FileOutBase, MethParaFlow = MethParaFlow,Read = Read,VersDp = VersDp,VersEddy = VersEddy, ...) + ParaFlow <- list(Deve = Deve, DirFilePara = DirFilePara,DirInp = DirInp,DirMnt = DirMnt,DirOut = DirOut,DirTmp = DirTmp,DirWrk = DirWrk, DateOut = DateOut, FileOutBase = FileOutBase, MethParaFlow = MethParaFlow,Read = Read,VersDp = VersDp,VersEddy = VersEddy, MethDp01Api = MethDp01Api, MethCh4Conc = MethCh4Conc, ...) if(MethParaFlow == "EnvVar"){ #Create a list with all the specified function arguments diff --git a/pack/eddy4R.base/R/def.rglr.R b/pack/eddy4R.base/R/def.rglr.R index 52b15ce2..7b0c662e 100644 --- a/pack/eddy4R.base/R/def.rglr.R +++ b/pack/eddy4R.base/R/def.rglr.R @@ -9,30 +9,31 @@ #' @description Function defintion. #' Takes a (potentially) irregularly spaced timeseries \code{timeMeas} of data \code{dataMeas} and returns a strictuly regularly spaced timeseries \code{timeRglr} of data \code{dataRglr}. \strong{ATTENTION}: \code{MethRglr = "zoo"} uses the zoo:na.approx() function, which does not currently abide by its \code{maxgap} argument version 1.7-13. In result, where gaps exist currently the last known value is repeated instead of NAs being inserted. An Email with a request for bugfixing has been sent to \email{Achim.Zeileis@R-project.org} (2016-05-08). -#' @param timeMeas A vector containing the observation times. Of class "POSIXlt" including timezone attribute, and of the same length as \code{dataMeas}. [-] -#' @param dataMeas A named data.frame containing the observations. Columns may be of class "numeric" or "integer", and of the same length as \code{timeMeas}. Columns of classes other than "numeric" or "integer" are removed and not included in the returned \code{dataRegl}. [user-defined] +#' @param timeMeas A vector containing the observation times. Of class "POSIXlt" including timezone attribute, and of the same row length as \code{dataMeas}. [-] +#' @param dataMeas A named data.frame containing the observations, with row length matching that of \code{timeMeas}. Not that if the zoo method is chosen in \code{MethRglr} or input \code{DropNotNumc} is TRUE, any columns with class other than "numeric" or "integer" are removed and not included in the returned \code{dataRegl}. [user-defined] #' @param unitMeas A vector containing the unit of each column in \code{dataMeas}. Of class "character". It is recommended to conform to the "unit representation" guidelines documented in the eddy4R.base package. -#' @param BgnRglr Desired begin time for the regularized dataset. Of class "POSIXlt" including timezone attribute, and \code{length(BgnRglr) = 1}. This input is not used in the "cybiDflt" method. [-] -#' @param EndRglr Desired end time for the regularized dataset. Of class "POSIXlt" including timezone attribute, and \code{length(EndRglr) = 1}. This input is not used in the "cybiDflt" method. [-] -#' @param TzRglr Desired timezone for the regularized dataset. Of class "character" and \code{length(TzRglr) = 1}, defaults to the same timezone as \code{BgnRglr}. For the "cybiDflt" method, the same time zone as timeMeas is used. [-] +#' @param BgnRglr Desired begin time for the regularized dataset. Of class "POSIXlt" including timezone attribute, and \code{length(BgnRglr) = 1}. [-] +#' @param EndRglr Desired end time for the regularized dataset. Of class "POSIXlt" including timezone attribute, and \code{length(EndRglr) = 1}. [-] +#' @param TzRglr Desired timezone for the regularized dataset. Of class "character" and \code{length(TzRglr) = 1}, defaults to the same timezone as \code{BgnRglr}. [-] #' @param FreqRglr Desired frequency of the regularized dataset. Of class "numeric" or "integer" and \code{length(FreqRglr) = 1}. [Hz] #' @param MethRglr Switch for different regularization methods. Of class "character", currently defaults to "CybiEc". [-] \cr -#' Method "cybiDflt" implements the default for metereological variable regularization performed by NEON CI. Namely, a new time series is created -#' from the first measurement time, rounded toward zero, using the expected data frequency. The first measurement falling -#' in between one time stamp and the next is assigned to the first of these, and all other measurements falling in this range are ignored.\cr #' Method "CybiEc" implements the default regularization method for eddy-covariance processing utilized CI. The procedure #' is documented in NEON.DOC.001069.\cr +#' Method "CybiEcTimeMeas" is a modification of CybiEc that replaces the regularized timestamp with the actual value of \code{timeMeas} for the observation (if any) that was selected following the binning options specified in \code{WndwRglr} and \code{idxWndw}. \cr #' Method "zoo" implements the regularization method using the zoo::na.approx function. This method can only handle up to millisecond precision (PrcsSec=3) #' @param WndwRglr Position of the window for binning in the "CybiEc" method. \code{WndwRglr} can be centered [Cntr], leading [Lead], or trailing [Trlg] (defaults to centered).\cr #' @param IdxWndw Determines which observation to allocate to a bin if multiple observations fall into a single bin when using the "CybiEc" method.. \code{IdxWndw} can be set to closest [Clst], first [IdxWndwMin], or last [IdxWndwMax] (defaults to closest).\cr +#' @param DropNotNumc Logical. TRUE (default) for removing any non-numeric data columns prior to regularization (this is done automatically for zoo method). FALSE to attempt to regularize all data columns. +#' @param RptTimeWndw Logical. TRUE for including the start and end time of each bin with the output, in list element timeWndw. Defaults to FALSE. Not available as TRUE for zoo method. #' @param PrcsSec A single numeric (integer) value indicating the operational precision of the seconds field of time vectors. Defaults to 6 (microsecond-precision). Values higher than 6 cannot be guaranteed to produce desired results. -#' @return Returns a list with elements \code{TzRglr}, \code{FreqRglr}, \code{MethRglr}, \code{timeRglr}, and \code{dataRglr}. +#' @return Returns a list with elements \code{TzRglr}, \code{FreqRglr}, \code{MethRglr}, \code{timeRglr}, and \code{dataRglr}. +#' An additional list element \code{timeWndw} will be included if input \code{RptTimeWndw=TRUE} #' @references #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007. \cr -#' NEON.DOC.001069 Preprocessing ATBD: The ATBD that describes the CybiEc and cybiDflt regularization methods. \cr +#' NEON.DOC.001069 Preprocessing ATBD: The ATBD that describes the CybiEc regularization method. \cr #' #' @keywords regularization, equidistant, preprocessing @@ -104,6 +105,16 @@ # Added error checking for empty data when using cybiDflt method # Cove Sturtevant (2018-05-23) # Changed term 'pos' to 'idx' for single indices, to 'set' for multiple indices +# Cove Sturtevant (2020-02-18) +# Removed MethRglr "cybiDflt", as it is no longer used by NEON CI (CybiEc is used) +# Added MethRglr "CybiEcTimeMeas" +# Cove Sturtevant (2021-02-02) +# Added option to retain non-numeric columns for everything but zoo method +# Added option to output the time bins (start and end times) in a new list element in the output +# Cove Sturtevant (2021-02-15) +# bug fix. Sometimes class returns 'array', which was causing an error when forcing the type of +# each regularized variable to the same as that in the input data frame. +# Replaced 'class' with 'typeof' to fix. ############################################################################################## def.rglr <- function( @@ -114,9 +125,11 @@ def.rglr <- function( EndRglr=NULL, TzRglr = base::attributes(BgnRglr)$tzone, FreqRglr, - MethRglr= c("CybiEc", "cybiDflt", "zoo")[1], + MethRglr= c("CybiEc", "CybiEcTimeMeas", "zoo")[1], WndwRglr = c("Cntr", "Lead", "Trlg")[1], IdxWndw = c("Clst","IdxWndwMin","IdxWndwMax")[1], + DropNotNumc = TRUE, + RptTimeWndw = FALSE, PrcsSec = 6 ){ @@ -132,23 +145,22 @@ def.rglr <- function( } # Error-check - if(!(MethRglr %in% c("zoo","cybiDflt","CybiEc"))){ - stop(base::paste0('Unrecognized value for input MethRglr. Options are "zoo","cybiDflt","CybiEc" (case-sensitive)')) + if(!(MethRglr %in% c("zoo","CybiEcTimeMeas","CybiEc"))){ + stop(base::paste0('Unrecognized value for input MethRglr. Options are "zoo","CybiEc", and "CybiEcTimeMeas" (case-sensitive)')) } - # CI uses the first value as the starting point for the regularization, rounding down to the nearest second - # Note: the rounding down aspect is a change implemented week of 1 May 2016. Previously the starting point was - # the exact time (to the decimal second). - if(MethRglr == "cybiDflt"){ - BgnRglr <- base::as.POSIXlt(base::trunc.POSIXt(timeMeas[1],units="secs")) - EndRglr <- base::as.POSIXlt(utils::tail(timeMeas,1) + 1/FreqRglr) - - if(base::is.null(TzRglr)) { - TzRglr <- base::attributes(BgnRglr)$tzone - } + # Error-check + if(MethRglr == "zoo" && DropNotNumc == FALSE){ + warning('Input parameter DropNotNumc is always set to TRUE for MethRglr=zoo. User selected input has been overwritten.') + } + + # Error-check + if(MethRglr == "zoo" && RptTimeWndw == TRUE){ + warning('Input parameter RptTimeWndw = TRUE is not currently an option for MethRglr=zoo. User selected input has been overwritten.') } + - if(MethRglr %in% c("zoo","cybiDflt","CybiEc")){ + if(MethRglr %in% c("zoo","CybiEcTimeMeas","CybiEc")){ if(!("POSIXlt" %in% base::class(timeMeas))){ stop("Input 'timeMeas' must be in POSIXlt") } @@ -214,14 +226,7 @@ def.rglr <- function( if(MethRglr == 'zoo' && PrcsSec > 3){ PrcsSec <- 3 } - - # Quit if there is no data and we are using cybiDflt method, since the cybiDflt method - # relies on the first data point to generate the time seq - if(base::length(timeMeas) == 0 && MethRglr == 'cybiDflt'){ - rpt <- base::list(TzRglr=NULL,FreqRglr=FreqRglr,MethRglr=MethRglr,timeRglr=timeMeas,dataRglr=dataMeas) - return(rpt) - } - + # POSIX time has some issues with sub-second precision, often rounding down to a lower value without an # obvious reason. As a result, use numeric representation of time and round to a specified precision. # When returning to POSIX time, ensure use of POSIXlt so that down-rounding does not occur. @@ -246,6 +251,7 @@ def.rglr <- function( rpt$FreqRglr <- FreqRglr rpt$MethRglr <- MethRglr rpt$timeRglr <- timeRglr + numRglr <- length(timeRglr) # default: using the zoo::na.approx() function # takes 3 s for 1,728,000 observations, i.e. one day of one 20 Hz variable @@ -279,7 +285,7 @@ def.rglr <- function( # if less than 2 values (minimum required by na.approx() function) if(whr03 < 2) { - rpt$dataRglr[,idx] <- base::rep(NaN, base::length(rpt$timeRglr)) + rpt$dataRglr[,idx] <- base::rep(NaN, numRglr) #else interpolate dataMeas } else { @@ -308,52 +314,9 @@ def.rglr <- function( # end MethRglr == zoo } - - - # Regularize time series according to default NEON cyber infrastructure (CI) L0 -> L0' procedure. - # NEON CI transforms raw L0 data into a regularized time series according to the expected data frequency. - # Namely, a new time series is created from the first measurement time, rounded toward zero, using the - # expected data frequency. The first measurement falling in between one time stamp and the next is assigned - # to the first of these, and all other measurements falling in this range are ignored. - # This code replicates this procedure in order to compare expected output to that produced by CI. - if(MethRglr == "cybiDflt") { - - # Which time bin does each measurement time fit into? - idxRglr <- base::.bincode(timeMeasNumc,timeRglrNumc,right=FALSE) # which bin? - dataMeas <- base::subset(dataMeas,!base::is.na(idxRglr),select=1:numVar) # Get rid of anomalous times/data not fitting in any bin - timeMeasNumc <- base::subset(timeMeasNumc,!base::is.na(idxRglr)) - idxRglr <- base::subset(idxRglr,!base::is.na(idxRglr)) - dupl <- base::duplicated(idxRglr) # which fall into an already occupied bin? - - # intialize regularized timeseries - if(base::is.character(dataMeas$data) || base::is.factor(dataMeas$data)){ - dataRglr <- base::matrix(data="NA",nrow=length(timeRglrNumc)-1,ncol=numVar) # initialize - } else { - dataRglr <- base::matrix(data=NA*1.5,nrow=length(timeRglrNumc)-1,ncol=numVar) # initialize - } - - # Pull the first value that falls within each bin - for(idxVar in 1:numVar){ - # place the first value falling into each bin - dataRglr[idxRglr[!dupl],idxVar] <- dataMeas[which(!dupl),idxVar] - } - dataRglr <- base::as.data.frame(dataRglr,stringsAsFactors=FALSE) # Make data frame - base::names(dataRglr) <- nameVar # Assign names same as dataMeas - - # Report output - rpt$timeRglr <- base::as.POSIXlt(timeRglr) - - rpt$timeRglr <- rpt$timeRglr[-length(rpt$timeRglr)] - rpt$dataRglr <- dataRglr - - # assign unit attributes - base::attributes(rpt$dataRglr)$unit <- unitMeas - } - - # Method "CybiEc" implements the default regularization method for eddy-covariance # processing utilized CI. The procedure is documented in NEON.DOC.001069. - if(MethRglr == "CybiEc") { + if(MethRglr %in% c("CybiEc","CybiEcTimeMeas")) { # delete rows with times that are duplicates of rows with smaller indices set01 <- !base::duplicated(timeMeasNumc) @@ -364,12 +327,14 @@ def.rglr <- function( # reduce dataMeas to variables that are of type double or integer (not character!) - set02 <- base::sapply(1:base::ncol(dataMeas), function(x) base::typeof(dataMeas[[x]])) - set02 <- which((set02 %in% c("double", "integer"))) - dataMeas <- base::subset(dataMeas, select = set02) - unitMeas <- unitMeas[set02] - base::rm(set02) - + if(DropNotNumc == TRUE){ + set02 <- base::sapply(1:base::ncol(dataMeas), function(x) base::typeof(dataMeas[[x]])) + set02 <- which((set02 %in% c("double", "integer"))) + dataMeas <- base::subset(dataMeas, select = set02) + unitMeas <- unitMeas[set02] + base::rm(set02) + } + # Number of variables in dataframe numVar <- base::ncol(dataMeas) # Variable names @@ -422,13 +387,34 @@ def.rglr <- function( }}else{dupl <- rep(FALSE, length(idxRglr))} #If no duplicates exist, all equal FALSE # Pull the value that chosen by IdxWndw within each bin - dataRglr <- base::data.frame(base::matrix(data=NA*1.5,nrow=length(timeRglrNumc),ncol=numVar)) # initialize, mulitply by 1.5 to give numeric + classData <- lapply(dataMeas,base::class) # Get the type of each variable so we can make sure the output gets the same + typeData <- lapply(dataMeas,base::typeof) + dataRglr <- base::data.frame(base::matrix(data=NA*1.5,nrow=numRglr,ncol=numVar)) # initialize, multiply by 1.5 to give numeric for(idxVar in 1:numVar){ + # Give the column its original class + base::class(dataRglr[[idxVar]]) <- tryCatch( + base::class(dataRglr[[idxVar]]) <- classData[[idxVar]], + error=function(e){base::class(dataRglr[[idxVar]]) <- typeData[[idxVar]]}) + + # place the value falling into each bin dataRglr[idxRglr[!dupl],idxVar] <- dataMeas[which(!dupl),idxVar] } base::names(dataRglr) <- nameVar # Assign names same as dataMeas + # For CybiEcTimeMeas, replace the regularized timestamps with the actual timestamps of the measured values selected for each bin + if(MethRglr == 'CybiEcTimeMeas'){ + rpt$timeRglr[idxRglr[!dupl]] <- timeMeas[which(!dupl)] + } + + if(RptTimeWndw==TRUE){ + secRtioWndw <- base::round(timeWndw-base::floor(timeWndw),digits=PrcsSec) # Grab the fractional seconds + timeWndw <- base::as.POSIXlt(base::floor(timeWndw),tz=TzRglr,origin=epoc,digits=20) # Convert to POSIXlt + timeWndw$sec <- timeWndw$sec+secRtioWndw # Add back in the fractional seconds + rpt$timeWndw <- data.frame(timeWndwBgn=timeWndw[1:numRglr],timeWndwEnd=timeWndw[2:(numRglr+1)]) + } + + # Report output rpt$dataRglr <- dataRglr # assign unit attributes diff --git a/pack/eddy4R.base/R/wrap.dp01.agr.prd.R b/pack/eddy4R.base/R/wrap.dp01.agr.prd.R index 4c548edd..25ee257d 100644 --- a/pack/eddy4R.base/R/wrap.dp01.agr.prd.R +++ b/pack/eddy4R.base/R/wrap.dp01.agr.prd.R @@ -50,6 +50,8 @@ # rename function from wrap.neon.dp01.agr.prd() to wrap.dp01.agr.prd() # Natchaya P-Durden (2018-05-23) # rename function from wrap.neon.dp01.qfqm.ec() to wrap.dp01.qfqm.ecte() +# David Durden (2021-07-15) +# dealing with R4.0.3 update bug, tmp$qfqm <- NULL was removing tmp$qfqmOut values, put in failsafe ############################################################################################## @@ -133,12 +135,13 @@ for(idxAgr in 1:iter){ ) tmp$qfqmOut[[lvlAgr]] <- eddy4R.base::wrap.dp01.qfqm.ecte(qfqm = tmp$qfqm, idx = names(tmp$data), MethMeas = "ecte", RptExpd = FALSE ) - + +tmp$qfqm[names(tmp$qfqm)] <- NULL tmp$data <- NULL -tmp$qfqm <- NULL invisible(gc()) } +#Format for output rpt <- eddy4R.base::def.dp01.agr.ecte(inpList = tmp) return(rpt) diff --git a/pack/eddy4R.base/R/wrap.hdf5.wrte.dp01.R b/pack/eddy4R.base/R/wrap.hdf5.wrte.dp01.R index eb0b85ba..98ae0e11 100755 --- a/pack/eddy4R.base/R/wrap.hdf5.wrte.dp01.R +++ b/pack/eddy4R.base/R/wrap.hdf5.wrte.dp01.R @@ -15,6 +15,7 @@ #' @param MethUcrt Logical: Determines if uncertainty information is available for output. #' @param MethDp04 logical indicating if ECTE dp04 HDF5 data should be included. #' @param MethSubAgr Logical: Determines if 1-minute data is available for output. +#' @param Meta A list of parameters and metadata for updating output HDF5 metadata #' #' @return An HDF5 file with dp01 data, qfqm, and uncertainty written @@ -64,6 +65,16 @@ # Adding dp04 low resolution output on top of validation code # Natchaya P-Durden (2019-09-30) # only write qfFinl out in basic file +# Natchaya P-Durden (2020-01-17) +# adding rtioMoleDryH2o during validation +# Natchaya P-Durden (2020-01-22) +# adding timeBgn and timeEnd attributes +# David Durden (2020-05-01) +# adding Pfit coefficient output metadata +# David Durden (2021-08-17) +# Fixing attribute issues related to rhdf5 not being able to write lists +# David Durden (2021-08-17) +# Adding difference to UTC to output ############################################################################################## @@ -77,7 +88,8 @@ wrap.hdf5.wrte.dp01 <- function( LvlTowr, MethUcrt = TRUE, MethDp04 = FALSE, - MethSubAgr = TRUE + MethSubAgr = TRUE, + Meta ){ #Determine if the output file should be expanded or basic by creating a logical determined from the filename @@ -121,11 +133,14 @@ if(MethSubAgr == TRUE){ #adding irgaTurb validation data outList$vali$data$co2Turb <- inpList$vali$data$co2Turb + outList$vali$data$h2oTurb <- inpList$vali$data$h2oTurb #If values come in as Posix, they must first be converted to characters for (idxTime in c("timeBgn", "timeEnd")){ if(!is.character(outList$vali$data$co2Turb$rtioMoleDryCo2Vali[[idxTime]])){ outList$vali$data$co2Turb$rtioMoleDryCo2Vali[[idxTime]] <- strftime(outList$vali$data$co2Turb$rtioMoleDryCo2Vali[[idxTime]], format="%Y-%m-%dT%H:%M:%OSZ", tz="UTC")} + if(!is.character(outList$vali$data$h2oTurb$rtioMoleDryH2oVali[[idxTime]])){ + outList$vali$data$h2oTurb$rtioMoleDryH2oVali[[idxTime]] <- strftime(outList$vali$data$h2oTurb$rtioMoleDryH2oVali[[idxTime]], format="%Y-%m-%dT%H:%M:%OSZ", tz="UTC")} } #Unit conversion for dp01 sub-aggregated irgaTurb validation data @@ -133,6 +148,7 @@ if(MethSubAgr == TRUE){ #adding validation data to dp01AgrSub outList$dp01AgrSub$data$co2Turb$rtioMoleDryCo2Vali <- outList$vali$data$co2Turb$rtioMoleDryCo2Vali + outList$dp01AgrSub$data$h2oTurb$rtioMoleDryH2oVali <- outList$vali$data$h2oTurb$rtioMoleDryH2oVali #Packaging sub-aggregated (e.g.1-min) dp01 qfqm for writing to HDF5 file outList$dp01AgrSub$qfqm <- sapply(names(inpList$dp01AgrSub$qfqm), function(x) eddy4R.base::def.hdf5.pack(inpList = inpList$dp01AgrSub$qfqm, time = inpList$dp01AgrSub$time, Dp = x)) @@ -165,17 +181,22 @@ if(MethDp04 == TRUE){ idFile <- rhdf5::H5Fopen(FileOut) for(idxDp04 in names(inpList$dp04$data)){ - #idxDp04 <- names(inpList$dp04$data)[5] + #idxDp04 <- names(inpList$dp04$data)[1] if(idxDp04 == "foot") { #Adding time to output dataframe rptDp04 <- cbind(timeBgn = outList$data$soni$veloXaxsErth$timeBgn, timeEnd = outList$data$soni$veloXaxsErth$timeEnd, inpList$dp04$data[[idxDp04]]$stat, stringsAsFactors = FALSE) #Writing unit attributes to each variable to the dataframe level attributes(rptDp04)$unit <- attributes(inpList$dp04$data[[idxDp04]]$stat)$unit + #Adding timeBgn and timeEnd attributes + attributes(rptDp04)$unit <- c("NA", "NA", attributes(rptDp04)$unit) #Open connection to dp04 data level idDataDp04 <- rhdf5::H5Gopen(idFile,paste0("/", SiteLoca, "/dp04/data/",idxDp04)) + #Output the attributes + rhdf5::h5writeAttribute("North,East,Down (NED)", h5obj = idDataDp04, name = "sysCordRefe") + #Writing flux data to output HDF5 file rhdf5::h5writeDataset.data.frame(obj = rptDp04, h5loc = idDataDp04, name = "stat", DataFrameAsCompound = TRUE) @@ -204,6 +225,11 @@ if(MethDp04 == TRUE){ #Writing unit attributes to each variable to the dataframe level attributes(rptDp04Qfqm)$unit <- sapply(names(inpList$dp04$qfqm[[idxDp04]]$turb), function(x) attributes(inpList$dp04$qfqm[[idxDp04]]$turb[[x]])$unit) + #Adding timeBgn and timeEnd attributes + tmpAttr <- c() + attributes(tmpAttr)$unit[["timeBgn"]] <- "NA" + attributes(tmpAttr)$unit[["timeEnd"]] <- "NA" + attributes(rptDp04Qfqm)$unit <- base::as.character(c(attributes(tmpAttr)$unit, attributes(rptDp04Qfqm)$unit)) #Open connection to dp04 data level idQfqmDp04 <- rhdf5::H5Gopen(idFile,paste0("/", SiteLoca, "/dp04/qfqm/",idxDp04)) @@ -217,9 +243,10 @@ if(MethDp04 == TRUE){ rhdf5::h5writeAttribute(attributes(rptDp04Qfqm)$unit, h5obj = idQfqmDp04Df, name = "unit") - } else { + } + else { #output only flux for fluxCo2 in basic file - if (idxDp04 == c("fluxCo2") & MethExpd == FALSE){ + if (idxDp04 %in% c("fluxCo2", "fluxH2o") & MethExpd == FALSE){ inpList$dp04$data[[idxDp04]]$turb$fluxCor <- NULL inpList$dp04$data[[idxDp04]]$turb$fluxRaw <- NULL } @@ -230,6 +257,12 @@ if(MethDp04 == TRUE){ #Writing unit attributes to each variable to the dataframe level attributes(rptDp04)$unit <- sapply(names(inpList$dp04$data[[idxDp04]]$turb), function(x) attributes(inpList$dp04$data[[idxDp04]]$turb[[x]])$unit) + #Adding timeBgn and timeEnd attributes + tmpAttr <- c() + attributes(tmpAttr)$unit[["timeBgn"]] <- "NA" + attributes(tmpAttr)$unit[["timeEnd"]] <- "NA" + attributes(rptDp04)$unit <- base::as.character(c(attributes(tmpAttr)$unit, attributes(rptDp04)$unit)) + #Open connection to dp04 data level idDataDp04 <- rhdf5::H5Gopen(idFile,paste0("/", SiteLoca, "/dp04/data/",idxDp04)) @@ -259,6 +292,11 @@ if(MethDp04 == TRUE){ #Writing unit attributes to each variable to the dataframe level attributes(rptDp04Qfqm)$unit <- sapply(names(inpList$dp04$qfqm[[idxDp04]]$turb), function(x) attributes(inpList$dp04$qfqm[[idxDp04]]$turb[[x]])$unit) + #Adding timeBgn and timeEnd attributes + tmpAttr <- c() + attributes(tmpAttr)$unit[["timeBgn"]] <- "NA" + attributes(tmpAttr)$unit[["timeEnd"]] <- "NA" + attributes(rptDp04Qfqm)$unit <- base::as.character(c(attributes(tmpAttr)$unit, attributes(rptDp04Qfqm)$unit)) #Open connection to dp04 data level idQfqmDp04 <- rhdf5::H5Gopen(idFile,paste0("/", SiteLoca, "/dp04/qfqm/",idxDp04)) @@ -272,7 +310,8 @@ if(MethDp04 == TRUE){ rhdf5::h5writeAttribute(attributes(rptDp04Qfqm)$unit, h5obj = idQfqmDp04Df, name = "unit") } } - rhdf5::h5closeAll() + #Close HDF5 connections + rhdf5::h5closeAll() } ###################################################################### @@ -280,4 +319,18 @@ if(MethDp04 == TRUE){ ###################################################################### eddy4R.base::def.hdf5.copy.para(FileInp = FileInp, FileOut = FileOut) +#Create HDF5 connection to the output file +idFile <- rhdf5::H5Fopen(FileOut) +#Open connection to dp04 data level +idSite <- rhdf5::H5Gopen(idFile, paste0("/", SiteLoca)) + +#Write updated Pfit coefficients and UTC time diff to output file +#Output the attributes +rhdf5::h5writeAttribute(round(Meta$Sci$`Pf$AngEnuXaxs`, digits = 6), h5obj = idSite, name = "Pf$AngEnuXaxs") +rhdf5::h5writeAttribute(round(Meta$Sci$`Pf$AngEnuYaxs`, digits = 6), h5obj = idSite, name = "Pf$AngEnuYaxs") +rhdf5::h5writeAttribute(round(Meta$Sci$`Pf$Ofst`, digits = 6), h5obj = idSite, name = "Pf$Ofst") +rhdf5::h5writeAttribute(Meta$Site$TimeDiffUtcLt, h5obj = idSite, name = "TimeDiffUtcLt") + +#Close HDF5 connections +rhdf5::h5closeAll() } diff --git a/pack/eddy4R.base/R/wrap.irga.vali.R b/pack/eddy4R.base/R/wrap.irga.vali.R index 4fbe4980..5ef04bf1 100644 --- a/pack/eddy4R.base/R/wrap.irga.vali.R +++ b/pack/eddy4R.base/R/wrap.irga.vali.R @@ -10,6 +10,10 @@ #' @param qfqmFlag List consisting of \code{ff::ffdf} file-backed objects containing the IRGA quality flags. #' @param gasRefe List containing the values of the reference gases. [mol mol-1] #' @param DateProc A vector of class "character" containing the processing date. +#' @param ScalMax Maximum scale value. The validation correction will not apply if scale (resulted from maximum-likelihood fitting of a functional relationship (MLFR)) is greater than ScalMax or ScalMax = FALSE. Defaults to FALSE. +#' @param FracSlp Upper and lower bounds of slope values. The validation correction will not apply if slope (resulted from regression fitting) is greater/lower than the FracSlp maximum or minimum value or FracSlp = FALSE. Defaults to FALSE. +#' @param OfstMax Maximum offset value. The validation correction will not apply if slope (resulted from regression fitting) is greater than the OfstMax (unit in mol mol-1) or OfstMax = FALSE. Defaults to FALSE. + #' @return #' The returned object consists of:\cr @@ -71,13 +75,37 @@ # Natchaya P-Durden (2019-05-09) # updating logic in fail-safe to fill in dataframe with NaN when there is only archive gas or no validation at all # bug fix on selecting the validation gas based on timeCrit +# Natchaya P-Durden (2020-01-15) +# reporting the rtioMoleDryH2oVali table +# Natchaya P-Durden (2020-01-31) +# adjust workflow to run MLFR even missing one gas cylinder +# Natchaya P-Durden (2020-03-05) +# Set all thresholds to screen linear coefficients to FALSE. +# Chris Florian (2021-08-03) +# add thresholding based on benchmarking regression +# Chris Florian (2021-08-09) +# adding -1 flag for missing validations +# Chris Florian (2021-08-26) +# adding failsafe for extra validation gas rows +# Chris Florian (2021-08-27) +# retaining the rest of the rtioMoleDryCo2Cor for failed validations +# adding NaNs for meanCor with failed validations to keep structure the same +# Chris Florian (2021-08-27) +# resetting attributes on rtioMoleDryCo2Cor to fix issues when the corrected data was removed +# Chris Florian (2021-02-15) +# setting corrected data to NaN if qfEvalThsh is -1 to prevent bad data passing through if the evaluation doesn't run +# Chris Florian (2022-03-02) +# Updating the slope filter to allow for values not evenly centered around 1 ############################################################################################## wrap.irga.vali <- function( data, qfqmFlag, gasRefe, - DateProc + DateProc, + ScalMax = FALSE, + FracSlp = FALSE, + OfstMax = FALSE ) { #adding library @@ -119,8 +147,8 @@ wrap.irga.vali <- function( #check if there are data and qfqmFlag if(length(locDate) == 0){ #create the empty dataframe - subData <- data.frame(matrix(ncol = length(data$irgaTurb), nrow = length(data$irgaTurb[[1]]))) - subQfqmFlag <- data.frame(matrix(ncol = length(qfqmFlag$irgaTurb), nrow = length(qfqmFlag$irgaTurb[[1]]))) + subData <- data.frame(matrix(ncol = length(data$irgaTurb), nrow = 24*60*60*Freq))#20Hz data over 1 day + subQfqmFlag <- data.frame(matrix(ncol = length(qfqmFlag$irgaTurb), nrow = 24*60*60*Freq)) colnames(subData) <- names(data$irgaTurb) colnames(subQfqmFlag) <- names(qfqmFlag$irgaTurb) #add time @@ -156,11 +184,12 @@ wrap.irga.vali <- function( #statistical names (will be used when no validation occured at all) NameStat <- c("mean", "min", "max", "vari", "numSamp", "se") - - #assign list - tmp <- list() - rptTmp <- list() - + + #for each loop of rtioMoleDryCo2 and rtioMoleDryH2o + for (idxVar in c("rtioMoleDryCo2", "rtioMoleDryH2o")) { + #assign list + tmp <- list() + rptTmp <- list() #calculate statistical for each gas for (idxNameQf in nameQf){ #idxNameQf <- nameQf[2] @@ -181,25 +210,31 @@ wrap.irga.vali <- function( for (idxAgr in 1:length(idxVali$timeBgn)){ #idxAgr <- 1 - inpTmp <- data.frame(rtioMoleDryCo2 = allSubData$rtioMoleDryCo2[idxVali$idxBgn[idxAgr]:idxVali$idxEnd[idxAgr]]) - + inpTmp <- data.frame(idxVar = allSubData[[idxVar]][idxVali$idxBgn[idxAgr]:idxVali$idxEnd[idxAgr]]) + colnames(inpTmp) <- idxVar + #statistical processing tmp[[idxNameQf]][[idxAgr]] <- eddy4R.base::wrap.dp01(data = inpTmp) #report data rptTmp[[idxNameQf]][[idxAgr]] <- tmp[[idxNameQf]][[idxAgr]] #report time - rptTmp[[idxNameQf]][[idxAgr]]$timeBgn <- data.frame(rtioMoleDryCo2 = idxVali$timeBgn[[idxAgr]]) - rptTmp[[idxNameQf]][[idxAgr]]$timeEnd <- data.frame(rtioMoleDryCo2 = idxVali$timeEnd[[idxAgr]]) + rptTmp[[idxNameQf]][[idxAgr]]$timeBgn <- data.frame(idxVar = idxVali$timeBgn[[idxAgr]]) + rptTmp[[idxNameQf]][[idxAgr]]$timeEnd <- data.frame(idxVar = idxVali$timeEnd[[idxAgr]]) + colnames(rptTmp[[idxNameQf]][[idxAgr]]$timeBgn) <- idxVar + colnames(rptTmp[[idxNameQf]][[idxAgr]]$timeEnd) <- idxVar }#end for each idxAgr #end for at least there is one measurement } else {#if there is no measurement for(idxStat in NameStat){ #report data - rptTmp[[idxNameQf]][[1]][[idxStat]] <- data.frame(rtioMoleDryCo2 = NaN) + rptTmp[[idxNameQf]][[1]][[idxStat]] <- data.frame(idxVar = NaN) + colnames(rptTmp[[idxNameQf]][[1]][[idxStat]]) <- idxVar }#end idxStat #report time - rptTmp[[idxNameQf]][[1]]$timeBgn <- data.frame(rtioMoleDryCo2 = base::as.POSIXlt(paste(DateBgn, " ", "00:00:00.000", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC")) - rptTmp[[idxNameQf]][[1]]$timeEnd <- data.frame(rtioMoleDryCo2 = base::as.POSIXlt(paste(DateBgn, " ", "23:59:59.950", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC")) + rptTmp[[idxNameQf]][[1]]$timeBgn <- data.frame(idxVar = base::as.POSIXlt(paste(DateBgn, " ", "00:00:00.000", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC")) + rptTmp[[idxNameQf]][[1]]$timeEnd <- data.frame(idxVar = base::as.POSIXlt(paste(DateBgn, " ", "23:59:59.950", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC")) + colnames(rptTmp[[idxNameQf]][[1]]$timeBgn) <- idxVar + colnames(rptTmp[[idxNameQf]][[1]]$timeEnd) <- idxVar } }#end of each qf in nameQf @@ -215,7 +250,7 @@ wrap.irga.vali <- function( for (idxLoc in 1:length(rptTmp[[idxGas]])){ for (idxStat in names(rptTmp[[idxGas]][[idxLoc]])){ #idxStat <- names(rptTmp[[idxGas]][[idxLoc]])[1] - outTmp00[[idxStat]] <- data.frame(rptTmp[[idxGas]][[idxLoc]][[idxStat]]$rtioMoleDryCo2) + outTmp00[[idxStat]] <- data.frame(rptTmp[[idxGas]][[idxLoc]][[idxStat]][[idxVar]]) } outTmp01[[idxLoc]] <- do.call(cbind, outTmp00) } @@ -229,13 +264,14 @@ wrap.irga.vali <- function( } #combine row and save statistical outputs into rpt[[idxDate]]$rtioMoleDryCo2Vali - rpt[[idxDate]]$rtioMoleDryCo2Vali <- do.call(rbind, outTmp02) + valiTmp <- paste0(idxVar,"Vali") + rpt[[idxDate]][[valiTmp]] <- do.call(rbind, outTmp02) #assign column names - colnames(rpt[[idxDate]]$rtioMoleDryCo2Vali) <- c("mean", "min", "max", "vari", "numSamp", "se", "timeBgn", "timeEnd", "gasType") + colnames(rpt[[idxDate]][[valiTmp]]) <- c("mean", "min", "max", "vari", "numSamp", "se", "timeBgn", "timeEnd", "gasType") #remove row names - rownames(rpt[[idxDate]]$rtioMoleDryCo2Vali) <- NULL + rownames(rpt[[idxDate]][[valiTmp]]) <- NULL #remove unuse objects rm(outTmp00, outTmp01, outTmp02, rptTmp, idxGas) @@ -245,32 +281,33 @@ wrap.irga.vali <- function( timeMin <- base::as.POSIXlt(paste(DateBgn, " ", "00:01:29.950", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC") timeMax <- base::as.POSIXlt(paste(DatePost, " ", "00:01:29.950", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC") #determine index when timeEnd fall in DateProc - rpt[[idxDate]]$rtioMoleDryCo2Vali <- rpt[[idxDate]]$rtioMoleDryCo2Vali[which(rpt[[idxDate]]$rtioMoleDryCo2Vali$timeEnd >= timeMin & rpt[[idxDate]]$rtioMoleDryCo2Vali$timeBgn < timeMax),] + rpt[[idxDate]][[valiTmp]] <- rpt[[idxDate]][[valiTmp]][which(rpt[[idxDate]][[valiTmp]]$timeEnd >= timeMin & rpt[[idxDate]][[valiTmp]]$timeBgn < timeMax),] #fail safe: fill in dataframe with NaN values when there is only qfIrgaTurbValiGas01 or no validation at all - if (length(rpt[[idxDate]]$rtioMoleDryCo2Vali$mean) <= 1){ - if(length(rpt[[idxDate]]$rtioMoleDryCo2Vali$mean) == 1 & rpt[[idxDate]]$rtioMoleDryCo2Vali$gasType[1] == "qfIrgaTurbValiGas01"){ - rpt[[idxDate]]$rtioMoleDryCo2Vali[2:5,] <- NA - rpt[[idxDate]]$rtioMoleDryCo2Vali$numSamp <- NaN - rpt[[idxDate]]$rtioMoleDryCo2Vali$timeBgn[2:5] <- base::as.POSIXlt(paste(idxDate, " ", "00:00:00.000", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC") - rpt[[idxDate]]$rtioMoleDryCo2Vali$timeEnd[2:5] <- base::as.POSIXlt(paste(idxDate, " ", "23:59:59.950", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC") + if (length(rpt[[idxDate]][[valiTmp]]$mean) <= 1){ + if(length(rpt[[idxDate]][[valiTmp]]$mean) == 1 & rpt[[idxDate]][[valiTmp]]$gasType[1] == "qfIrgaTurbValiGas01"){ + rpt[[idxDate]][[valiTmp]][2:5,] <- NA + rpt[[idxDate]][[valiTmp]]$numSamp <- NaN + rpt[[idxDate]][[valiTmp]]$timeBgn[2:5] <- base::as.POSIXlt(paste(idxDate, " ", "00:00:00.000", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC") + rpt[[idxDate]][[valiTmp]]$timeEnd[2:5] <- base::as.POSIXlt(paste(idxDate, " ", "23:59:59.950", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC") #replace gasType - rpt[[idxDate]]$rtioMoleDryCo2Vali$gasType <- nameQf + rpt[[idxDate]][[valiTmp]]$gasType <- nameQf }else{ - if(length(rpt[[idxDate]]$rtioMoleDryCo2Vali$mean) == 0){ - rpt[[idxDate]]$rtioMoleDryCo2Vali[1:5,] <- NA - rpt[[idxDate]]$rtioMoleDryCo2Vali$numSamp <- NaN - rpt[[idxDate]]$rtioMoleDryCo2Vali$timeBgn <- base::as.POSIXlt(paste(idxDate, " ", "00:00:00.000", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC") - rpt[[idxDate]]$rtioMoleDryCo2Vali$timeEnd <- base::as.POSIXlt(paste(idxDate, " ", "23:59:59.950", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC") + if(length(rpt[[idxDate]][[valiTmp]]$mean) == 0){ + rpt[[idxDate]][[valiTmp]][1:5,] <- NA + rpt[[idxDate]][[valiTmp]]$numSamp <- NaN + rpt[[idxDate]][[valiTmp]]$timeBgn <- base::as.POSIXlt(paste(idxDate, " ", "00:00:00.000", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC") + rpt[[idxDate]][[valiTmp]]$timeEnd <- base::as.POSIXlt(paste(idxDate, " ", "23:59:59.950", sep=""), format="%Y-%m-%d %H:%M:%OS", tz="UTC") #replace gasType - rpt[[idxDate]]$rtioMoleDryCo2Vali$gasType <- nameQf + rpt[[idxDate]][[valiTmp]]$gasType <- nameQf }else{ - rpt[[idxDate]]$rtioMoleDryCo2Vali <- rpt[[idxDate]]$rtioMoleDryCo2Vali + rpt[[idxDate]][[valiTmp]] <- rpt[[idxDate]][[valiTmp]] } } } #add gasRefe values into rpt + if (idxVar == "rtioMoleDryCo2") { #create temporary dataframe tmpGasRefe <- data.frame(matrix(ncol = 3, nrow = 5)) #assign column name @@ -404,43 +441,72 @@ wrap.irga.vali <- function( tmpCoef[[idxDate]][[idxData]] <- data.frame(matrix(ncol = 3, nrow = 2)) #assign column name colnames(tmpCoef[[idxDate]][[idxData]]) <- c("coef", "se", "scal") - - if (length(valiData[[idxDate]][[idxData]]$mean) < 4 | - sum(is.na(valiData[[idxDate]][[idxData]]$mean)) > 0 | sum(is.na(valiData[[idxDate]][[idxData]]$se)) >0 | - sum(is.na(valiData[[idxDate]][[idxData]]$rtioMoleDryCo2Refe)) > 0 | sum(is.na(valiData[[idxDate]][[idxData]]$rtioMoleDryCo2RefeSe)) > 1){ - tmpCoef[[idxDate]][[idxData]][,] <- NA - } else{ - #x are sensor readings; y are reference gas values - rtioMoleDryCo2Mlfr <- deming::deming(rtioMoleDryCo2Refe[1:4] ~ mean[1:4], data = valiData[[idxDate]][[idxData]], - xstd = se[1:4], ystd = rtioMoleDryCo2RefeSe[1:4]) - #write output to table - #intercept - tmpCoef[[idxDate]][[idxData]][1,1] <- rtioMoleDryCo2Mlfr$coefficients[[1]] - #slope - tmpCoef[[idxDate]][[idxData]][2,1] <- rtioMoleDryCo2Mlfr$coefficients[[2]] - #se - tmpCoef[[idxDate]][[idxData]][,2] <- sqrt(diag(rtioMoleDryCo2Mlfr$variance)) - #scale - tmpCoef[[idxDate]][[idxData]][1,3] <- rtioMoleDryCo2Mlfr$sigma + + #get the temporary valiData table without NA + tmpValiData <- na.omit(valiData[[idxDate]][[idxData]]) + #report NA for regression coefficients if input validation data less than 2 values + if (nrow(tmpValiData) < 2){ + tmpCoef[[idxDate]][[idxData]][,] <- NA + } + + #do simple linear regression when there are only 2 input data + if (nrow(tmpValiData) == 2){ + rtioMoleDryCo2Mlfr <- stats::lm(rtioMoleDryCo2Refe ~ mean, data = tmpValiData) + #write output to table + #intercept + tmpCoef[[idxDate]][[idxData]][1,1] <- rtioMoleDryCo2Mlfr$coefficients[[1]] + #slope + tmpCoef[[idxDate]][[idxData]][2,1] <- rtioMoleDryCo2Mlfr$coefficients[[2]] + #se + tmpCoef[[idxDate]][[idxData]][,2] <- NA + #scale + tmpCoef[[idxDate]][[idxData]][1,3] <- NA + } + + #do MLFR if more than 2 input data avaliable + if (nrow(tmpValiData) > 2){ + #x are sensor readings; y are reference gas values + rtioMoleDryCo2Mlfr <- deming::deming(rtioMoleDryCo2Refe[1:nrow(tmpValiData)] ~ mean[1:nrow(tmpValiData)], data = tmpValiData, + xstd = se[1:nrow(tmpValiData)], ystd = rtioMoleDryCo2RefeSe[1:nrow(tmpValiData)]) + #write output to table + #intercept + tmpCoef[[idxDate]][[idxData]][1,1] <- rtioMoleDryCo2Mlfr$coefficients[[1]] + #slope + tmpCoef[[idxDate]][[idxData]][2,1] <- rtioMoleDryCo2Mlfr$coefficients[[2]] + #se + tmpCoef[[idxDate]][[idxData]][,2] <- sqrt(diag(rtioMoleDryCo2Mlfr$variance)) + #scale + tmpCoef[[idxDate]][[idxData]][1,3] <- rtioMoleDryCo2Mlfr$sigma } }#end of for loop of idxData - #report output rpt[[idxDate]]$rtioMoleDryCo2Mlf <- tmpCoef[[idxDate]]$data00 + #close if idxVar == rtioMoleDryCo2 + }else{ + rpt[[idxDate]]$rtioMoleDryH2oVali$rtioMoleDryH2oRefe <- ifelse(rpt[[idxDate]]$rtioMoleDryH2oVali$gasType == "qfIrgaTurbValiGas02", 0, NA) + } #reorder column - rpt[[idxDate]]$rtioMoleDryCo2Vali <- rpt[[idxDate]]$rtioMoleDryCo2Vali[,c(1:5, 10, 7, 8)] + rpt[[idxDate]][[valiTmp]] <- rpt[[idxDate]][[valiTmp]][,c(1:5, 10, 7, 8)] + #unit attributes - unitRtioMoleDryCo2Vali <- attributes(data$irgaTurb$rtioMoleDryCo2)$unit + unitVali <- attributes(data$irgaTurb[[idxVar]])$unit + #unit attributes for gasRefe + if (idxVar == "rtioMoleDryCo2") { + unitRefe <- attributes(gasRefe$rtioMoleDryCo2Refe01[[idxDate]]$`702_000`)$unit #"rtioMoleDryCo2Refe" + } else{ + unitRefe <- "molH2o mol-1" + } - attributes(rpt[[idxDate]]$rtioMoleDryCo2Vali)$unit <- c(unitRtioMoleDryCo2Vali, #"mean" - unitRtioMoleDryCo2Vali, #"min" - unitRtioMoleDryCo2Vali, #"max" - unitRtioMoleDryCo2Vali,#"vari" - "NA", #"numSamp" - attributes(gasRefe$rtioMoleDryCo2Refe01[[idxDate]]$`702_000`)$unit,#"rtioMoleDryCo2Refe" - "NA", #"timeBgn" - "NA")#"timeEnd" + attributes(rpt[[idxDate]][[valiTmp]])$unit <- c(unitVali, #"mean" + unitVali, #"min" + unitVali, #"max" + unitVali,#"vari" + "NA", #"numSamp" + unitRefe,#gasRefe + "NA", #"timeBgn" + "NA")#"timeEnd" + }#end of for loop of idxVar }; rm(valiCrit, allSubData, allSubQfqm, allSubTime)#end of idxDate invisible(gc()) @@ -456,8 +522,80 @@ wrap.irga.vali <- function( #applying the calculated coefficients to measured data #Calculate time-series (20Hz) of slope and zero offset - rpt[[DateProc]]$rtioMoleDryCo2Cor <- eddy4R.base::def.irga.vali.cor(data = data, DateProc = DateProc, coef = tmpCoef, valiData = valiData, valiCrit = valiCrit, ScalMax = 20, FracSlpMax = 0.1, Freq = 20) + rpt[[DateProc]]$rtioMoleDryCo2Cor <- eddy4R.base::def.irga.vali.cor(data = data, DateProc = DateProc, coef = tmpCoef, valiData = valiData, valiCrit = valiCrit, ScalMax = ScalMax, FracSlp = FracSlp, OfstMax = OfstMax, Freq = 20) + + #run the benchmarking regression to determine if the validation was good + valiEval <- eddy4R.base::def.irga.vali.thsh(data = rpt[[DateProc]], DateProc = DateProc, evalSlpMax = 1.05, evalSlpMin = 0.95, evalOfstMax = 100, evalOfstMin = -100) + + #remove corrected data if validation fails benchmarking test + if (valiEval$valiEvalPass == FALSE){ + rpt[[DateProc]]$rtioMoleDryCo2Cor$rtioMoleDryCo2Cor <- NaN #data are removed if the validation does not pass the thresholds set for evaluation slope and offset + #raise quality flag in validation table to indicate validation status + rpt[[DateProc]]$rtioMoleDryCo2Mlf$qfEvalThsh <- c(NA, 1) + msg <- paste0("validation did not pass evaluation threshold, corrected data were set to NaN") + tryCatch({rlog$debug(msg)}, error=function(cond){print(msg)}) + } else if (valiEval$valiEvalPass == TRUE) { + rpt[[DateProc]]$rtioMoleDryCo2Mlf$qfEvalThsh <- c(NA, 0) #corrected data will be included in the processed file in this case + } else { + rpt[[DateProc]]$rtioMoleDryCo2Mlf$qfEvalThsh <- c(NA, -1) + rpt[[DateProc]]$rtioMoleDryCo2Cor$rtioMoleDryCo2Cor <- NaN #also remove data in the -1 missing validation case, prevents unexpected inclusion of questionable validations and also removes data if the eval regression can't run due to lack of span gasses + } + + #force qfValiEval to -1 if slope is outside the threshold because this validation can't be applied + + if(!is.na(rpt[[DateProc]]$rtioMoleDryCo2Mlf$coef[2])){ # only run if there are coefficients to check + if (rpt[[DateProc]]$rtioMoleDryCo2Mlf$coef[2] < base::min(FracSlp) | rpt[[DateProc]]$rtioMoleDryCo2Mlf$coef[2] > base::max(FracSlp)){ + rpt[[DateProc]]$rtioMoleDryCo2Mlf$qfEvalThsh <- c(NA, -1) + } + } + + + #add additional coefficients to mlf table + rpt[[DateProc]]$rtioMoleDryCo2Mlf$evalCoef <- valiEval$evalCoef + rpt[[DateProc]]$rtioMoleDryCo2Mlf$evalCoefSe <- valiEval$evalCoefSe + rpt[[DateProc]]$rtioMoleDryCo2Mlf$evalSlpThsh <- valiEval$evalSlpThsh + rpt[[DateProc]]$rtioMoleDryCo2Mlf$evalOfstThsh <- valiEval$evalOfstThsh + + + #add corrected reference gas values to vali table + + if(base::nrow(rpt[[DateProc]]$rtioMoleDryCo2Vali) == base::length(valiEval$meanCor)+1){ # failsafe for row mismatches, valiEval$meanCor will always be one short because the archive gas is not included + + rpt[[DateProc]]$rtioMoleDryCo2Vali$meanCor <- c(NaN, valiEval$meanCor) # need to add the NaN to account for the archive gas in the first position of the vali table + + #reorder to place the corrected reference values next to the original reference values + rpt[[DateProc]]$rtioMoleDryCo2Vali <- rpt[[DateProc]]$rtioMoleDryCo2Vali[c("mean", "min", "max", "vari", "numSamp", "rtioMoleDryCo2Refe", "meanCor", "timeBgn", "timeEnd")] + + #rename rtioMoleDryCo2Refe to refe, this could be implemented in the rest of the functions in the future + names(rpt[[DateProc]]$rtioMoleDryCo2Vali) <- c("mean", "min", "max", "vari", "numSamp", "refe", "meanCor", "timeBgn", "timeEnd") + } else { + #fill meanCor with NaN if there were extra validation gas rows + rpt[[DateProc]]$rtioMoleDryCo2Vali$meanCor <- NaN + + #reorder to place the corrected reference values next to the original reference values + rpt[[DateProc]]$rtioMoleDryCo2Vali <- rpt[[DateProc]]$rtioMoleDryCo2Vali[c("mean", "min", "max", "vari", "numSamp", "rtioMoleDryCo2Refe", "meanCor", "timeBgn", "timeEnd")] + + names(rpt[[DateProc]]$rtioMoleDryCo2Vali) <- c("mean", "min", "max", "vari", "numSamp", "refe", "meanCor", "timeBgn", "timeEnd") + } + + #rename rtioMoleDryH2oRefe to refe to match CO2 + names(rpt[[DateProc]]$rtioMoleDryH2oVali) <- c("mean", "min", "max", "vari", "numSamp", "refe", "timeBgn", "timeEnd") + + #reset attributes + + attributes(rpt[[DateProc]]$rtioMoleDryCo2Vali)$unit <- c("molCo2 mol-1Dry", #"mean" + "molCo2 mol-1Dry", #"min" + "molCo2 mol-1Dry", #"max" + "molCo2 mol-1Dry",#"vari" + "NA", #"numSamp" + "molCo2 mol-1Dry",#gasRefe + "molCo2 mol-1Dry",#gasRefeCor + "NA", #"timeBgn" + "NA")#"timeEnd" + + attributes(rpt[[DateProc]]$rtioMoleDryCo2Cor$rtioMoleDryCo2Cor)$unit <- "molCo2 mol-1Dry" + #return results return(rpt) } diff --git a/pack/eddy4R.base/R/wrap.unit.conv.out.ec.R b/pack/eddy4R.base/R/wrap.unit.conv.out.ec.R index 763e0df6..57eb87c6 100644 --- a/pack/eddy4R.base/R/wrap.unit.conv.out.ec.R +++ b/pack/eddy4R.base/R/wrap.unit.conv.out.ec.R @@ -38,6 +38,15 @@ # adding units conversion for ecse # Natchaya P-Durden (2019-07-23) # converting time format in ecse qfqm +# Natchaya P-Durden (2020-01-17) +# adding unit conversion for rtioMoleDryH2o during validation + +# Chris Florian (2021-06-22) +# adding CH4 unit conversion +# Chris Florian (2021-08-05) +# adding units for rtioMoleDryCo2RefeCor +# Chris Florian (2021-08-18) +# updating naming convention, removing redundant rtioMoleDryCo2 from validation table ############################################################################################ wrap.unit.conv.out.ec <- function( @@ -83,6 +92,8 @@ outAttr$co2Turb <- base::list( outAttr$h2oTurb <- base::list( "rtioMoleDryH2o"= c("mean" = "mmolH2o mol-1Dry", "min" = "mmolH2o mol-1Dry", "max" = "mmolH2o mol-1Dry", "vari" = "mmolH2o mol-1Dry", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA"), + "rtioMoleDryH2oCor"= c("mean" = "mmolH2o mol-1Dry", "min" = "mmolH2o mol-1Dry", "max" = "mmolH2o mol-1Dry", "vari" = "mmolH2o mol-1Dry", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA"), + "rtioMoleDryH2oRaw"= c("mean" = "mmolH2o mol-1Dry", "min" = "mmolH2o mol-1Dry", "max" = "mmolH2o mol-1Dry", "vari" = "mmolH2o mol-1Dry", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA"), "densMoleH2o"= c("mean" = "mmolH2o m-3", "min" = "mmolH2o m-3", "max" = "mmolH2o m-3", "vari" = "mmolH2o m-3", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA"), "tempDew"= c("mean" = "C", "min" = "C", "max" = "C", "vari" = "C", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA"), "presAtm"= c("mean" = "kPa", "min" = "kPa", "max" = "kPa", "vari" = "kPa", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA"), @@ -118,6 +129,8 @@ if(MethType == "ucrt"){ outAttr$h2oTurb <- base::list( "rtioMoleDryH2o"= c("mean" = "mmolH2o mol-1Dry", "vari" = "mmolH2o mol-1Dry", "se" = "mmolH2o mol-1Dry", "timeBgn" = "NA", "timeEnd" = "NA"), + "rtioMoleDryH2oCor"= c("mean" = "mmolH2o mol-1Dry", "vari" = "mmolH2o mol-1Dry", "se" = "mmolH2o mol-1Dry", "timeBgn" = "NA", "timeEnd" = "NA"), + "rtioMoleDryH2oRaw"= c("mean" = "mmolH2o mol-1Dry", "vari" = "mmolH2o mol-1Dry", "se" = "mmolH2o mol-1Dry", "timeBgn" = "NA", "timeEnd" = "NA"), "densMoleH2o"= c("mean" = "mmolH2o m-3", "vari" = "mmolH2o m-3", "se" = "mmolH2o m-3", "timeBgn" = "NA", "timeEnd" = "NA"), "tempDew"= c("mean" = "C", "vari" = "C", "se" = "C", "timeBgn" = "NA", "timeEnd" = "NA"), "presAtm"= c("mean" = "kPa", "vari" = "kPa", "se" = "kPa", "timeBgn" = "NA", "timeEnd" = "NA"), @@ -128,7 +141,9 @@ if(MethType == "ucrt"){ if(MethType == "vali"){ outAttr$co2Turb <- base::list( - "rtioMoleDryCo2Vali"= c("mean" = "umolCo2 mol-1Dry", "min" = "umolCo2 mol-1Dry", "max" = "umolCo2 mol-1Dry", "vari" = "umolCo2 mol-1Dry", "numSamp" = "NA", "rtioMoleDryCo2Refe" = "umolCo2 mol-1Dry", "timeBgn" = "NA", "timeEnd" = "NA")) + "rtioMoleDryCo2Vali"= c("mean" = "umolCo2 mol-1Dry", "min" = "umolCo2 mol-1Dry", "max" = "umolCo2 mol-1Dry", "vari" = "umolCo2 mol-1Dry", "numSamp" = "NA", "refe" = "umolCo2 mol-1Dry", "meanCor" = "umolCo2 mol-1Dry", "timeBgn" = "NA", "timeEnd" = "NA")) + outAttr$h2oTurb <- base::list( + "rtioMoleDryH2oVali"= c("mean" = "mmolH2o mol-1Dry", "min" = "mmolH2o mol-1Dry", "max" = "mmolH2o mol-1Dry", "vari" = "mmolH2o mol-1Dry", "numSamp" = "NA", "refe" = "mmolH2o mol-1Dry", "timeBgn" = "NA", "timeEnd" = "NA")) } #assign output attributes @@ -153,6 +168,9 @@ outAttr$vari <- base::list( "frt00Samp"= "dm6 min-2", "tempAve"= "C2", "rtioMoleDryH2o"= "mmol2H2o mol-2Dry", + "rtioMoleDryH2oCor"= "mmol2H2o mol-2Dry", + "rtioMoleDryH2oRaw"= "mmol2H2o mol-2Dry", + "rtioMoleDryH2oVali"= "mmol2H2o mol-2Dry", "densMoleH2o"= "mmol2H2o m-6", "tempDew"= "C2" ) @@ -198,9 +216,9 @@ for(idxDp in base::names(rpt)){ if(MethType == "vali"){ #refeName <- names(outAttr[[idxDp]][[idxVar]][6]) - nameAttr <- c("mean", "min", "max", "vari", "numSamp" , names(outAttr[[idxDp]][[idxVar]][6]), "timeBgn", "timeEnd" ) + nameAttr <- c("mean", "min", "max", "vari", "numSamp" , names(outAttr[[idxDp]][[idxVar]][6]), names(outAttr[[idxDp]][[idxVar]][7]), "timeBgn", "timeEnd" ) #wrkAttr[[idxDp]][[idxVar]] <- c("mean"= baseAttr, "min" = baseAttr, "max" = baseAttr, "vari" = baseAttr, "numSamp" = baseAttr, refeName = baseAttr, "timeBgn" = "NA", "timeEnd" = "NA") - wrkAttr[[idxDp]][[idxVar]] <- c(baseAttr, baseAttr, baseAttr, baseAttr, baseAttr, baseAttr, "NA", "NA") + wrkAttr[[idxDp]][[idxVar]] <- c(baseAttr, baseAttr, baseAttr, baseAttr, "NA", baseAttr, baseAttr, "NA", "NA") names(wrkAttr[[idxDp]][[idxVar]]) <- nameAttr } @@ -230,6 +248,9 @@ if(MethMeas == "ecse"){ "rtioMoleDryCo2" = "umol2Co2 mol-2", "rtioMoleWetCo2" = "umol2Co2 mol-2", "rtioMoleDryCo2Refe" = "umol2Co2 mol-2", + "rtioMoleDryCh4" = "umol2Ch4 mol-2", + "rtioMoleWetCh4" = "umol2Ch4 mol-2", + "rtioMoleDryCh4Refe" = "umol2Ch4 mol-2", "temp" = "C2", "tempEnvHut" = "C2", "rtioMoleDryH2o" = "mmol2H2o mol-2", @@ -256,6 +277,9 @@ if(MethMeas == "ecse"){ wrkAttr$rtioMoleDryCo2 <- c("mean" = "molCo2 mol-1", "min" = "molCo2 mol-1", "max" = "molCo2 mol-1", "vari" = "molCo2 mol-1", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") wrkAttr$rtioMoleWetCo2 <- c("mean" = "molCo2 mol-1", "min" = "molCo2 mol-1", "max" = "molCo2 mol-1", "vari" = "molCo2 mol-1", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") wrkAttr$rtioMoleDryCo2Refe <- c("mean" = "molCo2 mol-1", "min" = "molCo2 mol-1", "max" = "molCo2 mol-1", "vari" = "molCo2 mol-1", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") + wrkAttr$rtioMoleDryCh4 <- c("mean" = "molCh4 mol-1", "min" = "molCh4 mol-1", "max" = "molCh4 mol-1", "vari" = "molCh4 mol-1", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") + wrkAttr$rtioMoleWetCh4 <- c("mean" = "molCh4 mol-1", "min" = "molCh4 mol-1", "max" = "molCh4 mol-1", "vari" = "molCh4 mol-1", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") + wrkAttr$rtioMoleDryCh4Refe <- c("mean" = "molCh4 mol-1", "min" = "molCh4 mol-1", "max" = "molCh4 mol-1", "vari" = "molCh4 mol-1", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") wrkAttr$temp <- c("mean" = "K", "min" = "K", "max" = "K", "vari" = "C", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA")#with means removed unit of wrk vari of temp is equal to out wrkAttr$tempEnvHut <- c("mean" = "K", "min" = "K", "max" = "K", "vari" = "C", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") wrkAttr$rtioMoleDryH2o <- c("mean" = "molH2o mol-1", "min" = "molH2o mol-1", "max" = "molH2o mol-1", "vari" = "molH2o mol-1", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") @@ -293,6 +317,9 @@ if(MethMeas == "ecse"){ outAttr$rtioMoleDryCo2 <- c("mean" = "umolCo2 mol-1", "min" = "umolCo2 mol-1", "max" = "umolCo2 mol-1", "vari" = "umolCo2 mol-1", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") outAttr$rtioMoleWetCo2 <- c("mean" = "umolCo2 mol-1", "min" = "umolCo2 mol-1", "max" = "umolCo2 mol-1", "vari" = "umolCo2 mol-1", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") outAttr$rtioMoleDryCo2Refe <- c("mean" = "umolCo2 mol-1", "min" = "umolCo2 mol-1", "max" = "umolCo2 mol-1", "vari" = "umolCo2 mol-1", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") + outAttr$rtioMoleDryCh4 <- c("mean" = "umolCh4 mol-1", "min" = "umolCh4 mol-1", "max" = "umolCh4 mol-1", "vari" = "umolCh4 mol-1", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") + outAttr$rtioMoleWetCh4 <- c("mean" = "umolCh4 mol-1", "min" = "umolCh4 mol-1", "max" = "umolCh4 mol-1", "vari" = "umolCh4 mol-1", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") + outAttr$rtioMoleDryCh4Refe <- c("mean" = "umolCh4 mol-1", "min" = "umolCh4 mol-1", "max" = "umolCh4 mol-1", "vari" = "umolCh4 mol-1", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") outAttr$temp <- c("mean" = "C", "min" = "C", "max" = "C", "vari" = "C", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") outAttr$tempEnvHut <- c("mean" = "C", "min" = "C", "max" = "C", "vari" = "C", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") outAttr$rtioMoleDryH2o <- c("mean" = "mmolH2o mol-1", "min" = "mmolH2o mol-1", "max" = "mmolH2o mol-1", "vari" = "mmolH2o mol-1", "numSamp" = "NA", "timeBgn" = "NA", "timeEnd" = "NA") @@ -515,6 +542,9 @@ if(MethMeas == "ecse"){ wrkAttr$rhEnvHut <- c("mean" = "-", "vari" = "-", "se" = "-", "timeBgn" = "NA", "timeEnd" = "NA") wrkAttr$rtioMoleDryCo2 <- c("mean" = "molCo2 mol-1", "vari" = "molCo2 mol-1", "se" = "molCo2 mol-1", "timeBgn" = "NA", "timeEnd" = "NA") wrkAttr$rtioMoleWetCo2 <- c("mean" = "molCo2 mol-1", "vari" = "molCo2 mol-1", "se" = "molCo2 mol-1", "timeBgn" = "NA", "timeEnd" = "NA") + wrkAttr$rtioMoleDryCh4 <- c("mean" = "molCh4 mol-1", "vari" = "molCh4 mol-1", "se" = "molCh4 mol-1", "timeBgn" = "NA", "timeEnd" = "NA") + wrkAttr$rtioMoleWetCh4 <- c("mean" = "molCh4 mol-1", "vari" = "molCh4 mol-1", "se" = "molCh4 mol-1", "timeBgn" = "NA", "timeEnd" = "NA") + wrkAttr$rtioMoleDryCh4Refe <- c("mean" = "molCh4 mol-1", "vari" = "molCh4 mol-1", "se" = "molCh4 mol-1", "timeBgn" = "NA", "timeEnd" = "NA") wrkAttr$temp <- c("mean" = "C", "vari" = "C", "se" = "C", "NA", "NA")#with means removed unit of wrk mean, vari, and se of temp is equal to out wrkAttr$tempEnvHut <- c("mean" = "C", "vari" = "C", "se" = "C", "timeBgn" = "NA", "timeEnd" = "NA") wrkAttr$rtioMoleDryH2o <- c("mean" = "molH2o mol-1", "vari" = "molH2o mol-1", "se" = "molH2o mol-1", "timeBgn" = "NA", "timeEnd" = "NA") @@ -536,6 +566,9 @@ if(MethMeas == "ecse"){ outAttr$rhEnvHut <- c("mean" = "%", "vari" = "%", "se" = "%", "timeBgn" = "NA", "timeEnd" = "NA") outAttr$rtioMoleDryCo2 <- c("mean" = "umolCo2 mol-1", "vari" = "umolCo2 mol-1", "se" = "umolCo2 mol-1", "timeBgn" = "NA", "timeEnd" = "NA") outAttr$rtioMoleWetCo2 <- c("mean" = "umolCo2 mol-1", "vari" = "umolCo2 mol-1", "se" = "umolCo2 mol-1", "timeBgn" = "NA", "timeEnd" = "NA") + outAttr$rtioMoleDryCh4 <- c("mean" = "umolCh4 mol-1", "vari" = "umolCh4 mol-1", "se" = "umolCh4 mol-1", "timeBgn" = "NA", "timeEnd" = "NA") + outAttr$rtioMoleWetCh4 <- c("mean" = "umolCh4 mol-1", "vari" = "umolCh4 mol-1", "se" = "umolCh4 mol-1", "timeBgn" = "NA", "timeEnd" = "NA") + outAttr$rtioMoleDryCh4Refe <- c("mean" = "umolCh4 mol-1", "vari" = "umolCh4 mol-1", "se" = "umolCh4 mol-1", "timeBgn" = "NA", "timeEnd" = "NA") outAttr$temp <- c("mean" = "C", "vari" = "C", "se" = "C", "timeBgn" = "NA", "timeEnd" = "NA") outAttr$tempEnvHut <- c("mean" = "C", "vari" = "C", "se" = "C", "timeBgn" = "NA", "timeEnd" = "NA") outAttr$rtioMoleDryH2o <- c("mean" = "mmolH2o mol-1", "vari" = "mmolH2o mol-1", "se" = "mmolH2o mol-1", "timeBgn" = "NA", "timeEnd" = "NA") diff --git a/pack/eddy4R.base/man/IntlConv.Rd b/pack/eddy4R.base/man/IntlConv.Rd index 4cf28d0f..c1c7525e 100644 --- a/pack/eddy4R.base/man/IntlConv.Rd +++ b/pack/eddy4R.base/man/IntlConv.Rd @@ -4,7 +4,8 @@ \name{IntlConv} \alias{IntlConv} \title{Internal data: Parameter set: Conversion Factors} -\format{A named list of numerical polynomial conversion coefficients. Names are constructed from +\format{ +A named list of numerical polynomial conversion coefficients. Names are constructed from the quantities to be converted. The quantity to be converted from is named first, followed by the quantity to be converted to.\cr \cr @@ -167,7 +168,8 @@ quantity to be converted to.\cr \describe{ \item{NewtPndf}{force conversion, Newton to pound-force = c(0,0.224809) [lbf N-1]} \item{PndfNewt}{force conversion, pound-force to Newton = c(0,4.44822) [N lbf-1]} -}} +} +} \source{ Conversion factors are defined within flow.save.intl.cnst.R, available in the data-raw/ folder of the source version of the package diff --git a/pack/eddy4R.base/man/IntlNatu.Rd b/pack/eddy4R.base/man/IntlNatu.Rd index c6aff837..3bb7f0f7 100644 --- a/pack/eddy4R.base/man/IntlNatu.Rd +++ b/pack/eddy4R.base/man/IntlNatu.Rd @@ -4,7 +4,8 @@ \name{IntlNatu} \alias{IntlNatu} \title{Internal data: Parameter set: Natural constants} -\format{A list of named constants: +\format{ +A list of named constants: \describe{ \item{Grav}{acceleration of gravity = 9.81 [m s-2]} \item{PrdErth}{rotation period of the Earth (one sidereal day) = 86164.1 [s]} @@ -30,7 +31,8 @@ \item{RsH2o}{specific gas constant for water vapour = 461.96 [J kg-1 K-1]} \item{GmmaH2o}{CpH2o / CvH2o = 1.333776 [-]} \item{KppaH2o}{water vapour Kappa exponent for ideal gas law (Poisson), RsH2o / CpH2o = 0.2502 [-]} -}} +} +} \source{ Natural constants are defined within flow.save.intl.cnst.R, available in the data-raw/ folder of the source version of the package diff --git a/pack/eddy4R.base/man/IntlUnit.Rd b/pack/eddy4R.base/man/IntlUnit.Rd index f05ed10a..5468f046 100644 --- a/pack/eddy4R.base/man/IntlUnit.Rd +++ b/pack/eddy4R.base/man/IntlUnit.Rd @@ -4,7 +4,8 @@ \name{IntlUnit} \alias{IntlUnit} \title{Internal data: Parameter set: Unit representations} -\format{A nested list of named unit categories: +\format{ +A nested list of named unit categories: \describe{ \item{Base}{ \describe{ @@ -127,7 +128,8 @@ \code{Forc = "N"}\cr \code{Freq = "Hz"}\cr } -}} +} +} \source{ Units are defined within flow.save.intl.cnst.R, available in the data-raw/ folder of the source version of the package diff --git a/pack/eddy4R.base/man/def.bin.Rd b/pack/eddy4R.base/man/def.bin.Rd index ce2416db..03064f7f 100644 --- a/pack/eddy4R.base/man/def.bin.Rd +++ b/pack/eddy4R.base/man/def.bin.Rd @@ -4,8 +4,14 @@ \alias{def.bin} \title{Definition function: Binning data} \usage{ -def.bin(idep, depe, RngMinMax = NULL, NumBin, widtBin = c("lin", - "log10", "exp10", "logExp", "expLog"), meanFunc = c("mean", "median")) +def.bin( + idep, + depe, + RngMinMax = NULL, + NumBin, + widtBin = c("lin", "log10", "exp10", "logExp", "expLog"), + meanFunc = c("mean", "median") +) } \arguments{ \item{idep}{Either a vector or matrix of class numeric or integer containing the independent variable and of the same length as \code{depe}. []} diff --git a/pack/eddy4R.base/man/def.dens.pres.pois.Rd b/pack/eddy4R.base/man/def.dens.pres.pois.Rd index 9a34b4d9..f18e6743 100644 --- a/pack/eddy4R.base/man/def.dens.pres.pois.Rd +++ b/pack/eddy4R.base/man/def.dens.pres.pois.Rd @@ -4,8 +4,12 @@ \alias{def.dens.pres.pois} \title{Definition function: Poisson's equation (adiabatic change) - density as function of pressure change} \usage{ -def.dens.pres.pois(dens01, pres01, pres02, - Kppa = eddy4R.base::IntlNatu$KppaDry) +def.dens.pres.pois( + dens01, + pres01, + pres02, + Kppa = eddy4R.base::IntlNatu$KppaDry +) } \arguments{ \item{dens01}{Measured density, Amount per volume [same unit as returned density, e.g. kg/m3 or kmol/m3].} diff --git a/pack/eddy4R.base/man/def.dens.temp.pois.Rd b/pack/eddy4R.base/man/def.dens.temp.pois.Rd index f5bf0ac7..6e5cb603 100644 --- a/pack/eddy4R.base/man/def.dens.temp.pois.Rd +++ b/pack/eddy4R.base/man/def.dens.temp.pois.Rd @@ -4,8 +4,12 @@ \alias{def.dens.temp.pois} \title{Definition function: Poisson's equation (adiabatic change) - density as function of temperature change} \usage{ -def.dens.temp.pois(dens01, temp01, temp02, - Kppa = eddy4R.base::IntlNatu$KppaDry) +def.dens.temp.pois( + dens01, + temp01, + temp02, + Kppa = eddy4R.base::IntlNatu$KppaDry +) } \arguments{ \item{dens01}{Measured density, Amount per volume [same unit as returned density, e.g. kg/m3 or kmol/m3].} diff --git a/pack/eddy4R.base/man/def.dp01.agr.ecte.Rd b/pack/eddy4R.base/man/def.dp01.agr.ecte.Rd index 2bf7fe84..20aee572 100644 --- a/pack/eddy4R.base/man/def.dp01.agr.ecte.Rd +++ b/pack/eddy4R.base/man/def.dp01.agr.ecte.Rd @@ -4,8 +4,12 @@ \alias{def.dp01.agr.ecte} \title{Definition function: aggregation of ecte dp01 outputs} \usage{ -def.dp01.agr.ecte(inpList, MethSubAgr = FALSE, MethUcrt = FALSE, - RptExpd = FALSE) +def.dp01.agr.ecte( + inpList, + MethSubAgr = FALSE, + MethUcrt = FALSE, + RptExpd = FALSE +) } \arguments{ \item{inpList}{a list of dp01 computed output statistics and dp01 quality flags and quality metrics over multiple aggregations periods that need to be combined.} diff --git a/pack/eddy4R.base/man/def.hdf5.crte.Rd b/pack/eddy4R.base/man/def.hdf5.crte.Rd index 8a6dc07a..f80cb693 100644 --- a/pack/eddy4R.base/man/def.hdf5.crte.Rd +++ b/pack/eddy4R.base/man/def.hdf5.crte.Rd @@ -4,10 +4,18 @@ \alias{def.hdf5.crte} \title{Definition function: Create the ECTE HDF5 file structure} \usage{ -def.hdf5.crte(Date, Site = "SERC", LvlTowr, DirOut, FileOutBase = NULL, - MethExpd = TRUE, MethDp04 = FALSE, FileNameReadMe = NULL, - FileNameObjDesc = NULL, LvlGasRefe = c("702_000", "703_000", - "704_000", "705_000", "706_000")) +def.hdf5.crte( + Date, + Site = "SERC", + LvlTowr, + DirOut, + FileOutBase = NULL, + MethExpd = TRUE, + MethDp04 = FALSE, + FileNameReadMe = NULL, + FileNameObjDesc = NULL, + LvlGasRefe = c("702_000", "703_000", "704_000", "705_000", "706_000") +) } \arguments{ \item{Date}{is the date for the output file being generated.} diff --git a/pack/eddy4R.base/man/def.hdf5.extr.Rd b/pack/eddy4R.base/man/def.hdf5.extr.Rd index f1373953..586f41bd 100644 --- a/pack/eddy4R.base/man/def.hdf5.extr.Rd +++ b/pack/eddy4R.base/man/def.hdf5.extr.Rd @@ -4,9 +4,15 @@ \alias{def.hdf5.extr} \title{Definition function: Extract data and attributes from HDF5 input file to another HDF5 output file} \usage{ -def.hdf5.extr(FileInp = NULL, rpt = NULL, FileOut = NULL, - MethExtrData = TRUE, MethExtrAttr = TRUE, dp01 = NULL, - MethDp0p = FALSE) +def.hdf5.extr( + FileInp = NULL, + rpt = NULL, + FileOut = NULL, + MethExtrData = TRUE, + MethExtrAttr = TRUE, + dp01 = NULL, + MethDp0p = FALSE +) } \arguments{ \item{FileInp}{is the input HDF5 file (turb or stor) the data and metadata are being read from. It is ignored if \code{rpt} is specified.} diff --git a/pack/eddy4R.base/man/def.hdf5.read.qfqm.Rd b/pack/eddy4R.base/man/def.hdf5.read.qfqm.Rd index 4b38aae4..115317f6 100644 --- a/pack/eddy4R.base/man/def.hdf5.read.qfqm.Rd +++ b/pack/eddy4R.base/man/def.hdf5.read.qfqm.Rd @@ -4,9 +4,16 @@ \alias{def.hdf5.read.qfqm} \title{Wrapper function: Reading quality flags from NEON HDF5 files} \usage{ -def.hdf5.read.qfqm(DirInpLoca, SiteLoca, DateLoca, VarLoca, - LvlTowr = c("000_040", "000_050", "000_060")[3], FreqLoca, - MethMeas = c("ecte", "ecse")[1]) +def.hdf5.read.qfqm( + DirInpLoca, + SiteLoca, + DateLoca, + VarLoca, + LvlTowr = c("000_040", "000_050", "000_060")[3], + FreqLoca, + DataType = c("data", "qfqm")[1], + MethMeas = c("ecte", "ecse")[1] +) } \arguments{ \item{DirInpLoca}{Character: Input directory.} @@ -21,10 +28,12 @@ def.hdf5.read.qfqm(DirInpLoca, SiteLoca, DateLoca, VarLoca, \item{FreqLoca}{Integer: Measurement frequency.} +\item{DataType}{Character: Specify between data and qfqm for read in.} + \item{MethMeas}{A vector of class "character" containing the name of measurement method (eddy-covariance turbulent exchange or storage exchange), MethMeas = c("ecte", "ecse"). Defaults to "ecte".} } \value{ -Named list \code{qfqm} containing time-series of quality flags. +Named list \code{rpt} containing time-series of quality flags. } \description{ definition function. Reads an HDF5 input file in NEON standard format from \code{DirInpLoca}. diff --git a/pack/eddy4R.base/man/def.hdf5.wrte.dp01.Rd b/pack/eddy4R.base/man/def.hdf5.wrte.dp01.Rd index 6b8c3904..13e648f6 100644 --- a/pack/eddy4R.base/man/def.hdf5.wrte.dp01.Rd +++ b/pack/eddy4R.base/man/def.hdf5.wrte.dp01.Rd @@ -4,8 +4,15 @@ \alias{def.hdf5.wrte.dp01} \title{Definition function: Write NEON Level 1 data, qfqm, and uncertainty to output HDF5} \usage{ -def.hdf5.wrte.dp01(inpList, FileOut, SiteLoca, LvlTowr, Dp01, - MethUcrt = TRUE, MethSubAgr = TRUE) +def.hdf5.wrte.dp01( + inpList, + FileOut, + SiteLoca, + LvlTowr, + Dp01, + MethUcrt = TRUE, + MethSubAgr = TRUE +) } \arguments{ \item{inpList}{A list of including dp01 data, quality flags and quality metrics, and uncertainty calculations to package and write to an output HDF5 file.} diff --git a/pack/eddy4R.base/man/def.idx.agr.Rd b/pack/eddy4R.base/man/def.idx.agr.Rd index da6387ce..268ac536 100644 --- a/pack/eddy4R.base/man/def.idx.agr.Rd +++ b/pack/eddy4R.base/man/def.idx.agr.Rd @@ -4,8 +4,15 @@ \alias{def.idx.agr} \title{Definition function: indices for aggregation periods} \usage{ -def.idx.agr(time, PrdAgr, FreqLoca, MethIdx = c("rglr", "specBgn", - "specEnd")[1], crdH2oVali = FALSE, data = NULL, CritTime = 0) +def.idx.agr( + time, + PrdAgr, + FreqLoca, + MethIdx = c("rglr", "specBgn", "specEnd")[1], + crdH2oVali = FALSE, + data = NULL, + CritTime = 0 +) } \arguments{ \item{time}{a vector of timestamps} diff --git a/pack/eddy4R.base/man/def.inst.depe.Rd b/pack/eddy4R.base/man/def.inst.depe.Rd index b04b4a38..d57d4a95 100644 --- a/pack/eddy4R.base/man/def.inst.depe.Rd +++ b/pack/eddy4R.base/man/def.inst.depe.Rd @@ -4,8 +4,12 @@ \alias{def.inst.depe} \title{Definition function: Install dependent packages} \usage{ -def.inst.depe(DirPack = ".", Depe = TRUE, - Repo = base::getOption("repos")[1], Lib = .libPaths()[1]) +def.inst.depe( + DirPack = ".", + Depe = TRUE, + Repo = base::getOption("repos")[1], + Lib = .libPaths()[1] +) } \arguments{ \item{DirPack}{Path to the package \code{DESCRIPTION} file, defaults to the current working directory. Character vector with single entry [-].} diff --git a/pack/eddy4R.base/man/def.irga.vali.cor.Rd b/pack/eddy4R.base/man/def.irga.vali.cor.Rd index 83a0fef7..d88f5f4c 100644 --- a/pack/eddy4R.base/man/def.irga.vali.cor.Rd +++ b/pack/eddy4R.base/man/def.irga.vali.cor.Rd @@ -4,8 +4,17 @@ \alias{def.irga.vali.cor} \title{definition function: Calculation of time-series of the correction IRGA sub data products.} \usage{ -def.irga.vali.cor(data, DateProc, coef, valiData, valiCrit = FALSE, - ScalMax = 20, FracSlpMax = 0.1, Freq = 20) +def.irga.vali.cor( + data, + DateProc, + coef, + valiData, + valiCrit = FALSE, + ScalMax = FALSE, + FracSlp = FALSE, + OfstMax = FALSE, + Freq = 20 +) } \arguments{ \item{data}{List consisting of \code{ff::ffdf} file-backed objects containing the dp0p input IRGA.} @@ -16,11 +25,13 @@ def.irga.vali.cor(data, DateProc, coef, valiData, valiCrit = FALSE, \item{valiData}{List consisting of descriptive statistics (mean, min, max, vari, numSamp, se) of CO2 dry mole concentration during performing validation and CO2 dry mole concentration of reference gases.} -\item{valiCrit}{A logical stating if there are more than one validation occurred within DateProc. Defaut to FALSE.} +\item{valiCrit}{A logical stating if there are more than one validation occurred within DateProc.} -\item{ScalMax}{Maximum scale value (resulted from maximum-likelihood fitting of a functional relationship (MLFR)). Defaults to 20.} +\item{ScalMax}{Maximum scale value. The validation correction will not apply if scale (resulted from maximum-likelihood fitting of a functional relationship (MLFR)) is greater than ScalMax or ScalMax = FALSE. Defaults to FALSE.} -\item{FracSlpMax}{Maximum fraction of slope value (resulted from maximum-likelihood fitting of a functional relationship (MLFR)). Defaults to 0.1.} +\item{FracSlp}{Upper and lower bounds of slope values. The validation correction will not apply if slope (resulted from regression fitting) is greater/lower than the FracSlp maximum or minimum value or FracSlp = FALSE. Defaults to FALSE.} + +\item{OfstMax}{Maximum offset value. The validation correction will not apply if slope (resulted from regression fitting) is greater than the OfstMax (unit in mol mol-1) or OfstMax = FALSE. Defaults to FALSE.} \item{Freq}{Measurement frequency. Defaults to 20. [Hz]} } diff --git a/pack/eddy4R.base/man/def.irga.vali.thsh.Rd b/pack/eddy4R.base/man/def.irga.vali.thsh.Rd new file mode 100644 index 00000000..4b765359 --- /dev/null +++ b/pack/eddy4R.base/man/def.irga.vali.thsh.Rd @@ -0,0 +1,53 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/def.irga.vali.thsh.R +\name{def.irga.vali.thsh} +\alias{def.irga.vali.thsh} +\title{Definition function: Threshold IRGA validation data based on benchmarking regression} +\usage{ +def.irga.vali.thsh( + data, + DateProc, + evalSlpMax, + evalSlpMin, + evalOfstMax, + evalOfstMin +) +} +\arguments{ +\item{data}{List of validation data as a report from eddy4R.base::wrap.irga.vali()} + +\item{DateProc}{A vector of class "character" containing the processing date.} + +\item{evalSlpMax}{Maximum acceptable slope of the benchmarking regression} + +\item{evalSlpMin}{Minimum acceptable slope of the benchmarking regression} + +\item{evalOfstMax}{Maximum acceptable offset of the benchmarking regression} + +\item{evalOfstMin}{Minimum acceptable offset of the benchmarking regression} + +\item{gasRefe}{List containing the values of the reference gases. [mol mol-1]} +} +\value{ +pass/fail criteria for a day's validation +} +\description{ +Wrapper function to apply IRGA validation. +} +\examples{ +Currently none. +} +\references{ +License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007. +} +\seealso{ +Currently none. +} +\author{ +Chris Florian \email{eddy4R.info@gmail.com} +} +\keyword{derived,} +\keyword{irgaTurb,} +\keyword{post-processing,} +\keyword{pre-processing,} +\keyword{validation} diff --git a/pack/eddy4R.base/man/def.lag.Rd b/pack/eddy4R.base/man/def.lag.Rd index c3a72fff..8e95a2b4 100644 --- a/pack/eddy4R.base/man/def.lag.Rd +++ b/pack/eddy4R.base/man/def.lag.Rd @@ -4,10 +4,22 @@ \alias{def.lag} \title{Definition function: Lag two datasets, so as to maximise their cross-correlation} \usage{ -def.lag(refe, meas, freq, dataRefe = refe, dataMeas = meas, - measVar = NULL, lagMax = 2 * freq, lagCnst = TRUE, - lagNgtvPstv = c("n", "p", "np")[3], lagAll = TRUE, hpf = TRUE, - fracMin = 0.1, plot = FALSE, DirPlot = NULL) +def.lag( + refe, + meas, + freq, + dataRefe = refe, + dataMeas = meas, + measVar = NULL, + lagMax = 2 * freq, + lagCnst = TRUE, + lagNgtvPstv = c("n", "p", "np")[3], + lagAll = TRUE, + hpf = TRUE, + fracMin = 0.1, + plot = FALSE, + DirPlot = NULL +) } \arguments{ \item{refe}{A vector with variable in reference time frame. Of class numeric. [-]} diff --git a/pack/eddy4R.base/man/def.met.body.Rd b/pack/eddy4R.base/man/def.met.body.Rd index bbf815da..a5eba925 100644 --- a/pack/eddy4R.base/man/def.met.body.Rd +++ b/pack/eddy4R.base/man/def.met.body.Rd @@ -4,9 +4,12 @@ \alias{def.met.body} \title{Definition function: Coordinate transformation from CSAT3 body coordinate system to meteorological coordinate system} \usage{ -def.met.body(AngZaxsSoniInst, - AngZaxsSoniOfst = eddy4R.base::def.unit.conv(data = 90, unitFrom = - "deg", unitTo = "rad"), veloBody) +def.met.body( + AngZaxsSoniInst, + AngZaxsSoniOfst = eddy4R.base::def.unit.conv(data = 90, unitFrom = "deg", unitTo = + "rad"), + veloBody +) } \arguments{ \item{AngZaxsSoniInst}{Parameter of class numeric. Azimuth (angle around z axis) direction against true north in which sonic anemometer installation (transducer array) is pointing [rad]} diff --git a/pack/eddy4R.base/man/def.mtch.out.refe.Rd b/pack/eddy4R.base/man/def.mtch.out.refe.Rd index a41ea9b6..8b9d84c6 100644 --- a/pack/eddy4R.base/man/def.mtch.out.refe.Rd +++ b/pack/eddy4R.base/man/def.mtch.out.refe.Rd @@ -20,7 +20,7 @@ Currently none } \description{ Function definition. A function to compare the file output from a workflow to a reference output file -to ensure that changes during development to functions called by the workflow have not impacted the +to ensure that changes during development to functions called by the workflow have not impacted the results.\cr The function reads in the new data produced in the tmp directory and compares the first 10 lines to the reference output data. } diff --git a/pack/eddy4R.base/man/def.para.flow.Rd b/pack/eddy4R.base/man/def.para.flow.Rd index f6f18e36..20e4fece 100644 --- a/pack/eddy4R.base/man/def.para.flow.Rd +++ b/pack/eddy4R.base/man/def.para.flow.Rd @@ -4,11 +4,26 @@ \alias{def.para.flow} \title{Definition function: Determine the workflow variables} \usage{ -def.para.flow(Deve = TRUE, DirFilePara = NULL, DirInp = NA, - DirMnt = NA, DirOut = NA, DirTmp = NA, DirWrk = NA, - DateOut = NULL, FileOutBase = NULL, Read = "hdf5", - VersDp = c("001", "004")[1], VersEddy = "latest", - MethParaFlow = c("DfltInp", "EnvVar")[1], UrlInpRefe, UrlOutRefe, ...) +def.para.flow( + Deve = TRUE, + DirFilePara = NULL, + DirInp = NA, + DirMnt = NA, + DirOut = NA, + DirTmp = NA, + DirWrk = NA, + DateOut = NULL, + FileOutBase = NULL, + Read = "hdf5", + VersDp = c("001", "004")[1], + VersEddy = "latest", + MethParaFlow = c("DfltInp", "EnvVar")[1], + MethDp01Api = TRUE, + UrlInpRefe, + UrlOutRefe, + MethCh4Conc = FALSE, + ... +) } \arguments{ \item{Deve}{is logical that determines if only a subset of the data should be read in to reduce testing time during development (\code{Deve = TRUE}) or all the input data should be read in (\code{Deve = FALSE})} @@ -37,12 +52,16 @@ def.para.flow(Deve = TRUE, DirFilePara = NULL, DirInp = NA, \item{MethParaFlow}{is the method used to specify workflow parameters, "EnvVar" will grab ParaFlow parameters from environmental variable and "DfltInp" will use whatever is specified in the function call.} +\item{MethDp01Api}{is a logical (TRUE/FALSE) determining if Dp01 reingest data for storage exchange workflow should be gathered from the API.} + \item{UrlInpRefe}{A single-entry vector of class "character" containing the web address of the reference input data zip file to be downloaded.} \item{UrlOutRefe}{A single-entry vector of class "character" containing the web address of the reference output data zip file to be downloaded.} + +\item{MethCh4Conc}{is a logical (TRUE/FALSE) determining if Dp01 methane concentration from the Picarro G2131 should be run} } \value{ -\code{ParaFlow} is a list returned that indicates the workflow control parameters, including \code{ParaFlow$DirFilePara},\code{ParaFlow$DirInp}, \code{ParaFlow$DirMnt}, \code{ParaFlow$DirOut}, \code{ParaFlow$DirTmp}, \code{ParaFlow$DirWrk}, \code{ParaFlow$DateOut}, \code{ParaFlow$FileOutBase}, \code{ParaFlow$Read}, \code{ParaFlow$VersDp}, \code{ParaFlow$VersEddy}. +\code{ParaFlow} is a list returned that indicates the workflow control parameters, including \code{ParaFlow$DirFilePara},\code{ParaFlow$DirInp}, \code{ParaFlow$DirMnt}, \code{ParaFlow$DirOut}, \code{ParaFlow$DirTmp}, \code{ParaFlow$DirWrk}, \code{ParaFlow$DateOut}, \code{ParaFlow$FileOutBase}, \code{ParaFlow$Read}, \code{ParaFlow$VersDp}, \code{ParaFlow$VersEddy}, \code{ParaFlow$MethDp01Api}. } \description{ Definition function. Function to determine the workflow variables by either reading them in from the environmental variables, defining them explicitly, or defining them by default values diff --git a/pack/eddy4R.base/man/def.rglr.Rd b/pack/eddy4R.base/man/def.rglr.Rd index a564962a..a4681e26 100644 --- a/pack/eddy4R.base/man/def.rglr.Rd +++ b/pack/eddy4R.base/man/def.rglr.Rd @@ -4,44 +4,56 @@ \alias{def.rglr} \title{Definition function: Regularizing irregular data to strictly regular / equidistant data} \usage{ -def.rglr(timeMeas, dataMeas, unitMeas = base::attributes(dataMeas)$unit, - BgnRglr = NULL, EndRglr = NULL, - TzRglr = base::attributes(BgnRglr)$tzone, FreqRglr, - MethRglr = c("CybiEc", "cybiDflt", "zoo")[1], WndwRglr = c("Cntr", - "Lead", "Trlg")[1], IdxWndw = c("Clst", "IdxWndwMin", "IdxWndwMax")[1], - PrcsSec = 6) +def.rglr( + timeMeas, + dataMeas, + unitMeas = base::attributes(dataMeas)$unit, + BgnRglr = NULL, + EndRglr = NULL, + TzRglr = base::attributes(BgnRglr)$tzone, + FreqRglr, + MethRglr = c("CybiEc", "CybiEcTimeMeas", "zoo")[1], + WndwRglr = c("Cntr", "Lead", "Trlg")[1], + IdxWndw = c("Clst", "IdxWndwMin", "IdxWndwMax")[1], + DropNotNumc = TRUE, + RptTimeWndw = FALSE, + PrcsSec = 6 +) } \arguments{ -\item{timeMeas}{A vector containing the observation times. Of class "POSIXlt" including timezone attribute, and of the same length as \code{dataMeas}. [-]} +\item{timeMeas}{A vector containing the observation times. Of class "POSIXlt" including timezone attribute, and of the same row length as \code{dataMeas}. [-]} -\item{dataMeas}{A named data.frame containing the observations. Columns may be of class "numeric" or "integer", and of the same length as \code{timeMeas}. Columns of classes other than "numeric" or "integer" are removed and not included in the returned \code{dataRegl}. [user-defined]} +\item{dataMeas}{A named data.frame containing the observations, with row length matching that of \code{timeMeas}. Not that if the zoo method is chosen in \code{MethRglr} or input \code{DropNotNumc} is TRUE, any columns with class other than "numeric" or "integer" are removed and not included in the returned \code{dataRegl}. [user-defined]} \item{unitMeas}{A vector containing the unit of each column in \code{dataMeas}. Of class "character". It is recommended to conform to the "unit representation" guidelines documented in the eddy4R.base package.} -\item{BgnRglr}{Desired begin time for the regularized dataset. Of class "POSIXlt" including timezone attribute, and \code{length(BgnRglr) = 1}. This input is not used in the "cybiDflt" method. [-]} +\item{BgnRglr}{Desired begin time for the regularized dataset. Of class "POSIXlt" including timezone attribute, and \code{length(BgnRglr) = 1}. [-]} -\item{EndRglr}{Desired end time for the regularized dataset. Of class "POSIXlt" including timezone attribute, and \code{length(EndRglr) = 1}. This input is not used in the "cybiDflt" method. [-]} +\item{EndRglr}{Desired end time for the regularized dataset. Of class "POSIXlt" including timezone attribute, and \code{length(EndRglr) = 1}. [-]} -\item{TzRglr}{Desired timezone for the regularized dataset. Of class "character" and \code{length(TzRglr) = 1}, defaults to the same timezone as \code{BgnRglr}. For the "cybiDflt" method, the same time zone as timeMeas is used. [-]} +\item{TzRglr}{Desired timezone for the regularized dataset. Of class "character" and \code{length(TzRglr) = 1}, defaults to the same timezone as \code{BgnRglr}. [-]} \item{FreqRglr}{Desired frequency of the regularized dataset. Of class "numeric" or "integer" and \code{length(FreqRglr) = 1}. [Hz]} \item{MethRglr}{Switch for different regularization methods. Of class "character", currently defaults to "CybiEc". [-] \cr -Method "cybiDflt" implements the default for metereological variable regularization performed by NEON CI. Namely, a new time series is created -from the first measurement time, rounded toward zero, using the expected data frequency. The first measurement falling -in between one time stamp and the next is assigned to the first of these, and all other measurements falling in this range are ignored.\cr Method "CybiEc" implements the default regularization method for eddy-covariance processing utilized CI. The procedure is documented in NEON.DOC.001069.\cr +Method "CybiEcTimeMeas" is a modification of CybiEc that replaces the regularized timestamp with the actual value of \code{timeMeas} for the observation (if any) that was selected following the binning options specified in \code{WndwRglr} and \code{idxWndw}. \cr Method "zoo" implements the regularization method using the zoo::na.approx function. This method can only handle up to millisecond precision (PrcsSec=3)} \item{WndwRglr}{Position of the window for binning in the "CybiEc" method. \code{WndwRglr} can be centered [Cntr], leading [Lead], or trailing [Trlg] (defaults to centered).\cr} \item{IdxWndw}{Determines which observation to allocate to a bin if multiple observations fall into a single bin when using the "CybiEc" method.. \code{IdxWndw} can be set to closest [Clst], first [IdxWndwMin], or last [IdxWndwMax] (defaults to closest).\cr} +\item{DropNotNumc}{Logical. TRUE (default) for removing any non-numeric data columns prior to regularization (this is done automatically for zoo method). FALSE to attempt to regularize all data columns.} + +\item{RptTimeWndw}{Logical. TRUE for including the start and end time of each bin with the output, in list element timeWndw. Defaults to FALSE. Not available as TRUE for zoo method.} + \item{PrcsSec}{A single numeric (integer) value indicating the operational precision of the seconds field of time vectors. Defaults to 6 (microsecond-precision). Values higher than 6 cannot be guaranteed to produce desired results.} } \value{ -Returns a list with elements \code{TzRglr}, \code{FreqRglr}, \code{MethRglr}, \code{timeRglr}, and \code{dataRglr}. +Returns a list with elements \code{TzRglr}, \code{FreqRglr}, \code{MethRglr}, \code{timeRglr}, and \code{dataRglr}. +An additional list element \code{timeWndw} will be included if input \code{RptTimeWndw=TRUE} } \description{ Function defintion. @@ -90,7 +102,7 @@ def.rglr( } \references{ License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007. \cr -NEON.DOC.001069 Preprocessing ATBD: The ATBD that describes the CybiEc and cybiDflt regularization methods. \cr +NEON.DOC.001069 Preprocessing ATBD: The ATBD that describes the CybiEc regularization method. \cr } \seealso{ ?zoo:na.approx, ?stats::approx diff --git a/pack/eddy4R.base/man/def.rmsd.diff.prcs.rsq.Rd b/pack/eddy4R.base/man/def.rmsd.diff.prcs.rsq.Rd index 18d99605..22f8d0ff 100644 --- a/pack/eddy4R.base/man/def.rmsd.diff.prcs.rsq.Rd +++ b/pack/eddy4R.base/man/def.rmsd.diff.prcs.rsq.Rd @@ -4,8 +4,7 @@ \alias{def.rmsd.diff.prcs.rsq} \title{Definition function: RMSD, bias, precision and coefficient of determination - incl. deadband} \usage{ -def.rmsd.diff.prcs.rsq(refe, test, Perc = FALSE, Deba = NULL, - DebaRltv = FALSE) +def.rmsd.diff.prcs.rsq(refe, test, Perc = FALSE, Deba = NULL, DebaRltv = FALSE) } \arguments{ \item{refe}{Variable of class numeric. Reference data.Same unit as test data.} diff --git a/pack/eddy4R.base/man/def.temp.pres.pois.Rd b/pack/eddy4R.base/man/def.temp.pres.pois.Rd index bc32d2c0..f73ecf7b 100644 --- a/pack/eddy4R.base/man/def.temp.pres.pois.Rd +++ b/pack/eddy4R.base/man/def.temp.pres.pois.Rd @@ -4,8 +4,12 @@ \alias{def.temp.pres.pois} \title{Definition function: Poisson's equation (adiabatic change) - temperature as function of pressure change} \usage{ -def.temp.pres.pois(temp01, pres01, pres02, - Kppa = eddy4R.base::IntlNatu$KppaDry) +def.temp.pres.pois( + temp01, + pres01, + pres02, + Kppa = eddy4R.base::IntlNatu$KppaDry +) } \arguments{ \item{temp01}{Measured air temperature [K]} diff --git a/pack/eddy4R.base/man/def.unit.conv.Rd b/pack/eddy4R.base/man/def.unit.conv.Rd index 707441a3..7f9337bb 100644 --- a/pack/eddy4R.base/man/def.unit.conv.Rd +++ b/pack/eddy4R.base/man/def.unit.conv.Rd @@ -4,9 +4,14 @@ \alias{def.unit.conv} \title{Definition function: Unit conversion} \usage{ -def.unit.conv(data, unitFrom = "arb", unitTo = "arb", - coefPoly = base::lapply(base::as.data.frame(data), function(x) c(0, - 1)), vrbs = FALSE, MethGc = TRUE) +def.unit.conv( + data, + unitFrom = "arb", + unitTo = "arb", + coefPoly = base::lapply(base::as.data.frame(data), function(x) c(0, 1)), + vrbs = FALSE, + MethGc = TRUE +) } \arguments{ \item{data}{Required. A named data frame of type numeric, containing the data to be converted. diff --git a/pack/eddy4R.base/man/wrap.agr.vari.seSq.Rd b/pack/eddy4R.base/man/wrap.agr.vari.seSq.Rd index 220d1b5b..c6646a77 100644 --- a/pack/eddy4R.base/man/wrap.agr.vari.seSq.Rd +++ b/pack/eddy4R.base/man/wrap.agr.vari.seSq.Rd @@ -4,8 +4,7 @@ \alias{wrap.agr.vari.seSq} \title{Wrapper function: Calculate aggregated variance and squared standard error from a small to large temporal scale.} \usage{ -wrap.agr.vari.seSq(data = data.frame(timeDoyDecm, mean, vari), zone, - MethAgr) +wrap.agr.vari.seSq(data = data.frame(timeDoyDecm, mean, vari), zone, MethAgr) } \arguments{ \item{data}{Dataframe of type numeric containing column vectors \code{timeDoyDecm}, \code{mean}, and \code{vari} of equal length.} diff --git a/pack/eddy4R.base/man/wrap.dp00.rglr.ecse.Rd b/pack/eddy4R.base/man/wrap.dp00.rglr.ecse.Rd index 39d1b393..920fe459 100644 --- a/pack/eddy4R.base/man/wrap.dp00.rglr.ecse.Rd +++ b/pack/eddy4R.base/man/wrap.dp00.rglr.ecse.Rd @@ -4,8 +4,15 @@ \alias{wrap.dp00.rglr.ecse} \title{Wrapper function: Time regularization for ECSE dp00} \usage{ -wrap.dp00.rglr.ecse(DirInp, Date, Site = "CPER", Dom = "D10", Freq, - IdDp00, horVer) +wrap.dp00.rglr.ecse( + DirInp, + Date, + Site = "CPER", + Dom = "D10", + Freq, + IdDp00, + horVer +) } \arguments{ \item{DirInp}{Character: Input directory. [-]} diff --git a/pack/eddy4R.base/man/wrap.dp01.qfqm.ecte.Rd b/pack/eddy4R.base/man/wrap.dp01.qfqm.ecte.Rd index 92fdc3cc..593701a3 100644 --- a/pack/eddy4R.base/man/wrap.dp01.qfqm.ecte.Rd +++ b/pack/eddy4R.base/man/wrap.dp01.qfqm.ecte.Rd @@ -4,8 +4,13 @@ \alias{wrap.dp01.qfqm.ecte} \title{Wrapper function: Create NEON Level 1 data product quality flags and quality metrics across list elements} \usage{ -wrap.dp01.qfqm.ecte(qfqm, idx = NULL, TypeMeas = "samp", - MethMeas = c("ecte", "ecse")[1], RptExpd = FALSE) +wrap.dp01.qfqm.ecte( + qfqm, + idx = NULL, + TypeMeas = "samp", + MethMeas = c("ecte", "ecse")[1], + RptExpd = FALSE +) } \arguments{ \item{qfqm}{A data.frame or list containing the L0p input data quality flags (sensor and plausibility flags) at native resolution. Of type numeric or integer. [-]} diff --git a/pack/eddy4R.base/man/wrap.hdf5.read.Rd b/pack/eddy4R.base/man/wrap.hdf5.read.Rd index c35023ad..25685617 100644 --- a/pack/eddy4R.base/man/wrap.hdf5.read.Rd +++ b/pack/eddy4R.base/man/wrap.hdf5.read.Rd @@ -4,10 +4,20 @@ \alias{wrap.hdf5.read} \title{Wrapper function: Reading NEON HDF5 files} \usage{ -wrap.hdf5.read(DirInpLoca, SiteLoca, DateLoca, VarLoca, - LvlTowr = c("000_040", "000_050", "000_060")[3], FreqLoca, - Rglr = FALSE, Diag = FALSE, Rng = FALSE, RngLoca = NULL, - DespLoca, MethMeas = c("ecte", "ecse")[1]) +wrap.hdf5.read( + DirInpLoca, + SiteLoca, + DateLoca, + VarLoca, + LvlTowr = c("000_040", "000_050", "000_060")[3], + FreqLoca, + Rglr = FALSE, + Diag = FALSE, + Rng = FALSE, + RngLoca = NULL, + DespLoca, + MethMeas = c("ecte", "ecse")[1] +) } \arguments{ \item{DirInpLoca}{Character: Input directory.} diff --git a/pack/eddy4R.base/man/wrap.hdf5.wrte.dp01.Rd b/pack/eddy4R.base/man/wrap.hdf5.wrte.dp01.Rd index 8cabaa42..1511ba50 100644 --- a/pack/eddy4R.base/man/wrap.hdf5.wrte.dp01.Rd +++ b/pack/eddy4R.base/man/wrap.hdf5.wrte.dp01.Rd @@ -4,8 +4,17 @@ \alias{wrap.hdf5.wrte.dp01} \title{Wrapper function: Write NEON Level 1 data, qfqm, and uncertainty to output HDF5} \usage{ -wrap.hdf5.wrte.dp01(inpList, FileInp, FileOut, SiteLoca, LvlTowr, - MethUcrt = TRUE, MethDp04 = FALSE, MethSubAgr = TRUE) +wrap.hdf5.wrte.dp01( + inpList, + FileInp, + FileOut, + SiteLoca, + LvlTowr, + MethUcrt = TRUE, + MethDp04 = FALSE, + MethSubAgr = TRUE, + Meta +) } \arguments{ \item{inpList}{A list of including dp01 data, quality flags and quality metrics, and uncertainty calculations to package and write to an output HDF5 file} @@ -23,6 +32,8 @@ wrap.hdf5.wrte.dp01(inpList, FileInp, FileOut, SiteLoca, LvlTowr, \item{MethDp04}{logical indicating if ECTE dp04 HDF5 data should be included.} \item{MethSubAgr}{Logical: Determines if 1-minute data is available for output.} + +\item{Meta}{A list of parameters and metadata for updating output HDF5 metadata} } \value{ An HDF5 file with dp01 data, qfqm, and uncertainty written diff --git a/pack/eddy4R.base/man/wrap.hdf5.wrte.dp01.api.Rd b/pack/eddy4R.base/man/wrap.hdf5.wrte.dp01.api.Rd index ef15d565..eb021715 100644 --- a/pack/eddy4R.base/man/wrap.hdf5.wrte.dp01.api.Rd +++ b/pack/eddy4R.base/man/wrap.hdf5.wrte.dp01.api.Rd @@ -4,9 +4,15 @@ \alias{wrap.hdf5.wrte.dp01.api} \title{Wrapper function: Gather/Write reingested NEON Level 1 data, qfqm, and uncertainty to output HDF5} \usage{ -wrap.hdf5.wrte.dp01.api(date, FileOut, SiteLoca, DpName = c("tempAirLvl", - "tempAirTop", "fluxHeatSoil", "radiNet", "tempSoil", "h2oSoilVol", - "presBaro"), LvlTowr, TimeAgr = c(1, 30)) +wrap.hdf5.wrte.dp01.api( + date, + FileOut, + SiteLoca, + DpName = c("tempAirLvl", "tempAirTop", "fluxHeatSoil", "radiNet", "tempSoil", + "h2oSoilVol", "presBaro"), + LvlTowr, + TimeAgr = c(1, 30) +) } \arguments{ \item{date}{Character: The date for the data to be gathered in ISO format ("YYYYmmdd").} diff --git a/pack/eddy4R.base/man/wrap.irga.vali.Rd b/pack/eddy4R.base/man/wrap.irga.vali.Rd index 030cf8ea..177d7805 100644 --- a/pack/eddy4R.base/man/wrap.irga.vali.Rd +++ b/pack/eddy4R.base/man/wrap.irga.vali.Rd @@ -4,7 +4,15 @@ \alias{wrap.irga.vali} \title{Wrapper function: Validation processing for IRGA} \usage{ -wrap.irga.vali(data, qfqmFlag, gasRefe, DateProc) +wrap.irga.vali( + data, + qfqmFlag, + gasRefe, + DateProc, + ScalMax = FALSE, + FracSlp = FALSE, + OfstMax = FALSE +) } \arguments{ \item{data}{List consisting of \code{ff::ffdf} file-backed objects containing the dp0p input IRGA.} @@ -14,6 +22,12 @@ wrap.irga.vali(data, qfqmFlag, gasRefe, DateProc) \item{gasRefe}{List containing the values of the reference gases. [mol mol-1]} \item{DateProc}{A vector of class "character" containing the processing date.} + +\item{ScalMax}{Maximum scale value. The validation correction will not apply if scale (resulted from maximum-likelihood fitting of a functional relationship (MLFR)) is greater than ScalMax or ScalMax = FALSE. Defaults to FALSE.} + +\item{FracSlp}{Upper and lower bounds of slope values. The validation correction will not apply if slope (resulted from regression fitting) is greater/lower than the FracSlp maximum or minimum value or FracSlp = FALSE. Defaults to FALSE.} + +\item{OfstMax}{Maximum offset value. The validation correction will not apply if slope (resulted from regression fitting) is greater than the OfstMax (unit in mol mol-1) or OfstMax = FALSE. Defaults to FALSE.} } \value{ The returned object consists of:\cr diff --git a/pack/eddy4R.base/man/wrap.para.thsh.Rd b/pack/eddy4R.base/man/wrap.para.thsh.Rd index c9919d52..27aac338 100644 --- a/pack/eddy4R.base/man/wrap.para.thsh.Rd +++ b/pack/eddy4R.base/man/wrap.para.thsh.Rd @@ -4,10 +4,12 @@ \alias{wrap.para.thsh} \title{Wrapper Function to read threshold table from CI-Parameter-Repo} \usage{ -wrap.para.thsh(DpName = c("IrgaTurb", "MfcSampTurb", "Soni", "Amrs"), +wrap.para.thsh( + DpName = c("IrgaTurb", "MfcSampTurb", "Soni", "Amrs"), DirInp = "~/eddy/data/Thresholds_EC/CI-Parameter-Repo/ParaSci/Ecte/Dp0p", DirOut = "~/eddy/data/Thresholds_EC/threshold_ecte", - FileWrte = FALSE) + FileWrte = FALSE +) } \arguments{ \item{DpName}{Character: Name of data product. [-]} diff --git a/pack/eddy4R.base/man/wrap.unit.conv.out.ec.Rd b/pack/eddy4R.base/man/wrap.unit.conv.out.ec.Rd index 8783842f..92603fb5 100644 --- a/pack/eddy4R.base/man/wrap.unit.conv.out.ec.Rd +++ b/pack/eddy4R.base/man/wrap.unit.conv.out.ec.Rd @@ -4,8 +4,11 @@ \alias{wrap.unit.conv.out.ec} \title{Wrapper function: Output unit conversion for ECTE} \usage{ -wrap.unit.conv.out.ec(inpList, MethMeas = c("ecte", "ecse")[1], - MethType = c("Data", "Ucrt", "Qfqm", "Vali")[1]) +wrap.unit.conv.out.ec( + inpList, + MethMeas = c("ecte", "ecse")[1], + MethType = c("Data", "Ucrt", "Qfqm", "Vali")[1] +) } \arguments{ \item{inpList}{Required. A named list of data frames of type numeric, containing the data to be converted.} diff --git a/pack/eddy4R.qaqc/DESCRIPTION b/pack/eddy4R.qaqc/DESCRIPTION index 76d2dbf1..7d0096be 100644 --- a/pack/eddy4R.qaqc/DESCRIPTION +++ b/pack/eddy4R.qaqc/DESCRIPTION @@ -1,6 +1,6 @@ Package: eddy4R.qaqc Title: Eddy-covariance calculation for R: Quality assurance and quality control -Version: 0.2.9 +Version: 0.2.14 Authors@R: c( person("Cove", "Sturtevant", email = "eddy4R.info@gmail.com", role = c("aut", "cre")), person("Stefan", "Metzger", email = "eddy4R.info@gmail.com", role = c("aut")), person("Natchaya", "Pingintha-Durden", email = "ndurden@battelleecology.org", role = c("aut")), @@ -17,9 +17,10 @@ Imports: lattice (>= 0.20-35), plyr (>= 1.8.4), reshape2 (>= 1.4.2), - Rmisc (>= 1.5) + Rmisc (>= 1.5), + RcppRoll (>= 0.3.0) Suggests: NEONprocIS.base License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 LazyData: true -RoxygenNote: 6.1.1 +RoxygenNote: 7.1.1 diff --git a/pack/eddy4R.qaqc/NAMESPACE b/pack/eddy4R.qaqc/NAMESPACE index 7e607b90..ec17e388 100644 --- a/pack/eddy4R.qaqc/NAMESPACE +++ b/pack/eddy4R.qaqc/NAMESPACE @@ -16,6 +16,7 @@ export(def.qf.irga.agc) export(def.qf.irga.vali) export(def.qf.rmv.data) export(def.qf.soni) +export(def.qi.qf) export(def.qm) export(wrap.dp01.qfqm) export(wrap.dp01.qfqm.ecse) diff --git a/pack/eddy4R.qaqc/R/def.agr.file.dp00.R b/pack/eddy4R.qaqc/R/def.agr.file.dp00.R index 80572c16..6432978c 100644 --- a/pack/eddy4R.qaqc/R/def.agr.file.dp00.R +++ b/pack/eddy4R.qaqc/R/def.agr.file.dp00.R @@ -37,6 +37,8 @@ # update @param format # Natchaya P-Durden (2018-04-11) # applied eddy4R term name convention; replaced pos by idx +# Cove Sturtevant (2020-07-09) +# adjust call to def.rglr to work with new version ############################################################################################## def.agr.file.dp00 <- function ( @@ -68,7 +70,9 @@ for(idxFile in 1:base::length(nameFile)){ data <- data[[2]] # Regularize time series. - data <- eddy4R.base::def.rglr(timeMeas=timeInp,dataMeas=as.data.frame(data),unitMeas=unitVar[idxFile],FreqRglr=Freq,MethRglr="cybiDflt") + BgnRglr <- base::as.POSIXlt(base::trunc.POSIXt(timeInp[1],units="secs")) # Round down to the nearest whole second + EndRglr <- base::as.POSIXlt(utils::tail(timeInp,1)) + data <- eddy4R.base::def.rglr(timeMeas=timeInp,dataMeas=as.data.frame(data),unitMeas=unitVar[idxFile],BgnRglr=BgnRglr,EndRglr=EndRglr,FreqRglr=Freq,MethRglr="CybiEc",WndwRglr='Trlg',IdxWndw='IdxWndwMin') timeRglr[[idxFile]] <- data$timeRglr dataRglr[[idxFile]] <- data$dataRglr[[1]] diff --git a/pack/eddy4R.qaqc/R/def.dp01.grp.qf.R b/pack/eddy4R.qaqc/R/def.dp01.grp.qf.R index b09798c5..5445563a 100644 --- a/pack/eddy4R.qaqc/R/def.dp01.grp.qf.R +++ b/pack/eddy4R.qaqc/R/def.dp01.grp.qf.R @@ -10,7 +10,7 @@ #' @param MethMeas A vector of class "character" containing the name of measurement method (eddy-covariance turbulent exchange or storage exchange), MethMeas = c("ecte", "ecse"). Defaults to "ecse". [-] #' @param TypeMeas A vector of class "character" containing the name of measurement type (sampling or validation), TypeMeas = c("samp", "ecse"). Defaults to "samp". [-] #' @param dp01 A vector of class "character" containing the name of NEON ECTE and ECSE L1 data products which the flags are being grouped, \cr -#' c("envHut", "co2Turb", "h2oTurb", "isoCo2", "isoH2o", "soni", "amrs", "tempAirLvl", "tempAirTop"). Defaults to "co2Turb". [-] +#' c("envHut", "co2Turb", "h2oTurb", "isoCo2", "ch4Conc", isoH2o", "soni", "amrs", "tempAirLvl", "tempAirTop"). Defaults to "co2Turb". [-] #' @param idGas A data frame contianing gas ID for isoCo2 measurement. Need to provide when dp01 = "isoCo2". Default to NULL. [-] #' @return A list of data frame of the quality flags related to that sub-data product. \cr @@ -84,13 +84,30 @@ # adding the presStor, presValiRegInStor and and presValiRegOutStor quality flags to ECSE # Natchaya P-Durden (2019-05-28) # adding qfHeat to ECSE +# Natchaya P-Durden (2020-03-11) +# removed qfCal and qfHeat from ECSE +# David Durden (2020-06-21) +# removing flags for external sensors from ECSE qfqm +# David Durden (2020-07-14) +# Added qfNull and qfSpk for all variables, removed qfNull from ancillary data streams +# Chris Florian (2020-04-09) +# adding ch4Conc to the dp01 list and adding all ch4 qfqm variables +# David Durden (2021-08-24) +# Changing ancillary data stream ECTE quality flags to quality indicators +# Chris Florian (2021-09-21) +# Adding logic to set CH4 qfs or not depending on if ch4Conc is in the dp01 list +# David Durden (2021-10-08) +# Changing qfSpkPresAtm to quality indicator and fixing soni tempAir flags +# Chris Florian (2021-11-12) +# Removing the step test from Picarro data products, it was not working as intended due to NaNs introduced during regularization + ############################################################################################## def.dp01.grp.qf <- function( qfInp = list(), MethMeas = c("ecte", "ecse")[1], TypeMeas = c("samp", "vali")[1], - dp01 = c("envHut", "co2Turb", "h2oTurb", "co2Stor", "h2oStor", "isoCo2", "isoH2o", "soni", "amrs", "tempAirLvl", "tempAirTop")[1], + dp01 = c("envHut", "co2Turb", "h2oTurb", "co2Stor", "h2oStor", "isoCo2", "ch4Conc", "isoH2o", "soni", "amrs", "tempAirLvl", "tempAirTop")[1], idGas =NULL ){ #check existing input list @@ -124,6 +141,12 @@ def.dp01.grp.qf <- function( } }# close if statement of dp01 + if (dp01 %in% c("ch4Conc")) { + if (!("crdCo2" %in% names(qfInp))){ + base::stop("Missing crdCo2 quality flags") + } + }# close if statement of dp01 + if (dp01 %in% c("isoH2o")) { if (!("crdH2o" %in% names(qfInp))){ base::stop("Missing crdH2o quality flags") @@ -172,158 +195,218 @@ if (MethMeas == "ecte") { setQf$tempIn <- data.frame("qfRngTempIn" = qfInp$irgaTurb$qfRngTempIn, "qfStepTempIn" = qfInp$irgaTurb$qfStepTempIn, - "qfPersTempIn" = qfInp$irgaTurb$qfPersTempIn) + "qfPersTempIn" = qfInp$irgaTurb$qfPersTempIn, + "qfSpkTempIn" = qfInp$irgaTurb$qfSpkTempIn, + "qfNullTempIn" = qfInp$irgaTurb$qfNullTempIn) #"qfCalTempIn" = qfInp$irgaTurb$qfCalTempIn) setQf$tempOut <- data.frame("qfRngTempOut" = qfInp$irgaTurb$qfRngTempOut, "qfStepTempOut" = qfInp$irgaTurb$qfStepTempOut, - "qfPersTempOut" = qfInp$irgaTurb$qfPersTempOut) + "qfPersTempOut" = qfInp$irgaTurb$qfPersTempOut, + "qfSpkTempOut" = qfInp$irgaTurb$qfSpkTempOut, + "qfNullTempOut" = qfInp$irgaTurb$qfNullTempOut) #"qfCalTempOut" = qfInp$irgaTurb$qfCalTempOut) setQf$tempAve <- data.frame ("qfRngTempMean" = qfInp$irgaTurb$qfRngTempMean, "qfStepTempMean" = qfInp$irgaTurb$qfStepTempMean, - "qfPersTempMean" = qfInp$irgaTurb$qfPersTempMean) + "qfPersTempMean" = qfInp$irgaTurb$qfPersTempMean, + "qfSpkTempMean" = qfInp$irgaTurb$qfSpkTempMean, + "qfNullTempMean" = qfInp$irgaTurb$qfNullTempMean) #"qfCalTempMean" = qfInp$irgaTurb$qfCalTempMean) setQf$presAtmIrgaTurb <- data.frame("qfRngPresAtm" = qfInp$irgaTurb$qfRngPresAtm, "qfStepPresAtm" = qfInp$irgaTurb$qfStepPresAtm, - "qfPersPresAtm" = qfInp$irgaTurb$qfPersPresAtm) + "qfPersPresAtm" = qfInp$irgaTurb$qfPersPresAtm, + "qiSpkPresAtm" = qfInp$irgaTurb$qfSpkPresAtm,#TODO remove hard coded qi for SpkPresAtm + "qfNullPresAtm" = qfInp$irgaTurb$qfNullPresAtm) #"qfCalPresAtm" = qfInp$irgaTurb$qfCalPresAtm) setQf$presDiffIrgaTurb <- data.frame("qfRngPresDiff" = qfInp$irgaTurb$qfRngPresDiff, "qfStepPresDiff" = qfInp$irgaTurb$qfStepPresDiff, - "qfPersPresDiff" = qfInp$irgaTurb$qfPersPresDiff) + "qfPersPresDiff" = qfInp$irgaTurb$qfPersPresDiff, + "qfSpkPresDiff" = qfInp$irgaTurb$qfSpkPresDiff, + "qfNullPresDiff" = qfInp$irgaTurb$qfNullPresDiff) #"qfCalPresDiff" = qfInp$irgaTurb$qfCalPresDiff) setQf$presSum <- data.frame("qfRngPresSum" = qfInp$irgaTurb$qfRngPresSum, "qfStepPresSum" = qfInp$irgaTurb$qfStepPresSum, - "qfPersPresSum" = qfInp$irgaTurb$qfPersPresSum) + "qfPersPresSum" = qfInp$irgaTurb$qfPersPresSum, + "qfSpkPresSum" = qfInp$irgaTurb$qfSpkPresSum, + "qfNullPresSum" = qfInp$irgaTurb$qfNullPresSum) #"qfCalPresSum" = qfInp$irgaTurb$qfCalPresSum) setQf$powrH2oSamp <- data.frame ("qfRngPowrH2oSamp" = qfInp$irgaTurb$qfRngPowrH2oSamp, "qfStepPowrH2oSamp" = qfInp$irgaTurb$qfStepPowrH2oSamp, - "qfPersPowrH2oSamp" = qfInp$irgaTurb$qfPersPowrH2oSamp) + "qfPersPowrH2oSamp" = qfInp$irgaTurb$qfPersPowrH2oSamp, + "qfSpkPowrH2oSamp" = qfInp$irgaTurb$qfSpkPowrH2oSamp, + "qfNullPowrH2oSamp" = qfInp$irgaTurb$qfNullPowrH2oSamp) #"qfCalPowrH2oSamp" = qfInp$irgaTurb$qfCalPowrH2oSamp) setQf$powrH2oRefe <- data.frame ("qfRngPowrH2oRefe" = qfInp$irgaTurb$qfRngPowrH2oRefe, "qfStepPowrH2oRefe" = qfInp$irgaTurb$qfStepPowrH2oRefe, - "qfPersPowrH2oRefe" = qfInp$irgaTurb$qfPersPowrH2oRefe) + "qfPersPowrH2oRefe" = qfInp$irgaTurb$qfPersPowrH2oRefe, + "qfSpkPowrH2oRefe" = qfInp$irgaTurb$qfSpkPowrH2oRefe, + "qfNullPowrH2oRefe" = qfInp$irgaTurb$qfNullPowrH2oRefe) #"qfCalPowrH2oRefe" = qfInp$irgaTurb$qfCalPowrH2oRefe) setQf$asrpH2o <- data.frame("qfRngAsrpH2o" = qfInp$irgaTurb$qfRngAsrpH2o, "qfStepAsrpH2o" = qfInp$irgaTurb$qfStepAsrpH2o, - "qfPersAsrpH2o" = qfInp$irgaTurb$qfPersAsrpH2o) + "qfPersAsrpH2o" = qfInp$irgaTurb$qfPersAsrpH2o, + "qfSpkAsrpH2o" = qfInp$irgaTurb$qfSpkAsrpH2o, + "qfNullAsrpH2o" = qfInp$irgaTurb$qfNullAsrpH2o) #"qfCalAsrpH2o" = qfInp$irgaTurb$qfCalAsrpH2o) setQf$densMoleH2o <- data.frame("qfRngDensMoleH2o" = qfInp$irgaTurb$qfRngDensMoleH2o, "qfStepDensMoleH2o" = qfInp$irgaTurb$qfStepDensMoleH2o, - "qfPersDensMoleH2o" = qfInp$irgaTurb$qfPersDensMoleH2o) + "qfPersDensMoleH2o" = qfInp$irgaTurb$qfPersDensMoleH2o, + "qfSpkDensMoleH2o" = qfInp$irgaTurb$qfSpkDensMoleH2o, + "qfNullDensMoleH2o" = qfInp$irgaTurb$qfNullDensMoleH2o) #"qfCalDensMoleH2o" = qfInp$irgaTurb$qfCalDensMoleH2o) setQf$rtioMoleDryH2o <- data.frame("qfRngRtioMoleDryH2o" = qfInp$irgaTurb$qfRngRtioMoleDryH2o, "qfStepRtioMoleDryH2o" = qfInp$irgaTurb$qfStepRtioMoleDryH2o, - "qfPersRtioMoleDryH2o" = qfInp$irgaTurb$qfPersRtioMoleDryH2o) + "qfPersRtioMoleDryH2o" = qfInp$irgaTurb$qfPersRtioMoleDryH2o, + "qfSpkRtioMoleDryH2o" = qfInp$irgaTurb$qfSpkRtioMoleDryH2o, + "qfNullRtioMoleDryH2o" = qfInp$irgaTurb$qfNullRtioMoleDryH2o) #"qfCalRtioMoleDryH2o" = qfInp$irgaTurb$qfCalRtioMoleDryH2o) setQf$powrCo2Samp <- data.frame("qfRngPowrCo2Samp" = qfInp$irgaTurb$qfRngPowrCo2Samp, "qfStepPowrCo2Samp" = qfInp$irgaTurb$qfStepPowrCo2Samp, - "qfPersPowrCo2Samp" = qfInp$irgaTurb$qfPersPowrCo2Samp) + "qfPersPowrCo2Samp" = qfInp$irgaTurb$qfPersPowrCo2Samp, + "qfSpkPowrCo2Samp" = qfInp$irgaTurb$qfSpkPowrCo2Samp, + "qfNullPowrCo2Samp" = qfInp$irgaTurb$qfNullPowrCo2Samp) #"qfCalPowrCo2Samp" = qfInp$irgaTurb$qfCalPowrCo2Samp) setQf$powrCo2Refe <- data.frame ("qfRngPowrCo2Refe" = qfInp$irgaTurb$qfRngPowrCo2Refe, "qfStepPowrCo2Refe" = qfInp$irgaTurb$qfStepPowrCo2Refe, - "qfPersPowrCo2Refe" = qfInp$irgaTurb$qfPersPowrCo2Refe) + "qfPersPowrCo2Refe" = qfInp$irgaTurb$qfPersPowrCo2Refe, + "qfSpkPowrCo2Refe" = qfInp$irgaTurb$qfSpkPowrCo2Refe, + "qfNullPowrCo2Refe" = qfInp$irgaTurb$qfNullPowrCo2Refe) #"qfCalPowrCo2Refe" = qfInp$irgaTurb$qfCalPowrCo2Refe) setQf$asrpCo2 <- data.frame("qfRngAsrpCo2" = qfInp$irgaTurb$qfRngAsrpCo2, "qfStepAsrpCo2" = qfInp$irgaTurb$qfStepAsrpCo2, - "qfPersAsrpCo2" = qfInp$irgaTurb$qfPersAsrpCo2) + "qfPersAsrpCo2" = qfInp$irgaTurb$qfPersAsrpCo2, + "qfSpkAsrpCo2" = qfInp$irgaTurb$qfSpkAsrpCo2, + "qfNullAsrpCo2" = qfInp$irgaTurb$qfNullAsrpCo2) #"qfCalAsrpCo2" = qfInp$irgaTurb$qfCalAsrpCo2) setQf$densMoleCo2 <- data.frame("qfRngDensMoleCo2" = qfInp$irgaTurb$qfRngDensMoleCo2, "qfStepDensMoleCo2" = qfInp$irgaTurb$qfStepDensMoleCo2, - "qfPersDensMoleCo2" = qfInp$irgaTurb$qfPersDensMoleCo2) + "qfPersDensMoleCo2" = qfInp$irgaTurb$qfPersDensMoleCo2, + "qfSpkDensMoleCo2" = qfInp$irgaTurb$qfSpkDensMoleCo2, + "qfNullDensMoleCo2" = qfInp$irgaTurb$qfNullDensMoleCo2) #"qfCalDensMoleCo2" = qfInp$irgaTurb$qfCalDensMoleCo2) setQf$rtioMoleDryCo2 <- data.frame("qfRngRtioMoleDryCo2" = qfInp$irgaTurb$qfRngRtioMoleDryCo2, "qfStepRtioMoleDryCo2" = qfInp$irgaTurb$qfStepRtioMoleDryCo2, - "qfPersRtioMoleDryCo2" = qfInp$irgaTurb$qfPersRtioMoleDryCo2) + "qfPersRtioMoleDryCo2" = qfInp$irgaTurb$qfPersRtioMoleDryCo2, + "qfSpkRtioMoleDryCo2" = qfInp$irgaTurb$qfSpkRtioMoleDryCo2, + "qfNullRtioMoleDryCo2" = qfInp$irgaTurb$qfNullRtioMoleDryCo2) # "qfCalRtioMoleDryCo2" = qfInp$irgaTurb$qfCalRtioMoleDryCo2) setQf$ssiCo2 <- data.frame("qfRngSsiCo2" = qfInp$irgaTurb$qfRngSsiCo2, "qfStepSsiCo2" = qfInp$irgaTurb$qfStepSsiCo2, - "qfPersSsiCo2" = qfInp$irgaTurb$qfPersSsiCo2) + "qfPersSsiCo2" = qfInp$irgaTurb$qfPersSsiCo2, + "qfSpkSsiCo2" = qfInp$irgaTurb$qfSpkSsiCo2, + "qfNullSsiCo2" = qfInp$irgaTurb$qfNullSsiCo2) #"qfCalSsiCo2" = qfInp$irgaTurb$qfCalSsiCo2) setQf$ssiH2o <- data.frame("qfRngSsiH2o" = qfInp$irgaTurb$qfRngSsiH2o, "qfStepSsiH2o" = qfInp$irgaTurb$qfStepSsiH2o, - "qfPersSsiH2o" = qfInp$irgaTurb$qfPersSsiH2o) + "qfPersSsiH2o" = qfInp$irgaTurb$qfPersSsiH2o, + "qfSpkSsiH2o" = qfInp$irgaTurb$qfSpkSsiH2o, + "qfNullSsiH2o" = qfInp$irgaTurb$qfNullSsiH2o) #"qfCalSsiH2o" = qfInp$irgaTurb$qfCalSsiH2o) #external quality flags from mfcSampTurb if ("mfcSampTurb" %in% names(qfInp)){ #mfcSampTurb - setQf$frt00MfcSampTurb <- data.frame("qfRngFrt00" = qfInp$mfcSampTurb$qfRngFrt00) + setQf$frt00MfcSampTurb <- data.frame("qfRngFrt00" = qfInp$mfcSampTurb$qfRngFrt00,"qfSpkFrt00" = qfInp$mfcSampTurb$qfSpkFrt00,"qfNullFrt00" = qfInp$mfcSampTurb$qfNullFrt00) #,"qfPersFrt00" = qfInp$mfcSampTurb$qfPersFrt00) setQf$frtMfcSampTurb <- data.frame("qfRngFrt" = qfInp$mfcSampTurb$qfRngFrt, - "qfPersFrt" = qfInp$mfcSampTurb$qfPersFrt) + "qfPersFrt" = qfInp$mfcSampTurb$qfPersFrt, + "qfSpkFrt" = qfInp$mfcSampTurb$qfSpkFrt, + "qfNullFrt" = qfInp$mfcSampTurb$qfNullFrt) setQf$presAtmMfcSampTurb <- data.frame("qfRngPresAtm" = qfInp$mfcSampTurb$qfRngPresAtm, "qfStepPresAtm" = qfInp$mfcSampTurb$qfStepPresAtm, - "qfPersPresAtm" = qfInp$mfcSampTurb$qfPersPresAtm) + "qfPersPresAtm" = qfInp$mfcSampTurb$qfPersPresAtm, + "qfSpkPresAtm" = qfInp$mfcSampTurb$qfSpkPresAtm, + "qfNullPresAtm" = qfInp$mfcSampTurb$qfNullPresAtm) setQf$tempMfcSampTurb <- data.frame("qfRngTemp" = qfInp$mfcSampTurb$qfRngTemp, "qfStepTemp" = qfInp$mfcSampTurb$qfStepTemp, - "qfPersTemp" = qfInp$mfcSampTurb$qfPersTemp) + "qfPersTemp" = qfInp$mfcSampTurb$qfPersTemp, + "qfSpkTemp" = qfInp$mfcSampTurb$qfSpkTemp, + "qfNullTemp" = qfInp$mfcSampTurb$qfNullTemp) } else { #assign qf for mfcSampTurb to -1 when qf mfcSampTurb is missing - setQf$frt00MfcSampTurb <- data.frame("qfRngFrt00" = -1) + setQf$frt00MfcSampTurb <- data.frame("qfRngFrt00" = -1,"qfSpkFrt00" = -1,"qfNullFrt00" = -1) #,"qfPersFrt00" = -1) setQf$frtMfcSampTurb <- data.frame("qfRngFrt" = -1, - "qfPersFrt" = -1) + "qfPersFrt" = -1, + "qfSpkFrt" = -1, + "qfNullFrt" = -1) setQf$presAtmMfcSampTurb <- data.frame("qfRngPresAtm" = -1, "qfStepPresAtm" = -1, - "qfPersPresAtm" = -1) + "qfPersPresAtm" = -1, + "qfSpkPresAtm" = -1, + "qfNullPresAtm" = -1) setQf$tempMfcSampTurb <- data.frame("qfRngTemp" = -1, "qfStepTemp" = -1, - "qfPersTemp" = -1) + "qfPersTemp" = -1, + "qfSpkTemp" = -1, + "qfNullTemp" = -1) } #external quality flags from mfcValiTurb if ("mfcValiTurb" %in% names(qfInp)){ #mfcValiTurb - setQf$frt00MfcValiTurb <- data.frame("qfRngFrt00" = qfInp$mfcValiTurb$qfRngFrt00) + setQf$frt00MfcValiTurb <- data.frame("qfRngFrt00" = qfInp$mfcValiTurb$qfRngFrt00, "qfSpkFrt00" = qfInp$mfcValiTurb$qfSpkFrt00, "qfNullFrt00" = qfInp$mfcValiTurb$qfNullFrt00) # "qfPersFrt00" = qfInp$mfcValiTurb$qfPersFrt00) setQf$frtMfcValiTurb <- data.frame("qfRngFrt" = qfInp$mfcValiTurb$qfRngFrt, - "qfPersFrt" = qfInp$mfcValiTurb$qfPersFrt) + "qfPersFrt" = qfInp$mfcValiTurb$qfPersFrt, + "qfSpkFrt" = qfInp$mfcValiTurb$qfSpkFrt, + "qfNullFrt" = qfInp$mfcValiTurb$qfNullFrt) setQf$presAtmMfcValiTurb <- data.frame("qfRngPresAtm" = qfInp$mfcValiTurb$qfRngPresAtm, "qfStepPresAtm" = qfInp$mfcValiTurb$qfStepPresAtm, - "qfPersPresAtm" = qfInp$mfcValiTurb$qfPersPresAtm) + "qfPersPresAtm" = qfInp$mfcValiTurb$qfPersPresAtm, + "qfSpkPresAtm" = qfInp$mfcValiTurb$qfSpkPresAtm, + "qfNullPresAtm" = qfInp$mfcValiTurb$qfNullPresAtm) setQf$tempMfcValiTurb <- data.frame("qfRngTemp" = qfInp$mfcValiTurb$qfRngTemp, "qfStepTemp" = qfInp$mfcValiTurb$qfStepTemp, - "qfPersTemp" = qfInp$mfcValiTurb$qfPersTemp) + "qfPersTemp" = qfInp$mfcValiTurb$qfPersTemp, + "qfSpkTemp" = qfInp$mfcValiTurb$qfSpkTemp, + "qfNullTemp" = qfInp$mfcValiTurb$qfNullTemp) } else { #assign qf for mfcValiTurb to -1 when qf mfcValiTurb is missing - setQf$frt00MfcValiTurb <- data.frame("qfRngFrt00" = -1) + setQf$frt00MfcValiTurb <- data.frame("qfRngFrt00" = -1, "qfSpkFrt00" = -1, "qfNullFrt00" = -1) #"qfPersFrt00" = -1) setQf$frtMfcValiTurb <- data.frame("qfRngFrt" = -1, - "qfPersFrt" = -1) + "qfPersFrt" = -1, + "qfSpkFrt" = -1, + "qfNullFrt" = -1) setQf$presAtmMfcValiTurb <- data.frame("qfRngPresAtm" = -1, "qfStepPresAtm" = -1, - "qfPersPresAtm" = -1) + "qfPersPresAtm" = -1, + "qfSpkPresAtm" = -1, + "qfNullPresAtm" = -1) setQf$tempMfcValiTurb <- data.frame("qfRngTemp" = -1, "qfStepTemp" = -1, - "qfPersTemp" = -1) + "qfPersTemp" = -1, + "qfSpkTemp" = -1, + "qfNullTemp" = -1) } #quality flags from soni for grouping qf of tempDew @@ -332,10 +415,14 @@ if (MethMeas == "ecte") { "qfPersVeloSoni" = qfInp$soni$qfPersVeloSoni, "qfRngVeloSoni" = qfInp$soni$qfRngVeloSoni, "qfStepVeloSoni" = qfInp$soni$qfStepVeloSoni, + "qfNullVeloSoni" = qfInp$soni$qfNullVeloSoni, + "qfSpkVeloSoni" = qfInp$soni$qfSpkVeloSoni, #"qfCalTempSoni" = qfInp$soni$qfCalTempSoni, "qfPersTempSoni" = qfInp$soni$qfPersTempSoni, "qfRngTempSoni" = qfInp$soni$qfRngTempSoni, "qfStepTempSoni" = qfInp$soni$qfStepTempSoni, + "qfNullTempSoni" = qfInp$soni$qfNullTempSoni, + "qfSpkTempSoni" = qfInp$soni$qfSpkTempSoni, "qfSoniUnrs" = qfInp$soni$qfSoniUnrs, "qfSoniData" = qfInp$soni$qfSoniData, "qfSoniTrig" = qfInp$soni$qfSoniTrig, @@ -351,10 +438,14 @@ if (MethMeas == "ecte") { "qfPersVeloSoni" = -1, "qfRngVeloSoni" = -1, "qfStepVeloSoni" = -1, + "qfNullVeloSoni" = -1, + "qfSpkVeloSoni" = -1, #"qfCalTempSoni" = -1, "qfPersTempSoni" = -1, "qfRngTempSoni" = -1, "qfStepTempSoni" = -1, + "qfNullTempSoni" = -1, + "qfSpkTempSoni" = -1, "qfSoniUnrs" = -1, "qfSoniData" = -1, "qfSoniTrig" = -1, @@ -371,131 +462,155 @@ if (MethMeas == "ecte") { #grouping qulity flags that related to co2Turb L1 sub-data product if (dp01 == "co2Turb"){ if (TypeMeas == "samp"){ - rpt$densMoleCo2 <- data.frame(setQf$sensIrgaTurb, setQf$sensIrgaTurbExt, - setQf$tempAve, setQf$presDiffIrgaTurb, + rpt$densMoleCo2 <- data.frame(setQf$sensIrgaTurb, setQf$sensIrgaTurbExt, setQf$densMoleCo2, + "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, + eddy4R.qaqc::def.qi.qf(qf = list(setQf$tempAve, setQf$presDiffIrgaTurb, setQf$powrCo2Samp, setQf$powrCo2Refe, - setQf$asrpCo2, setQf$densMoleCo2, + setQf$asrpCo2, setQf$rtioMoleDryCo2, setQf$ssiCo2, - "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, setQf$presAtmMfcSampTurb, - setQf$tempMfcSampTurb) + setQf$presAtmIrgaTurb, + setQf$presSum, + setQf$tempMfcSampTurb))) - rpt$rtioMoleDryCo2 <- data.frame(setQf$sensIrgaTurb, setQf$sensIrgaTurbExt, - setQf$tempAve, setQf$presDiffIrgaTurb, + rpt$rtioMoleDryCo2 <- data.frame(setQf$sensIrgaTurb, setQf$sensIrgaTurbExt, setQf$rtioMoleDryCo2, + "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, + eddy4R.qaqc::def.qi.qf(qf = list(setQf$tempAve, setQf$presDiffIrgaTurb, setQf$powrH2oSamp, setQf$powrH2oRefe, setQf$asrpH2o, setQf$rtioMoleDryH2o, setQf$powrCo2Samp, setQf$powrCo2Refe, - setQf$asrpCo2, setQf$rtioMoleDryCo2, + setQf$asrpCo2, setQf$ssiCo2, setQf$ssiH2o, - "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, setQf$presAtmMfcSampTurb, - setQf$tempMfcSampTurb) + setQf$presAtmIrgaTurb, + setQf$presSum, + setQf$tempMfcSampTurb))) rpt$presAtm <- data.frame(setQf$sensIrgaTurb, setQf$sensIrgaTurbExt, setQf$presAtmIrgaTurb) rpt$presSum <- data.frame(setQf$sensIrgaTurb, setQf$sensIrgaTurbExt, setQf$presSum) rpt$tempAve <- data.frame(setQf$sensIrgaTurb, setQf$sensIrgaTurbExt, - setQf$tempIn, setQf$tempOut, - setQf$tempAve) + setQf$tempAve, + eddy4R.qaqc::def.qi.qf(qf = list(setQf$tempIn, setQf$tempOut))) }#close if statement of TypeMeas == "samp" if (TypeMeas == "vali"){ - rpt$densMoleCo2 <- data.frame(setQf$sensIrgaTurb, setQf$tempAve, + rpt$densMoleCo2 <- data.frame(setQf$sensIrgaTurb, setQf$densMoleCo2, + "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, #TODO: qfRngFrt00 needs to be reconciled, vali qf not currently output + "qfRngFrt00" = setQf$frt00MfcValiTurb$qfRngFrt00, + eddy4R.qaqc::def.qi.qf(qf = list(setQf$tempAve, setQf$presDiffIrgaTurb, setQf$powrCo2Samp, setQf$powrCo2Refe, setQf$asrpCo2 , - setQf$densMoleCo2, setQf$rtioMoleDryCo2, - setQf$ssiCo2, "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, - setQf$presAtmMfcSampTurb, setQf$tempMfcSampTurb, - "qfRngFrt00" = setQf$frt00MfcValiTurb$qfRngFrt00, setQf$presAtmMfcValiTurb, - setQf$tempMfcValiTurb) + setQf$rtioMoleDryCo2, + setQf$ssiCo2, + setQf$presAtmIrgaTurb, + setQf$presSum, setQf$tempMfcSampTurb, + setQf$presAtmMfcValiTurb, + setQf$tempMfcValiTurb))) - rpt$rtioMoleDryCo2 <- data.frame(setQf$sensIrgaTurb, setQf$tempAve, + rpt$rtioMoleDryCo2 <- data.frame(setQf$sensIrgaTurb, setQf$rtioMoleDryCo2, + "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, + "qfRngFrt00" = setQf$frt00MfcValiTurb$qfRngFrt00, + eddy4R.qaqc::def.qi.qf(qf = list(setQf$tempAve, setQf$presDiffIrgaTurb, setQf$powrH2oSamp, setQf$powrH2oRefe, setQf$asrpH2o, setQf$rtioMoleDryH2o, setQf$powrCo2Samp, setQf$powrCo2Refe, setQf$asrpCo2, - setQf$rtioMoleDryCo2, setQf$ssiCo2, - setQf$ssiH2o, "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, - setQf$presAtmMfcSampTurb, setQf$tempMfcSampTurb, - "qfRngFrt00" = setQf$frt00MfcValiTurb$qfRngFrt00, setQf$presAtmMfcValiTurb, - setQf$tempMfcValiTurb) + setQf$ssiCo2, + setQf$ssiH2o, + setQf$presAtmIrgaTurb, + setQf$presSum, setQf$tempMfcSampTurb, + setQf$presAtmMfcValiTurb, + setQf$tempMfcValiTurb))) rpt$presAtm <- data.frame(setQf$sensIrgaTurb, setQf$presAtmIrgaTurb) rpt$presSum <- data.frame(setQf$sensIrgaTurb, setQf$presSum) - rpt$tempAve <- data.frame(setQf$sensIrgaTurb, setQf$tempIn, - setQf$tempOut, setQf$tempAve) + rpt$tempAve <- data.frame(setQf$sensIrgaTurb, setQf$tempAve, + eddy4R.qaqc::def.qi.qf(qf = list(setQf$tempIn, + setQf$tempOut))) }#close if statement of TypeMeas == "vali" - rpt$frt00Samp <- data.frame(setQf$frt00MfcSampTurb, setQf$frtMfcSampTurb, - setQf$presAtmMfcSampTurb, setQf$tempMfcSampTurb) + rpt$frt00Samp <- data.frame(setQf$frt00MfcSampTurb, + eddy4R.qaqc::def.qi.qf(qf = list(setQf$frtMfcSampTurb, + setQf$presAtmMfcSampTurb, setQf$tempMfcSampTurb))) } #grouping qulity flags that related to h2oTurb L1 sub-data product if (dp01 == "h2oTurb") { if (TypeMeas == "samp"){ - rpt$densMoleH2o <- data.frame(setQf$sensIrgaTurb, setQf$sensIrgaTurbExt, - setQf$tempAve, setQf$presDiffIrgaTurb, + rpt$densMoleH2o <- data.frame(setQf$sensIrgaTurb, setQf$sensIrgaTurbExt, setQf$densMoleH2o, + "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, + eddy4R.qaqc::def.qi.qf(qf = list(setQf$tempAve, setQf$presDiffIrgaTurb, setQf$powrH2oSamp, setQf$powrH2oRefe, - setQf$asrpH2o, setQf$densMoleH2o, - setQf$ssiH2o, "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, - setQf$presAtmMfcSampTurb, setQf$tempMfcSampTurb) + setQf$asrpH2o, + setQf$ssiH2o, + setQf$presAtmIrgaTurb, setQf$presSum, setQf$tempMfcSampTurb))) - rpt$rtioMoleDryH2o <- data.frame(setQf$sensIrgaTurb, setQf$sensIrgaTurbExt, - setQf$tempAve, setQf$presDiffIrgaTurb, + rpt$rtioMoleDryH2o <- data.frame(setQf$sensIrgaTurb, setQf$sensIrgaTurbExt, setQf$rtioMoleDryH2o, + "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, + eddy4R.qaqc::def.qi.qf(qf = list(setQf$tempAve, setQf$presDiffIrgaTurb, setQf$powrH2oSamp, setQf$powrH2oRefe, - setQf$asrpH2o, setQf$rtioMoleDryH2o, - setQf$ssiH2o, "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, - setQf$presAtmMfcSampTurb, setQf$tempMfcSampTurb) + setQf$asrpH2o, setQf$ssiH2o, + setQf$presAtmIrgaTurb, setQf$presSum, setQf$tempMfcSampTurb))) rpt$presAtm <- data.frame(setQf$sensIrgaTurb, setQf$sensIrgaTurbExt, setQf$presAtmIrgaTurb) rpt$presSum <- data.frame(setQf$sensIrgaTurb, setQf$sensIrgaTurbExt, setQf$presSum) rpt$tempAve <- data.frame(setQf$sensIrgaTurb, setQf$sensIrgaTurbExt, - setQf$tempIn, setQf$tempOut, - setQf$tempAve) + setQf$tempAve, + eddy4R.qaqc::def.qi.qf(qf = list(setQf$tempIn, setQf$tempOut))) rpt$tempDew <- data.frame(setQf$sensIrgaTurb, setQf$sensIrgaTurbExt, - setQf$tempAve, setQf$presDiffIrgaTurb, + setQf$tempAve, setQf$rtioMoleDryH2o, + eddy4R.qaqc::def.qi.qf(qf = list(setQf$presDiffIrgaTurb, setQf$presSum, setQf$powrH2oSamp, setQf$powrH2oRefe, setQf$asrpH2o, - setQf$rtioMoleDryH2o, setQf$ssiH2o)#,setQf$soni) Remove until we can deal with differance length + setQf$ssiH2o)))#,setQf$soni) Remove until we can deal with differance length }#close if statement of TypeMeas == "samp" if (TypeMeas == "vali"){ - rpt$densMoleH2o <- data.frame(setQf$sensIrgaTurb, setQf$tempAve, + rpt$densMoleH2o <- data.frame(setQf$sensIrgaTurb, setQf$densMoleH2o, + "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, + "qfRngFrt00" = setQf$frt00MfcValiTurb$qfRngFrt00, + eddy4R.qaqc::def.qi.qf(qf = list(setQf$tempAve, setQf$presDiffIrgaTurb, setQf$powrH2oSamp, setQf$powrH2oRefe, setQf$asrpH2o, - setQf$densMoleH2o, setQf$ssiH2o, - "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, setQf$presAtmMfcSampTurb, - setQf$tempMfcSampTurb, "qfRngFrt00" = setQf$frt00MfcValiTurb$qfRngFrt00, - setQf$presAtmMfcValiTurb, setQf$tempMfcValiTurb) + setQf$ssiH2o, + setQf$presAtmIrgaTurb, + setQf$presSum, + setQf$tempMfcSampTurb, + setQf$presAtmMfcValiTurb, setQf$tempMfcValiTurb))) - rpt$rtioMoleDryH2o <- data.frame(setQf$sensIrgaTurb, setQf$tempAve, + rpt$rtioMoleDryH2o <- data.frame(setQf$sensIrgaTurb, setQf$rtioMoleDryH2o, + "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, + "qfRngFrt00" = setQf$frt00MfcValiTurb$qfRngFrt00, + eddy4R.qaqc::def.qi.qf(qf = list(setQf$tempAve, setQf$presDiffIrgaTurb, setQf$powrH2oSamp, setQf$powrH2oRefe, setQf$asrpH2o, - setQf$rtioMoleDryH2o, setQf$ssiH2o, - "qfRngFrt00" = setQf$frt00MfcSampTurb$qfRngFrt00, setQf$presAtmMfcSampTurb, - setQf$tempMfcSampTurb, "qfRngFrt00" = setQf$frt00MfcValiTurb$qfRngFrt00, - setQf$presAtmMfcValiTurb, setQf$tempMfcValiTurb) + setQf$ssiH2o, + setQf$presAtmIrgaTurb, + setQf$presSum, + setQf$tempMfcSampTurb, + setQf$presAtmMfcValiTurb, setQf$tempMfcValiTurb))) rpt$presAtm <- data.frame(setQf$sensIrgaTurb, setQf$presAtmIrgaTurb) rpt$presSum <- data.frame(setQf$sensIrgaTurb, setQf$presSum) - rpt$tempAve <- data.frame(setQf$sensIrgaTurb, setQf$tempIn, - setQf$tempOut, setQf$tempAve) + rpt$tempAve <- data.frame(setQf$sensIrgaTurb, setQf$tempAve, + eddy4R.qaqc::def.qi.qf(qf = list(setQf$tempIn, setQf$tempOut))) - rpt$tempDew <- data.frame(setQf$sensIrgaTurb, setQf$tempAve, - setQf$presDiffIrgaTurb, setQf$presSum, + rpt$tempDew <- data.frame(setQf$sensIrgaTurb, setQf$tempAve, setQf$rtioMoleDryH2o, + eddy4R.qaqc::def.qi.qf(qf = list(setQf$presDiffIrgaTurb, setQf$presSum, setQf$powrH2oSamp,setQf$powrH2oRefe, - setQf$asrpH2o, setQf$rtioMoleDryH2o, - setQf$ssiH2o) #, setQf$soni) Remove until we can deal with the difference length + setQf$asrpH2o, + setQf$ssiH2o))) #, setQf$soni) Remove until we can deal with the difference length }#close if statement of TypeMeas == "vali" - rpt$frt00Samp <- data.frame(setQf$frt00MfcSampTurb, setQf$frtMfcSampTurb, - setQf$presAtmMfcSampTurb, setQf$tempMfcSampTurb) + rpt$frt00Samp <- data.frame(setQf$frt00MfcSampTurb, eddy4R.qaqc::def.qi.qf(qf = list(setQf$frtMfcSampTurb, + setQf$presAtmMfcSampTurb, setQf$tempMfcSampTurb))) } #remove setQf @@ -519,27 +634,37 @@ if (MethMeas == "ecte") { #qf for along-axis wind speed setQf$veloXaxs <- data.frame("qfRngVeloXaxs" = qfInp$soni$qfRngVeloXaxs, "qfStepVeloXaxs" = qfInp$soni$qfStepVeloXaxs, - "qfPersVeloXaxs" = qfInp$soni$qfPersVeloXaxs) + "qfPersVeloXaxs" = qfInp$soni$qfPersVeloXaxs, + "qfSpkVeloXaxs" = qfInp$soni$qfSpkVeloXaxs, + "qfNullVeloXaxs" = qfInp$soni$qfNullVeloXaxs) #"qfCalVeloXaxs" = qfInp$soni$qfCalVeloXaxs) #qf for cross-axis wind speed setQf$veloYaxs <- data.frame("qfRngVeloYaxs" = qfInp$soni$qfRngVeloYaxs, "qfStepVeloYaxs" = qfInp$soni$qfStepVeloYaxs, - "qfPersVeloYaxs" = qfInp$soni$qfPersVeloYaxs) + "qfPersVeloYaxs" = qfInp$soni$qfPersVeloYaxs, + "qfSpkVeloYaxs" = qfInp$soni$qfSpkVeloYaxs, + "qfNullVeloYaxs" = qfInp$soni$qfNullVeloYaxs) #"qfCalVeloYaxs" = qfInp$soni$qfCalVeloYaxs) #qf for vertical-axis wind speed setQf$veloZaxs <- data.frame("qfRngVeloZaxs" = qfInp$soni$qfRngVeloZaxs, "qfStepVeloZaxs" = qfInp$soni$qfStepVeloZaxs, - "qfPersVeloZaxs" = qfInp$soni$qfPersVeloZaxs) + "qfPersVeloZaxs" = qfInp$soni$qfPersVeloZaxs, + "qfSpkVeloZaxs" = qfInp$soni$qfSpkVeloZaxs, + "qfNullVeloZaxs" = qfInp$soni$qfNullVeloZaxs) #"qfCalVeloZaxs" = qfInp$soni$qfCalVeloZaxs) #qf for sonic velocity setQf$veloSoni <- data.frame("qfRngVeloSoni" = qfInp$soni$qfRngVeloSoni, "qfStepVeloSoni" = qfInp$soni$qfStepVeloSoni, - "qfPersVeloSoni" = qfInp$soni$qfPersVeloSoni) + "qfPersVeloSoni" = qfInp$soni$qfPersVeloSoni, + "qfSpkVeloSoni" = qfInp$soni$qfSpkVeloSoni, + "qfNullVeloSoni" = qfInp$soni$qfNullVeloSoni) #"qfCalVeloSoni" = qfInp$soni$qfCalVeloSoni) #qf for soic temperature setQf$tempSoni <- data.frame("qfRngTempSoni" = qfInp$soni$qfRngTempSoni, "qfStepTempSoni" = qfInp$soni$qfStepTempSoni, - "qfPersTempSoni" = qfInp$soni$qfPersTempSoni) + "qfPersTempSoni" = qfInp$soni$qfPersTempSoni, + "qfSpkTempSoni" = qfInp$soni$qfSpkTempSoni, + "qfNullTempSoni" = qfInp$soni$qfNullTempSoni) #"qfCalTempSoni" = qfInp$soni$qfCalTempSoni) #external quality flags from irgaTurb for grouping qf of tempAir @@ -555,33 +680,46 @@ if (MethMeas == "ecte") { "qfIrgaTurbSync" = qfInp$irgaTurb$qfIrgaTurbSync, "qfIrgaTurbAgc" = qfInp$irgaTurb$qfIrgaTurbAgc, "qfIrgaTurbVali" = qfInp$irgaTurb$qfIrgaTurbVali, - "qfRngTempMean" = qfInp$irgaTurb$qfRngTempMean, - "qfStepTempMean" = qfInp$irgaTurb$qfStepTempMean, - "qfPersTempMean" = qfInp$irgaTurb$qfPersTempMean, + "qfRngRtioMoleDryH2o" = qfInp$irgaTurb$qfRngRtioMoleDryH2o, + "qfStepRtioMoleDryH2o" = qfInp$irgaTurb$qfStepRtioMoleDryH2o, + "qfPersRtioMoleDryH2o" = qfInp$irgaTurb$qfPersRtioMoleDryH2o, + "qfSpkRtioMoleDryH2o" = qfInp$irgaTurb$qfSpkRtioMoleDryH2o, + "qfNullRtioMoleDryH2o" = qfInp$irgaTurb$qfNullRtioMoleDryH2o, + # "qfRngTempMean" = qfInp$irgaTurb$qfRngTempMean, + # "qfStepTempMean" = qfInp$irgaTurb$qfStepTempMean, + # "qfPersTempMean" = qfInp$irgaTurb$qfPersTempMean, + # "qfSpkTempMean" = qfInp$irgaTurb$qfSpkTempMean, #"qfCalTempMean" = qfInp$irgaTurb$qfCalTempMean, - "qfRngPresDiff" = qfInp$irgaTurb$qfRngPresDiff, - "qfStepPresDiff" = qfInp$irgaTurb$qfStepPresDiff, - "qfPersPresDiff" = qfInp$irgaTurb$qfPersPresDiff, - #"qfCalPresDiff" = qfInp$irgaTurb$qfCalPresDiff, - "qfRngPowrH2oSamp" = qfInp$irgaTurb$qfRngPowrH2oSamp, - "qfStepPowrH2oSamp" = qfInp$irgaTurb$qfStepPowrH2oSamp, - "qfPersPowrH2oSamp" = qfInp$irgaTurb$qfPersPowrH2oSamp, - #"qfCalPowrH2oSamp" = qfInp$irgaTurb$qfCalPowrH2oSamp, - "qfRngPowrH2oRefe" = qfInp$irgaTurb$qfRngPowrH2oRefe, - "qfStepPowrH2oRefe" = qfInp$irgaTurb$qfStepPowrH2oRefe, - "qfPersPowrH2oRefe" = qfInp$irgaTurb$qfPersPowrH2oRefe, - #"qfCalPowrH2oRefe" = qfInp$irgaTurb$qfCalPowrH2oRefe, - "qfRngAsrpH2o" = qfInp$irgaTurb$qfRngAsrpH2o, - "qfStepAsrpH2o" = qfInp$irgaTurb$qfStepAsrpH2o, - "qfPersAsrpH2o" = qfInp$irgaTurb$qfPersAsrpH2o, - #"qfCalAsrpH2o" = qfInp$irgaTurb$qfCalAsrpH2o, - "qfRngDensMoleH2o" = qfInp$irgaTurb$qfRngDensMoleH2o, - "qfStepDensMoleH2o" = qfInp$irgaTurb$qfStepDensMoleH2o, - "qfPersDensMoleH2o" = qfInp$irgaTurb$qfPersDensMoleH2o, + # "qfRngPresAtm" = qfInp$irgaTurb$qfRngPresAtm, + # "qfStepPresAtm" = qfInp$irgaTurb$qfStepPresAtm, + # "qfPersPresAtm" = qfInp$irgaTurb$qfPersPresAtm, + # "qfSpkPresAtm" = qfInp$irgaTurb$qfSpkPresAtm, + # "qfRngPresSum" = qfInp$irgaTurb$qfRngPresSum, + # "qfStepPresSum" = qfInp$irgaTurb$qfStepPresSum, + # "qfPersPresSum" = qfInp$irgaTurb$qfPersPresSum, + # "qfSpkPresSum" = qfInp$irgaTurb$qfSpkPresSum, + # # #"qfCalPresDiff" = qfInp$irgaTurb$qfCalPresDiff, + # # "qfRngPowrH2oSamp" = qfInp$irgaTurb$qfRngPowrH2oSamp, + # # "qfStepPowrH2oSamp" = qfInp$irgaTurb$qfStepPowrH2oSamp, + # # "qfPersPowrH2oSamp" = qfInp$irgaTurb$qfPersPowrH2oSamp, + # #"qfCalPowrH2oSamp" = qfInp$irgaTurb$qfCalPowrH2oSamp, + # # "qfRngPowrH2oRefe" = qfInp$irgaTurb$qfRngPowrH2oRefe, + # # "qfStepPowrH2oRefe" = qfInp$irgaTurb$qfStepPowrH2oRefe, + # # "qfPersPowrH2oRefe" = qfInp$irgaTurb$qfPersPowrH2oRefe, + # #"qfCalPowrH2oRefe" = qfInp$irgaTurb$qfCalPowrH2oRefe, + # # "qfRngAsrpH2o" = qfInp$irgaTurb$qfRngAsrpH2o, + # # "qfStepAsrpH2o" = qfInp$irgaTurb$qfStepAsrpH2o, + # # "qfPersAsrpH2o" = qfInp$irgaTurb$qfPersAsrpH2o, + # #"qfCalAsrpH2o" = qfInp$irgaTurb$qfCalAsrpH2o, + # "qfRngDensMoleH2o" = qfInp$irgaTurb$qfRngDensMoleH2o, + # "qfStepDensMoleH2o" = qfInp$irgaTurb$qfStepDensMoleH2o, + # "qfPersDensMoleH2o" = qfInp$irgaTurb$qfPersDensMoleH2o, + # "qfSpkDensMoleH2o" = qfInp$irgaTurb$qfSpkDensMoleH2o, + "qfRngFrt00" = qfInp$mfcSampTurb$qfRngFrt00)#, #"qfCalDensMoleH2o" = qfInp$irgaTurb$qfCalDensMoleH2o, - "qfRngSsiH2o" = qfInp$irgaTurb$qfRngSsiH2o, - "qfStepSsiH2o" = qfInp$irgaTurb$qfStepSsiH2o, - "qfPersSsiH2o" = qfInp$irgaTurb$qfPersSsiH2o) + #"qfRngSsiH2o" = qfInp$irgaTurb$qfRngSsiH2o, + #"qfStepSsiH2o" = qfInp$irgaTurb$qfStepSsiH2o, + ##"qfPersSsiH2o" = qfInp$irgaTurb$qfPersSsiH2o) #"qfCalSsiH2o" = qfInp$irgaTurb$qfCalSsiH2o) } else { setQf$irgaTurb <- data.frame("qfIrgaTurbHead" = -1, @@ -595,33 +733,45 @@ if (MethMeas == "ecte") { "qfIrgaTurbSync" = -1, "qfIrgaTurbAgc" = -1, "qfIrgaTurbVali" = -1, - "qfRngTempMean" = -1, - "qfStepTempMean" = -1, - "qfPersTempMean" = -1, + "qfRngRtioMoleDryH2o" = -1, + "qfStepRtioMoleDryH2o" = -1, + "qfPersRtioMoleDryH2o" = -1, + "qfSpkRtioMoleDryH2o" = -1, + "qfNullRtioMoleDryH2o" = -1, + # "qfRngTempMean" = -1, + # "qfStepTempMean" = -1, + # "qfPersTempMean" = -1, #"qfCalTempMean" = -1, - "qfRngPresDiff" = -1, - "qfStepPresDiff" = -1, - "qfPersPresDiff" = -1, - #"qfCalPresDiff" = -1, - "qfRngPowrH2oSamp" = -1, - "qfStepPowrH2oSamp" = -1, - "qfPersPowrH2oSamp" = -1, - #"qfCalPowrH2oSamp" = -1, - "qfRngPowrH2oRefe" = -1, - "qfStepPowrH2oRefe" = -1, - "qfPersPowrH2oRefe" = -1, - #"qfCalPowrH2oRefe" = -1, - "qfRngAsrpH2o" = -1, - "qfStepAsrpH2o" = -1, - "qfPersAsrpH2o" = -1, - #"qfCalAsrpH2o" = -1, - "qfRngDensMoleH2o" = -1, - "qfStepDensMoleH2o" = -1, - "qfPersDensMoleH2o" = -1, + # "qfRngPresAtm" = -1, + # "qfStepPresAtm" = -1, + # "qfPersPresAtm" = -1, + # "qfSpkPresAtm" = -1, + # "qfRngPresSum" = -1, + # "qfStepPresSum" = -1, + # "qfPersPresSum" = -1, + # "qfSpkPresSum" = -1, + # #"qfCalPresDiff" = -1, + # # "qfRngPowrH2oSamp" = -1, + # # "qfStepPowrH2oSamp" = -1, + # # "qfPersPowrH2oSamp" = -1, + # #"qfCalPowrH2oSamp" = -1, + # # "qfRngPowrH2oRefe" = -1, + # # "qfStepPowrH2oRefe" = -1, + # # "qfPersPowrH2oRefe" = -1, + # #"qfCalPowrH2oRefe" = -1, + # # "qfRngAsrpH2o" = -1, + # # "qfStepAsrpH2o" = -1, + # # "qfPersAsrpH2o" = -1, + # #"qfCalAsrpH2o" = -1, + # "qfRngDensMoleH2o" = -1, + # "qfStepDensMoleH2o" = -1, + # "qfPersDensMoleH2o" = -1, + # "qfSpkDensMoleH2o" = -1, + "qfRngFrt00" = -1)#, #"qfCalDensMoleH2o" = -1, - "qfRngSsiH2o" = -1, - "qfStepSsiH2o" = -1, - "qfPersSsiH2o" = -1) + # "qfRngSsiH2o" = -1, + # "qfStepSsiH2o" = -1, + # "qfPersSsiH2o" = -1) #"qfCalSsiH2o" = -1) }#close if else statement for irgaTurb ##TO DO##Considering later when the AMRS is collaborating to correct the SONI data @@ -695,7 +845,7 @@ if (MethMeas == "ecte") { # setQf$veloSoni, setQf$amrs) rpt$tempAir <- data.frame(setQf$sensSoni, setQf$veloSoni, - setQf$tempSoni) #setQf$irgaTurb) # Removing until we can handle flags of different lengths + setQf$tempSoni, setQf$irgaTurb) # Removing until we can handle flags of different lengths rpt$tempSoni <- data.frame(setQf$sensSoni, setQf$veloSoni, setQf$tempSoni) @@ -745,59 +895,84 @@ if (MethMeas == "ecte") { setQf$accXaxs <- data.frame("qfRngAccXaxs" = qfInp$amrs$qfRngAccXaxs, "qfStepAccXaxs" = qfInp$amrs$qfStepAccXaxs, - "qfPersAccXaxs" = qfInp$amrs$qfPersAccXaxs) + "qfPersAccXaxs" = qfInp$amrs$qfPersAccXaxs, + "qfSpkAccXaxs" = qfInp$amrs$qfSpkAccXaxs, + "qfNullAccXaxs" = qfInp$amrs$qfNullAccXaxs + ) setQf$accYaxs <- data.frame("qfRngAccYaxs" = qfInp$amrs$qfRngAccYaxs, "qfStepAccYaxs" = qfInp$amrs$qfStepAccYaxs, - "qfPersAccYaxs" = qfInp$amrs$qfPersAccYaxs) + "qfPersAccYaxs" = qfInp$amrs$qfPersAccYaxs, + "qfSpkAccYaxs" = qfInp$amrs$qfSpkAccYaxs, + "qfNullAccYaxs" = qfInp$amrs$qfNullAccYaxs) setQf$accZaxs <- data.frame("qfRngAccZaxs" = qfInp$amrs$qfRngAccZaxs, "qfStepAccZaxs" = qfInp$amrs$qfStepAccZaxs, - "qfPersAccZaxs" = qfInp$amrs$qfPersAccZaxs) + "qfPersAccZaxs" = qfInp$amrs$qfPersAccZaxs, + "qfSpkAccZaxs" = qfInp$amrs$qfSpkAccZaxs, + "qfNullAccZaxs" = qfInp$amrs$qfNullAccZaxs) setQf$accXaxsDiff <- data.frame("qfRngAccXaxsDiff" = qfInp$amrs$qfRngAccXaxsDiff, "qfStepAccXaxsDiff" = qfInp$amrs$qfStepAccXaxsDiff, - "qfPersAccXaxsDiff" = qfInp$amrs$qfPersAccXaxsDiff) + "qfPersAccXaxsDiff" = qfInp$amrs$qfPersAccXaxsDiff, + "qfSpkAccXaxsDiff" = qfInp$amrs$qfSpkAccXaxsDiff, + "qfNullAccXaxsDiff" = qfInp$amrs$qfNullAccXaxsDiff) setQf$accYaxsDiff <- data.frame("qfRngAccYaxsDiff" = qfInp$amrs$qfRngAccYaxsDiff, "qfStepAccYaxsDiff" = qfInp$amrs$qfStepAccYaxsDiff, - "qfPersAccYaxsDiff" = qfInp$amrs$qfPersAccYaxsDiff) + "qfPersAccYaxsDiff" = qfInp$amrs$qfPersAccYaxsDiff, + "qfSpkAccYaxsDiff" = qfInp$amrs$qfSpkAccYaxsDiff, + "qfNullAccYaxsDiff" = qfInp$amrs$qfNullAccYaxsDiff) setQf$accZaxsDiff <- data.frame("qfRngAccZaxsDiff" = qfInp$amrs$qfRngAccZaxsDiff, "qfStepAccZaxsDiff" = qfInp$amrs$qfStepAccZaxsDiff, - "qfPersAccZaxsDiff" = qfInp$amrs$qfPersAccZaxsDiff) + "qfPersAccZaxsDiff" = qfInp$amrs$qfPersAccZaxsDiff, + "qfSpkAccZaxsDiff" = qfInp$amrs$qfSpkAccZaxsDiff, + "qfNullAccZaxsDiff" = qfInp$amrs$qfNullAccZaxsDiff) setQf$avelXaxs <- data.frame("qfRngAvelXaxs" = qfInp$amrs$qfRngAvelXaxs, "qfStepAvelXaxs" = qfInp$amrs$qfStepAvelXaxs, - "qfPersAvelXaxs" = qfInp$amrs$qfPersAvelXaxs) + "qfPersAvelXaxs" = qfInp$amrs$qfPersAvelXaxs, + "qfSpkAvelXaxs" = qfInp$amrs$qfSpkAvelXaxs, + "qfNullAvelXaxs" = qfInp$amrs$qfNullAvelXaxs) setQf$avelYaxs <- data.frame("qfRngAvelYaxs" = qfInp$amrs$qfRngAvelYaxs, "qfStepAvelYaxs" = qfInp$amrs$qfStepAvelYaxs, - "qfPersAvelYaxs" = qfInp$amrs$qfPersAvelYaxs) + "qfPersAvelYaxs" = qfInp$amrs$qfPersAvelYaxs, + "qfSpkAvelYaxs" = qfInp$amrs$qfSpkAvelYaxs, + "qfNullAvelYaxs" = qfInp$amrs$qfNullAvelYaxs) setQf$avelZaxs <- data.frame("qfRngAvelZaxs" = qfInp$amrs$qfRngAvelZaxs, "qfStepAvelZaxs" = qfInp$amrs$qfStepAvelZaxs, - "qfPersAvelZaxs" = qfInp$amrs$qfPersAvelZaxs) + "qfPersAvelZaxs" = qfInp$amrs$qfPersAvelZaxs, + "qfSpkAvelZaxs" = qfInp$amrs$qfSpkAvelZaxs, + "qfNullAvelZaxs" = qfInp$amrs$qfNullAvelZaxs) setQf$angXaxs <- data.frame("qfRngAngXaxs" = qfInp$amrs$qfRngAngXaxs, "qfStepAngXaxs" = qfInp$amrs$qfStepAngXaxs, - "qfPersAngXaxs" = qfInp$amrs$qfPersAngXaxs) + "qfPersAngXaxs" = qfInp$amrs$qfPersAngXaxs, + "qfSpkAngXaxs" = qfInp$amrs$qfSpkAngXaxs, + "qfNullAngXaxs" = qfInp$amrs$qfNullAngXaxs) setQf$angYaxs <- data.frame("qfRngAngYaxs" = qfInp$amrs$qfRngAngYaxs, "qfStepAngYaxs" = qfInp$amrs$qfStepAngYaxs, - "qfPersAngYaxs" = qfInp$amrs$qfPersAngYaxs) + "qfPersAngYaxs" = qfInp$amrs$qfPersAngYaxs, + "qfSpkAngYaxs" = qfInp$amrs$qfSpkAngYaxs, + "qfNullAngYaxs" = qfInp$amrs$qfNullAngYaxs) setQf$angZaxs <- data.frame("qfRngAngZaxs" = qfInp$amrs$qfRngAngZaxs, "qfStepAngZaxs" = qfInp$amrs$qfStepAngZaxs, - "qfPersAngZaxs" = qfInp$amrs$qfPersAngZaxs) + "qfPersAngZaxs" = qfInp$amrs$qfPersAngZaxs, + "qfSpkAngZaxs" = qfInp$amrs$qfSpkAngZaxs, + "qfNullAngZaxs" = qfInp$amrs$qfNullAngZaxs) #grouping qulity flags that related to L1 sub-data product - rpt$angNedXaxs <- data.frame(setQf$sensAmrs, setQf$angXaxs) - rpt$angNedYaxs <- data.frame(setQf$sensAmrs, setQf$angYaxs) - rpt$angNedZaxs <- data.frame(setQf$sensAmrs, setQf$angZaxs) + rpt$angNedXaxs <- data.frame(setQf$sensAmrs, setQf$angXaxs, eddy4R.qaqc::def.qi.qf(qf = list(setQf$accXaxsDiff, setQf$avelXaxs))) + rpt$angNedYaxs <- data.frame(setQf$sensAmrs, setQf$angYaxs, eddy4R.qaqc::def.qi.qf(qf = list(setQf$accYaxsDiff, setQf$avelYaxs))) + rpt$angNedZaxs <- data.frame(setQf$sensAmrs, setQf$angZaxs, eddy4R.qaqc::def.qi.qf(qf = list(setQf$accZaxsDiff, setQf$avelZaxs))) }#close if statement of TypeMeas %in% c("samp", "vali") } #close if statement of dp01 == "amrs" -} #close if statement of MethMeas == "ecse" +} #close if statement of MethMeas == "ecte" # ecse ####################################################################################### if (MethMeas == "ecse") { @@ -866,43 +1041,43 @@ if (MethMeas == "ecse") { #grouping the flags setQf$asrpCo2 <- data.frame("qfRngAsrpCo2" = qfInp$irgaStor$qfRngAsrpCo2, "qfStepAsrpCo2" = qfInp$irgaStor$qfStepAsrpCo2, - "qfPersAsrpCo2" = qfInp$irgaStor$qfPersAsrpCo2, - "qfCalAsrpCo2" = qfInp$irgaStor$qfCalAsrpCo2) + "qfPersAsrpCo2" = qfInp$irgaStor$qfPersAsrpCo2) + #"qfCalAsrpCo2" = qfInp$irgaStor$qfCalAsrpCo2) setQf$asrpH2o <- data.frame("qfRngAsrpH2o" = qfInp$irgaStor$qfRngAsrpH2o, "qfStepAsrpH2o" = qfInp$irgaStor$qfStepAsrpH2o, - "qfPersAsrpH2o" = qfInp$irgaStor$qfPersAsrpH2o, - "qfCalAsrpH2o" = qfInp$irgaStor$qfCalAsrpH2o) + "qfPersAsrpH2o" = qfInp$irgaStor$qfPersAsrpH2o) + #"qfCalAsrpH2o" = qfInp$irgaStor$qfCalAsrpH2o) setQf$rtioMoleDryCo2 <- data.frame("qfRngRtioMoleDryCo2" = qfInp$irgaStor$qfRngRtioMoleDryCo2, "qfStepRtioMoleDryCo2" = qfInp$irgaStor$qfStepRtioMoleDryCo2, - "qfPersRtioMoleDryCo2" = qfInp$irgaStor$qfPersRtioMoleDryCo2, - "qfCalRtioMoleDryCo2" = qfInp$irgaStor$qfCalRtioMoleDryCo2) + "qfPersRtioMoleDryCo2" = qfInp$irgaStor$qfPersRtioMoleDryCo2) + #"qfCalRtioMoleDryCo2" = qfInp$irgaStor$qfCalRtioMoleDryCo2) setQf$rtioMoleDryH2o <- data.frame("qfRngRtioMoleDryH2o" = qfInp$irgaStor$qfRngRtioMoleDryH2o, "qfStepRtioMoleDryH2o" = qfInp$irgaStor$qfStepRtioMoleDryH2o, - "qfPersRtioMoleDryH2o" = qfInp$irgaStor$qfPersRtioMoleDryH2o, - "qfCalRtioMoleDryH2o" = qfInp$irgaStor$qfCalRtioMoleDryH2o) + "qfPersRtioMoleDryH2o" = qfInp$irgaStor$qfPersRtioMoleDryH2o) + #"qfCalRtioMoleDryH2o" = qfInp$irgaStor$qfCalRtioMoleDryH2o) setQf$rtioMoleWetCo2 <- data.frame("qfRngRtioMoleWetCo2" = qfInp$irgaStor$qfRngRtioMoleWetCo2, "qfStepRtioMoleWetCo2" = qfInp$irgaStor$qfStepRtioMoleWetCo2, - "qfPersRtioMoleWetCo2" = qfInp$irgaStor$qfPersRtioMoleWetCo2, - "qfCalRtioMoleWetCo2" = qfInp$irgaStor$qfCalRtioMoleWetCo2) + "qfPersRtioMoleWetCo2" = qfInp$irgaStor$qfPersRtioMoleWetCo2) + #"qfCalRtioMoleWetCo2" = qfInp$irgaStor$qfCalRtioMoleWetCo2) setQf$rtioMoleWetH2o <- data.frame("qfRngRtioMoleWetH2o" = qfInp$irgaStor$qfRngRtioMoleWetH2o, "qfStepRtioMoleWetH2o" = qfInp$irgaStor$qfStepRtioMoleWetH2o, - "qfPersRtioMoleWetH2o" = qfInp$irgaStor$qfPersRtioMoleWetH2o, - "qfCalRtioMoleWetH2o" = qfInp$irgaStor$qfCalRtioMoleWetH2o) + "qfPersRtioMoleWetH2o" = qfInp$irgaStor$qfPersRtioMoleWetH2o) + #"qfCalRtioMoleWetH2o" = qfInp$irgaStor$qfCalRtioMoleWetH2o) setQf$presIrga <- data.frame("qfRngPres" = qfInp$irgaStor$qfRngPres, "qfStepPres" = qfInp$irgaStor$qfStepPres, - "qfPersPres" = qfInp$irgaStor$qfPersPres, - "qfCalPres" = qfInp$irgaStor$qfCalPres) + "qfPersPres" = qfInp$irgaStor$qfPersPres) + #"qfCalPres" = qfInp$irgaStor$qfCalPres) setQf$tempIrga <- data.frame ("qfRngTemp" = qfInp$irgaStor$qfRngTemp, "qfStepTemp" = qfInp$irgaStor$qfStepTemp, - "qfPersTemp" = qfInp$irgaStor$qfPersTemp, - "qfCalTemp" = qfInp$irgaStor$qfCalTemp) + "qfPersTemp" = qfInp$irgaStor$qfPersTemp) + #"qfCalTemp" = qfInp$irgaStor$qfCalTemp) #change column names names(setQf$asrpCo2) <- paste0(colnames(setQf$asrpCo2), "IrgaStor") @@ -1051,54 +1226,60 @@ if (MethMeas == "ecse") { rpt$rtioMoleDryCo2 <- na.omit(data.frame(setQf$rtioMoleDryCo2, setQf$asrpCo2, setQf$asrpH2o, setQf$rtioMoleWetCo2, setQf$rtioMoleWetH2o, setQf$presIrga, - setQf$tempIrga, setQf$envHut, - setQf$valvAux, - setQf$heatInlt, - setQf$frt00MfcSampStor, setQf$frtMfcSampStor, - setQf$presAtmMfcSampStor, setQf$tempMfcSampStor, + setQf$tempIrga, #setQf$envHut, + #etQf$valvAux, + #setQf$heatInlt, + #setQf$frt00MfcSampStor, setQf$frtMfcSampStor, + #setQf$presAtmMfcSampStor, setQf$tempMfcSampStor, setQf$sensMfcSampStor, - setQf$frt00Mfm, setQf$frtMfm, - setQf$presAtmMfm, setQf$tempMfm, - setQf$sensMfm, setQf$presInl, - setQf$pumpStor, setQf$pumpIrgaStor)) + #setQf$frt00Mfm, setQf$frtMfm, + #setQf$presAtmMfm, setQf$tempMfm, + setQf$sensMfm#, setQf$presInl, + #setQf$pumpStor, setQf$pumpIrgaStor + )) rpt$rtioMoleWetCo2 <- na.omit(data.frame(setQf$rtioMoleWetCo2, setQf$asrpCo2, setQf$asrpH2o, setQf$rtioMoleWetH2o, setQf$presIrga, setQf$tempIrga, - setQf$envHut, setQf$valvAux, - setQf$heatInlt, - setQf$frt00MfcSampStor, - setQf$frtMfcSampStor, setQf$presAtmMfcSampStor, - setQf$tempMfcSampStor, setQf$sensMfcSampStor, - setQf$frt00Mfm, setQf$frtMfm, - setQf$presAtmMfm, setQf$tempMfm, - setQf$sensMfm, setQf$presInl, - setQf$pumpStor, setQf$pumpIrgaStor)) + #setQf$envHut, setQf$valvAux, + #setQf$heatInlt, + #setQf$frt00MfcSampStor, + #setQf$frtMfcSampStor, setQf$presAtmMfcSampStor, + #setQf$tempMfcSampStor, + setQf$sensMfcSampStor, + #setQf$frt00Mfm, setQf$frtMfm, + #setQf$presAtmMfm, setQf$tempMfm, + setQf$sensMfm #, setQf$presInl, + #setQf$pumpStor, setQf$pumpIrgaStor + )) }#close if statement of TypeMeas == "samp" if (TypeMeas == "vali"){ rpt$rtioMoleDryCo2 <- na.omit(data.frame(setQf$rtioMoleDryCo2, setQf$asrpCo2, setQf$asrpH2o, setQf$rtioMoleWetCo2, setQf$rtioMoleWetH2o, setQf$presIrga, - setQf$tempIrga, setQf$envHut, - setQf$valvAux, setQf$frt00MfcSampStor, - setQf$frtMfcSampStor, setQf$presAtmMfcSampStor, - setQf$tempMfcSampStor, setQf$sensMfcSampStor, - setQf$frt00MfcVali, setQf$frtMfcVali, - setQf$presAtmMfcVali, setQf$tempMfcVali, - setQf$sensMfcVali, setQf$pumpIrgaStor, - setQf$presValiRegInStor)) + setQf$tempIrga, #setQf$envHut, + #setQf$valvAux, setQf$frt00MfcSampStor, + # setQf$frtMfcSampStor, setQf$presAtmMfcSampStor, + #setQf$tempMfcSampStor, setQf$sensMfcSampStor, + #setQf$frt00MfcVali, setQf$frtMfcVali, + #setQf$presAtmMfcVali, setQf$tempMfcVali, + setQf$sensMfcVali#, setQf$pumpIrgaStor, + # setQf$presValiRegInStor + )) rpt$rtioMoleWetCo2 <- na.omit(data.frame(setQf$rtioMoleWetCo2, setQf$asrpCo2, setQf$asrpH2o, setQf$rtioMoleWetH2o, setQf$presIrga, setQf$tempIrga, - setQf$envHut, setQf$valvAux, - setQf$frt00MfcSampStor, setQf$frtMfcSampStor, - setQf$presAtmMfcSampStor, setQf$tempMfcSampStor, - setQf$sensMfcSampStor,setQf$frt00MfcVali, - setQf$frtMfcVali, setQf$presAtmMfcVali, - setQf$tempMfcVali, setQf$sensMfcVali, - setQf$pumpIrgaStor, setQf$presValiRegInStor)) + #setQf$envHut, setQf$valvAux, + #setQf$frt00MfcSampStor, setQf$frtMfcSampStor, + #setQf$presAtmMfcSampStor, setQf$tempMfcSampStor, + #setQf$sensMfcSampStor,setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali#, + #setQf$pumpIrgaStor, setQf$presValiRegInStor + )) }#close if statement of TypeMeas == "vali" rpt$pres <- na.omit(data.frame(setQf$presIrga)) @@ -1117,51 +1298,56 @@ if (MethMeas == "ecse") { if (TypeMeas == "samp"){ rpt$rtioMoleDryH2o <- na.omit(data.frame(setQf$rtioMoleDryH2o, setQf$asrpH2o, setQf$rtioMoleWetH2o, setQf$presIrga, - setQf$tempIrga, setQf$envHut, - setQf$valvAux, - setQf$heatInlt, - setQf$frt00MfcSampStor, setQf$frtMfcSampStor, - setQf$presAtmMfcSampStor, setQf$tempMfcSampStor, - setQf$sensMfcSampStor, - setQf$frt00Mfm, setQf$frtMfm, - setQf$presAtmMfm, setQf$tempMfm, - setQf$sensMfm, setQf$presInl, - setQf$pumpStor, setQf$pumpIrgaStor)) + setQf$tempIrga, #setQf$envHut, + #setQf$valvAux, + #setQf$heatInlt, + #setQf$frt00MfcSampStor, setQf$frtMfcSampStor, + #setQf$presAtmMfcSampStor, setQf$tempMfcSampStor, + #setQf$sensMfcSampStor, + #setQf$frt00Mfm, setQf$frtMfm, + #setQf$presAtmMfm, setQf$tempMfm, + setQf$sensMfm#, setQf$presInl, + #setQf$pumpStor, setQf$pumpIrgaStor + )) rpt$rtioMoleWetH2o <- na.omit(data.frame(setQf$rtioMoleWetH2o, setQf$asrpH2o, setQf$presIrga, setQf$tempIrga, - setQf$envHut, setQf$valvAux, - setQf$heatInlt, - setQf$frt00MfcSampStor, - setQf$frtMfcSampStor, setQf$presAtmMfcSampStor, - setQf$tempMfcSampStor, setQf$sensMfcSampStor, - setQf$frt00Mfm, setQf$frtMfm, - setQf$presAtmMfm, setQf$tempMfm, - setQf$sensMfm, setQf$presInl, - setQf$pumpStor, setQf$pumpIrgaStor)) + #setQf$envHut, setQf$valvAux, + #setQf$heatInlt, + #setQf$frt00MfcSampStor, + #setQf$frtMfcSampStor, setQf$presAtmMfcSampStor, + #setQf$tempMfcSampStor, setQf$sensMfcSampStor, + #setQf$frt00Mfm, setQf$frtMfm, + #setQf$presAtmMfm, setQf$tempMfm, + setQf$sensMfm #, setQf$presInl, + #setQf$pumpStor, setQf$pumpIrgaStor + )) }#close if statement of TypeMeas == "samp" if (TypeMeas == "vali"){ rpt$rtioMoleDryH2o <- na.omit(data.frame(setQf$rtioMoleDryH2o, setQf$asrpH2o, setQf$rtioMoleWetH2o, setQf$presIrga, - setQf$tempIrga, setQf$envHut, - setQf$valvAux, setQf$frt00MfcSampStor, - setQf$frtMfcSampStor, setQf$presAtmMfcSampStor, - setQf$tempMfcSampStor, setQf$sensMfcSampStor, - setQf$frt00MfcVali, setQf$frtMfcVali, - setQf$presAtmMfcVali, setQf$tempMfcVali, - setQf$sensMfcVali, setQf$pumpIrgaStor, - setQf$presValiRegInStor)) + setQf$tempIrga, #setQf$envHut, + #setQf$valvAux, setQf$frt00MfcSampStor, + #setQf$frtMfcSampStor, setQf$presAtmMfcSampStor, + #setQf$tempMfcSampStor, setQf$sensMfcSampStor, + #setQf$frt00MfcVali, setQf$frtMfcVali, + #setQf$presAtmMfcVali, setQf$tempMfcVali, + setQf$sensMfcVali#, setQf$pumpIrgaStor, + #setQf$presValiRegInStor + )) rpt$rtioMoleWetH2o <- na.omit(data.frame(setQf$rtioMoleWetH2o, setQf$asrpH2o, setQf$presIrga, setQf$tempIrga, - setQf$envHut, setQf$valvAux, - setQf$frt00MfcSampStor, setQf$frtMfcSampStor, - setQf$presAtmMfcSampStor, setQf$tempMfcSampStor, - setQf$sensMfcSampStor,setQf$frt00MfcVali, - setQf$frtMfcVali, setQf$presAtmMfcVali, - setQf$tempMfcVali, setQf$sensMfcVali, - setQf$pumpIrgaStor, setQf$presValiRegInStor)) + #setQf$envHut, setQf$valvAux, + #setQf$frt00MfcSampStor, setQf$frtMfcSampStor, + #setQf$presAtmMfcSampStor, setQf$tempMfcSampStor, + #setQf$sensMfcSampStor,setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali#, + #setQf$pumpIrgaStor, setQf$presValiRegInStor + )) }#close if statement of TypeMeas == "vali" rpt$pres <- na.omit(data.frame(setQf$presIrga)) @@ -1178,8 +1364,8 @@ if (MethMeas == "ecse") { setQf <- NULL }##close if statement of dp01 %in% c("co2Stor", "h2oStor") -#isoCo2 #################################################################################### - if (dp01 == "isoCo2") { +#isoCo2 and ch4Conc #################################################################################### + if (dp01 %in% c("isoCo2", "ch4Conc")) { #check if data are exist #external quality flags from envHut if (!("envHut" %in% names(qfInp)) || length(which(!is.na(qfInp$crdCo2$qfRngTemp))) == 0){ @@ -1227,76 +1413,96 @@ if (MethMeas == "ecse") { if (length(which(!is.na(qfInp$crdCo2$qfSensStus))) == 0){ qfInp$crdCo2$qfSensStus <- -1 } + #setQf for crdCo2 setQf$rtioMoleDryCo2 <- data.frame("qfRngRtioMoleDryCo2" = qfInp$crdCo2$qfRngRtioMoleDryCo2, - "qfStepRtioMoleDryCo2" = qfInp$crdCo2$qfStepRtioMoleDryCo2, - "qfPersRtioMoleDryCo2" = qfInp$crdCo2$qfPersRtioMoleDryCo2, - "qfCalRtioMoleDryCo2" = qfInp$crdCo2$qfCalRtioMoleDryCo2) + #"qfStepRtioMoleDryCo2" = qfInp$crdCo2$qfStepRtioMoleDryCo2, + "qfPersRtioMoleDryCo2" = qfInp$crdCo2$qfPersRtioMoleDryCo2) + #"qfCalRtioMoleDryCo2" = qfInp$crdCo2$qfCalRtioMoleDryCo2) setQf$rtioMoleDry12CCo2 <- data.frame("qfRngRtioMoleDry12CCo2" = qfInp$crdCo2$qfRngRtioMoleDry12CCo2, - "qfStepRtioMoleDry12CCo2" = qfInp$crdCo2$qfStepRtioMoleDry12CCo2, - "qfPersRtioMoleDry12CCo2" = qfInp$crdCo2$qfPersRtioMoleDry12CCo2, - "qfCalRtioMoleDry12CCo2" = qfInp$crdCo2$qfCalRtioMoleDry12CCo2) + #"qfStepRtioMoleDry12CCo2" = qfInp$crdCo2$qfStepRtioMoleDry12CCo2, + "qfPersRtioMoleDry12CCo2" = qfInp$crdCo2$qfPersRtioMoleDry12CCo2) + #"qfCalRtioMoleDry12CCo2" = qfInp$crdCo2$qfCalRtioMoleDry12CCo2) setQf$rtioMoleDry13CCo2 <- data.frame("qfRngRtioMoleDry13CCo2" = qfInp$crdCo2$qfRngRtioMoleDry13CCo2, - "qfStepRtioMoleDry13CCo2" = qfInp$crdCo2$qfStepRtioMoleDry13CCo2, - "qfPersRtioMoleDry13CCo2" = qfInp$crdCo2$qfPersRtioMoleDry13CCo2, - "qfCalRtioMoleDry13CCo2" = qfInp$crdCo2$qfCalRtioMoleDry13CCo2) + #"qfStepRtioMoleDry13CCo2" = qfInp$crdCo2$qfStepRtioMoleDry13CCo2, + "qfPersRtioMoleDry13CCo2" = qfInp$crdCo2$qfPersRtioMoleDry13CCo2) + #"qfCalRtioMoleDry13CCo2" = qfInp$crdCo2$qfCalRtioMoleDry13CCo2) + + + if (dp01 == "ch4Conc"){ + setQf$rtioMoleDryCh4 <- data.frame("qfRngRtioMoleDryCh4" = qfInp$crdCo2$qfRngRtioMoleDryCh4, + #"qfStepRtioMoleDryCh4" = qfInp$crdCo2$qfStepRtioMoleDryCh4, + "qfPersRtioMoleDryCh4" = qfInp$crdCo2$qfPersRtioMoleDryCh4) + #"qfCalRtioMoleDryCh4" = qfInp$crdCo2$qfCalRtioMoleDryCh4) + } + + setQf$rtioMoleDryH2o <- data.frame("qfRngRtioMoleDryH2o" = qfInp$crdCo2$qfRngRtioMoleDryH2o, #"qfStepRtioMoleDryH2o" = qfInp$crdCo2$qfStepRtioMoleDryH2o, - "qfPersRtioMoleDryH2o" = qfInp$crdCo2$qfPersRtioMoleDryH2o, - "qfCalRtioMoleDryH2o" = qfInp$crdCo2$qfCalRtioMoleDryH2o) + "qfPersRtioMoleDryH2o" = qfInp$crdCo2$qfPersRtioMoleDryH2o) + #"qfCalRtioMoleDryH2o" = qfInp$crdCo2$qfCalRtioMoleDryH2o) setQf$rtioMoleWetCo2 <- data.frame("qfRngRtioMoleWetCo2" = qfInp$crdCo2$qfRngRtioMoleWetCo2, - "qfStepRtioMoleWetCo2" = qfInp$crdCo2$qfStepRtioMoleWetCo2, - "qfPersRtioMoleWetCo2" = qfInp$crdCo2$qfPersRtioMoleWetCo2, - "qfCalRtioMoleWetCo2" = qfInp$crdCo2$qfCalRtioMoleWetCo2) + #"qfStepRtioMoleWetCo2" = qfInp$crdCo2$qfStepRtioMoleWetCo2, + "qfPersRtioMoleWetCo2" = qfInp$crdCo2$qfPersRtioMoleWetCo2) + #"qfCalRtioMoleWetCo2" = qfInp$crdCo2$qfCalRtioMoleWetCo2) setQf$rtioMoleWet12CCo2 <- data.frame("qfRngRtioMoleWet12CCo2" = qfInp$crdCo2$qfRngRtioMoleWet12CCo2, - "qfStepRtioMoleWet12CCo2" = qfInp$crdCo2$qfStepRtioMoleWet12CCo2, - "qfPersRtioMoleWet12CCo2" = qfInp$crdCo2$qfPersRtioMoleWet12CCo2, - "qfCalRtioMoleWet12CCo2" = qfInp$crdCo2$qfCalRtioMoleWet12CCo2) + #"qfStepRtioMoleWet12CCo2" = qfInp$crdCo2$qfStepRtioMoleWet12CCo2, + "qfPersRtioMoleWet12CCo2" = qfInp$crdCo2$qfPersRtioMoleWet12CCo2) + #"qfCalRtioMoleWet12CCo2" = qfInp$crdCo2$qfCalRtioMoleWet12CCo2) setQf$rtioMoleWet13CCo2 <- data.frame("qfRngRtioMoleWet13CCo2" = qfInp$crdCo2$qfRngRtioMoleWet13CCo2, - "qfStepRtioMoleWet13CCo2" = qfInp$crdCo2$qfStepRtioMoleWet13CCo2, - "qfPersRtioMoleWet13CCo2" = qfInp$crdCo2$qfPersRtioMoleWet13CCo2, - "qfCalRtioMoleWet13CCo2 " = qfInp$crdCo2$qfCalRtioMoleWet13CCo2) + #"qfStepRtioMoleWet13CCo2" = qfInp$crdCo2$qfStepRtioMoleWet13CCo2, + "qfPersRtioMoleWet13CCo2" = qfInp$crdCo2$qfPersRtioMoleWet13CCo2) + #"qfCalRtioMoleWet13CCo2 " = qfInp$crdCo2$qfCalRtioMoleWet13CCo2) + + if (dp01 == "ch4Conc"){ + setQf$rtioMoleWetCh4 <- data.frame("qfRngRtioMoleWetCh4" = qfInp$crdCo2$qfRngRtioMoleWetCh4, + #"qfStepRtioMoleWetCh4" = qfInp$crdCo2$qfStepRtioMoleWetCh4, + "qfPersRtioMoleWetCh4" = qfInp$crdCo2$qfPersRtioMoleWetCh4) + #"qfCalRtioMoleWetCh4" = qfInp$crdCo2$qfCalRtioMoleWetCh4) + } setQf$rtioMoleWetH2o <- data.frame("qfRngRtioMoleWetH2o" = qfInp$crdCo2$qfRngRtioMoleWetH2o, #"qfStepRtioMoleWetH2o" = qfInp$crdCo2$qfStepRtioMoleWetH2o, - "qfPersRtioMoleWetH2o" = qfInp$crdCo2$qfPersRtioMoleWetH2o, - "qfCalRtioMoleWetH2o" = qfInp$crdCo2$qfCalRtioMoleWetH2o) + "qfPersRtioMoleWetH2o" = qfInp$crdCo2$qfPersRtioMoleWetH2o) + #"qfCalRtioMoleWetH2o" = qfInp$crdCo2$qfCalRtioMoleWetH2o) setQf$dlta13CCo2 <- data.frame("qfRngDlta13CCo2" = qfInp$crdCo2$qfRngDlta13CCo2, - "qfStepDlta13CCo2" = qfInp$crdCo2$qfStepDlta13CCo2, - "qfPersDlta13CCo2" = qfInp$crdCo2$qfPersDlta13CCo2, - "qfCalDlta13CCo2" = qfInp$crdCo2$qfCalDlta13CCo2) + #"qfStepDlta13CCo2" = qfInp$crdCo2$qfStepDlta13CCo2, + "qfPersDlta13CCo2" = qfInp$crdCo2$qfPersDlta13CCo2) + #"qfCalDlta13CCo2" = qfInp$crdCo2$qfCalDlta13CCo2) setQf$presCrdCo2 <- data.frame("qfRngPres" = qfInp$crdCo2$qfRngPres, - "qfStepPres" = qfInp$crdCo2$qfStepPres, - "qfPersPres" = qfInp$crdCo2$qfPersPres, - "qfCalPres" = qfInp$crdCo2$qfCalPres) + #"qfStepPres" = qfInp$crdCo2$qfStepPres, + "qfPersPres" = qfInp$crdCo2$qfPersPres) + #"qfCalPres" = qfInp$crdCo2$qfCalPres) setQf$tempCrdCo2 <- data.frame("qfRngTemp" = qfInp$crdCo2$qfRngTemp, - "qfStepTemp" = qfInp$crdCo2$qfStepTemp, - "qfPersTemp" = qfInp$crdCo2$qfPersTemp, - "qfCalTemp" = qfInp$crdCo2$qfCalTemp) + #"qfStepTemp" = qfInp$crdCo2$qfStepTemp, + "qfPersTemp" = qfInp$crdCo2$qfPersTemp) + #"qfCalTemp" = qfInp$crdCo2$qfCalTemp) setQf$tempWbox <- data.frame("qfRngTempWbox" = qfInp$crdCo2$qfRngTempWbox, - "qfStepTempWbox" = qfInp$crdCo2$qfStepTempWbox, - "qfPersTempWbox" = qfInp$crdCo2$qfPersTempWbox, - "qfCalTempWbox" = qfInp$crdCo2$qfCalTempWbox) + #"qfStepTempWbox" = qfInp$crdCo2$qfStepTempWbox, + "qfPersTempWbox" = qfInp$crdCo2$qfPersTempWbox) + #"qfCalTempWbox" = qfInp$crdCo2$qfCalTempWbox) setQf$sensCrdCo2 <- data.frame("qfSensStus" = qfInp$crdCo2$qfSensStus) #change column names names(setQf$rtioMoleDryCo2) <- paste0(colnames(setQf$rtioMoleDryCo2), "CrdCo2") names(setQf$rtioMoleDry12CCo2) <- paste0(colnames(setQf$rtioMoleDry12CCo2), "CrdCo2") names(setQf$rtioMoleDry13CCo2) <- paste0(colnames(setQf$rtioMoleDry13CCo2), "CrdCo2") + if (dp01 == "ch4Conc"){names(setQf$rtioMoleDryCh4) <- paste0(colnames(setQf$rtioMoleDryCh4), "CrdCo2")} names(setQf$rtioMoleDryH2o) <- paste0(colnames(setQf$rtioMoleDryH2o), "CrdCo2") names(setQf$rtioMoleWetCo2) <- paste0(colnames(setQf$rtioMoleWetCo2), "CrdCo2") names(setQf$rtioMoleWet12CCo2) <- paste0(colnames(setQf$rtioMoleWet12CCo2), "CrdCo2") names(setQf$rtioMoleWet13CCo2) <- paste0(colnames(setQf$rtioMoleWet13CCo2), "CrdCo2") + if (dp01 == "ch4Conc"){names(setQf$rtioMoleWetCh4) <- paste0(colnames(setQf$rtioMoleWetCh4), "CrdCo2")} names(setQf$rtioMoleWetH2o) <- paste0(colnames(setQf$rtioMoleWetH2o), "CrdCo2") names(setQf$dlta13CCo2) <- paste0(colnames(setQf$dlta13CCo2), "CrdCo2") names(setQf$presCrdCo2) <- paste0(colnames(setQf$presCrdCo2), "CrdCo2") @@ -1398,398 +1604,630 @@ if (MethMeas == "ecse") { #change column names names(setQf$presValiRegInStor) <- paste0(colnames(setQf$presValiRegInStor), "PresValiRegInStor") - #define qf which use only sampling period - if (TypeMeas == "samp") { - #temporary list - tmp <- list() - #if not all idGas = NA or all qfRngTemp = NA - if (length(which(!is.na(idGas))) > 0){ - #grouping qulity flags that related to isoCo2 L1 sub-data product - tmp$rtioMoleWetCo2 <- data.frame(setQf$rtioMoleWetCo2, setQf$dlta13CCo2, - setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00Mfm, - setQf$frtMfm, setQf$presAtmMfm, - setQf$tempMfm,setQf$sensMfm, - setQf$presInlt, setQf$pumpStor, - setQf$heatInlt, idGas = idGas - ) - rpt$rtioMoleWetCo2 <- na.omit(tmp$rtioMoleWetCo2[which(tmp$rtioMoleWetCo2$idGas == 105 | (is.na(tmp$rtioMoleWetCo2$idGas) & tmp$rtioMoleWetCo2$qfSensStus == -1)),]) - - tmp$rtioMoleDryCo2 <- data.frame(setQf$rtioMoleDryCo2, setQf$dlta13CCo2, - setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00Mfm, - setQf$frtMfm, setQf$presAtmMfm, - setQf$tempMfm, setQf$sensMfm, - setQf$presInlt, setQf$pumpStor, - setQf$heatInlt, idGas = idGas - - ) - rpt$rtioMoleDryCo2 <- na.omit(tmp$rtioMoleDryCo2[which(tmp$rtioMoleDryCo2$idGas == 105 | (is.na(tmp$rtioMoleDryCo2$idGas) & tmp$rtioMoleDryCo2$qfSensStus == -1)),]) - - tmp$rtioMoleWet12CCo2 <-data.frame(setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, + #grouping quality flags related to the isoCo2 L1 data product + + if (dp01 %in% "isoCo2") { + #define qf which use only sampling period + if (TypeMeas == "samp") { + #temporary list + tmp <- list() + #if not all idGas = NA or all qfRngTemp = NA + if (length(which(!is.na(idGas))) > 0){ + #grouping qulity flags that related to isoCo2 L1 sub-data product + tmp$rtioMoleWetCo2 <- data.frame(setQf$rtioMoleWetCo2, setQf$rtioMoleWet13CCo2, #setQf$dlta13CCo2, + setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, + #setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm, + #setQf$presInlt, setQf$pumpStor, + #setQf$heatInlt, + idGas = idGas + ) + rpt$rtioMoleWetCo2 <- na.omit(tmp$rtioMoleWetCo2[which(tmp$rtioMoleWetCo2$idGas == 105 | (is.na(tmp$rtioMoleWetCo2$idGas) & tmp$rtioMoleWetCo2$qfSensStus == -1)),]) + + tmp$rtioMoleDryCo2 <- data.frame(setQf$rtioMoleDryCo2, #setQf$dlta13CCo2, + setQf$rtioMoleDry12CCo2, setQf$rtioMoleDry13CCo2, setQf$presCrdCo2, setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00Mfm, - setQf$frtMfm, setQf$presAtmMfm, - setQf$tempMfm, setQf$sensMfm, - setQf$presInlt, setQf$pumpStor, - setQf$heatInlt, idGas = idGas + setQf$sensCrdCo2, + #setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm, + #setQf$presInlt, setQf$pumpStor, + #setQf$heatInlt, + idGas = idGas - ) - rpt$rtioMoleWet12CCo2 <- na.omit(tmp$rtioMoleWet12CCo2[which(tmp$rtioMoleWet12CCo2$idGas == 105 | (is.na(tmp$rtioMoleWet12CCo2$idGas) & tmp$rtioMoleWet12CCo2$qfSensStus == -1)),]) - - tmp$rtioMoleDry12CCo2 <- data.frame(setQf$rtioMoleDry12CCo2, setQf$rtioMoleWet13CCo2, + ) + rpt$rtioMoleDryCo2 <- na.omit(tmp$rtioMoleDryCo2[which(tmp$rtioMoleDryCo2$idGas == 105 | (is.na(tmp$rtioMoleDryCo2$idGas) & tmp$rtioMoleDryCo2$qfSensStus == -1)),]) + + tmp$rtioMoleWet12CCo2 <-data.frame(setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, + #, setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm, + #setQf$presInlt, setQf$pumpStor, + #setQf$heatInlt, + idGas = idGas + + ) + rpt$rtioMoleWet12CCo2 <- na.omit(tmp$rtioMoleWet12CCo2[which(tmp$rtioMoleWet12CCo2$idGas == 105 | (is.na(tmp$rtioMoleWet12CCo2$idGas) & tmp$rtioMoleWet12CCo2$qfSensStus == -1)),]) + + tmp$rtioMoleDry12CCo2 <- data.frame(setQf$rtioMoleDry12CCo2, setQf$rtioMoleWet12CCo2, + setQf$presCrdCo2, setQf$tempCrdCo2, + setQf$tempWbox, setQf$sensCrdCo2, + #setQf$frt00Mfm, setQf$frtMfm, + #setQf$presAtmMfm, setQf$tempMfm, + setQf$sensMfm, + #, setQf$presInlt, + #setQf$pumpStor, + #setQf$heatInlt, + idGas = idGas + ) + rpt$rtioMoleDry12CCo2 <- na.omit(tmp$rtioMoleDry12CCo2[which(tmp$rtioMoleDry12CCo2$idGas == 105 | (is.na(tmp$rtioMoleDry12CCo2$idGas) & tmp$rtioMoleDry12CCo2$qfSensStus == -1)),]) + + tmp$rtioMoleWet13CCo2 <- data.frame(setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, + #setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm, + #setQf$presInlt, setQf$pumpStor, + #setQf$heatInlt, + idGas = idGas + + ) + rpt$rtioMoleWet13CCo2 <- na.omit(tmp$rtioMoleWet13CCo2[which(tmp$rtioMoleWet13CCo2$idGas == 105 | (is.na(tmp$rtioMoleWet13CCo2$idGas) & tmp$rtioMoleWet13CCo2$qfSensStus == -1)),]) + + tmp$rtioMoleDry13CCo2<- data.frame(setQf$rtioMoleDry13CCo2, setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, setQf$tempCrdCo2, setQf$tempWbox, setQf$sensCrdCo2, - setQf$frt00Mfm, setQf$frtMfm, - setQf$presAtmMfm, setQf$tempMfm, - setQf$sensMfm, setQf$presInlt, - setQf$pumpStor, setQf$heatInlt, + #setQf$frt00Mfm, setQf$frtMfm, + #setQf$presAtmMfm, setQf$tempMfm, + setQf$sensMfm, + #setQf$presInlt, + #setQf$pumpStor, + #setQf$heatInlt, idGas = idGas - ) - rpt$rtioMoleDry12CCo2 <- na.omit(tmp$rtioMoleDry12CCo2[which(tmp$rtioMoleDry12CCo2$idGas == 105 | (is.na(tmp$rtioMoleDry12CCo2$idGas) & tmp$rtioMoleDry12CCo2$qfSensStus == -1)),]) - - tmp$rtioMoleWet13CCo2 <- data.frame(setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00Mfm, - setQf$frtMfm, setQf$presAtmMfm, - setQf$tempMfm, setQf$sensMfm, - setQf$presInlt, setQf$pumpStor, - setQf$heatInlt, idGas = idGas - - ) - rpt$rtioMoleWet13CCo2 <- na.omit(tmp$rtioMoleWet13CCo2[which(tmp$rtioMoleWet13CCo2$idGas == 105 | (is.na(tmp$rtioMoleWet13CCo2$idGas) & tmp$rtioMoleWet13CCo2$qfSensStus == -1)),]) - - tmp$rtioMoleDry13CCo2<- data.frame(setQf$rtioMoleDry13CCo2, setQf$rtioMoleWet13CCo2, - setQf$presCrdCo2, setQf$tempCrdCo2, - setQf$tempWbox, setQf$sensCrdCo2, - setQf$frt00Mfm, setQf$frtMfm, - setQf$presAtmMfm, setQf$tempMfm, - setQf$sensMfm, setQf$presInlt, - setQf$pumpStor, setQf$heatInlt, - idGas = idGas - - ) - rpt$rtioMoleDry13CCo2 <- na.omit(tmp$rtioMoleDry13CCo2[which(tmp$rtioMoleDry13CCo2$idGas == 105 | (is.na(tmp$rtioMoleDry13CCo2$idGas) & tmp$rtioMoleDry13CCo2$qfSensStus == -1)),]) - - tmp$dlta13CCo2 <- data.frame(setQf$dlta13CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00Mfm, - setQf$frtMfm, setQf$presAtmMfm, - setQf$tempMfm, setQf$sensMfm, - setQf$presInlt, setQf$pumpStor, - setQf$heatInlt, idGas = idGas - - ) - rpt$dlta13CCo2 <- na.omit(tmp$dlta13CCo2[which(tmp$dlta13CCo2$idGas == 105 | (is.na(tmp$dlta13CCo2$idGas) & tmp$dlta13CCo2$qfSensStus == -1)),]) - - tmp$rtioMoleWetH2o <- data.frame(setQf$rtioMoleWetH2o, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00Mfm, - setQf$frtMfm, setQf$presAtmMfm, - setQf$tempMfm, setQf$sensMfm, - setQf$presInlt, setQf$pumpStor, - setQf$heatInlt, idGas = idGas - - ) - rpt$rtioMoleWetH2o <- na.omit(tmp$rtioMoleWetH2o[which(tmp$rtioMoleWetH2o$idGas == 11 | (is.na(tmp$rtioMoleWetH2o$idGas) & tmp$rtioMoleWetH2o$qfSensStus == -1)),]) - - tmp$rtioMoleDryH2o <- data.frame(setQf$rtioMoleDryH2o, setQf$rtioMoleWetH2o, - setQf$presCrdCo2, setQf$tempCrdCo2, - setQf$tempWbox, setQf$sensCrdCo2, - setQf$frt00Mfm, setQf$frtMfm, - setQf$presAtmMfm, setQf$tempMfm, - setQf$sensMfm, setQf$presInlt, - setQf$pumpStor, setQf$heatInlt, - idGas = idGas - ) - rpt$rtioMoleDryH2o <- na.omit(tmp$rtioMoleDryH2o[which(tmp$rtioMoleDryH2o$idGas == 11 | (is.na(tmp$rtioMoleDryH2o$idGas) & tmp$rtioMoleDryH2o$qfSensStus == -1)),]) - - rpt$temp <- na.omit(data.frame(setQf$tempCrdCo2, setQf$sensCrdCo2)) - rpt$pres <- na.omit(data.frame(setQf$presCrdCo2, setQf$sensCrdCo2)) - rpt$presEnvHut <- na.omit(data.frame(setQf$presEnvHut)) - rpt$rhEnvHut <- na.omit(data.frame (setQf$rhEnvHut)) - rpt$tempEnvHut <- na.omit(data.frame (setQf$tempEnvHut)) - rpt$rtioMoleWetH2oEnvHut <- na.omit(data.frame (setQf$rtioMoleWetH2oEnvHut)) - #remove idGas column - lapply(names(rpt), function(x){ - rpt[[x]] <<- rpt[[x]][, ! names(rpt[[x]]) %in% "idGas", drop = F] - }) - #remove tmp - rm(tmp) - } else { - #grouping qulity flags that related to isoCo2 L1 sub-data product - rpt$rtioMoleWetCo2 <- na.omit(data.frame(setQf$rtioMoleWetCo2, setQf$dlta13CCo2, - setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00Mfm, - setQf$frtMfm, setQf$presAtmMfm, - setQf$tempMfm,setQf$sensMfm, - setQf$presInlt, setQf$pumpStor, - setQf$heatInlt - )) - - rpt$rtioMoleDryCo2 <- na.omit(data.frame(setQf$rtioMoleDryCo2, setQf$dlta13CCo2, - setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00Mfm, - setQf$frtMfm, setQf$presAtmMfm, - setQf$tempMfm, setQf$sensMfm, - setQf$presInlt, setQf$pumpStor, - setQf$heatInlt - )) - - rpt$rtioMoleWet12CCo2 <- na.omit(data.frame(setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00Mfm, - setQf$frtMfm, setQf$presAtmMfm, - setQf$tempMfm, setQf$sensMfm, - setQf$presInlt, setQf$pumpStor, - setQf$heatInlt - )) - - rpt$rtioMoleDry12CCo2 <- na.omit(data.frame(setQf$rtioMoleDry12CCo2, setQf$rtioMoleWet13CCo2, - setQf$presCrdCo2, setQf$tempCrdCo2, - setQf$tempWbox, setQf$sensCrdCo2, - setQf$frt00Mfm, setQf$frtMfm, - setQf$presAtmMfm, setQf$tempMfm, - setQf$sensMfm, setQf$presInlt, - setQf$pumpStor, setQf$heatInlt - )) - - rpt$rtioMoleWet13CCo2 <- na.omit(data.frame(setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00Mfm, - setQf$frtMfm, setQf$presAtmMfm, - setQf$tempMfm, setQf$sensMfm, - setQf$presInlt, setQf$pumpStor, - setQf$heatInlt - )) - - rpt$rtioMoleDry13CCo2 <- na.omit(data.frame(setQf$rtioMoleDry13CCo2, setQf$rtioMoleWet13CCo2, - setQf$presCrdCo2, setQf$tempCrdCo2, - setQf$tempWbox, setQf$sensCrdCo2, - setQf$frt00Mfm, setQf$frtMfm, - setQf$presAtmMfm, setQf$tempMfm, - setQf$sensMfm, setQf$presInlt, - setQf$pumpStor, setQf$heatInlt - )) - - rpt$dlta13CCo2 <- na.omit(data.frame(setQf$dlta13CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00Mfm, - setQf$frtMfm, setQf$presAtmMfm, - setQf$tempMfm, setQf$sensMfm, - setQf$presInlt, setQf$pumpStor, - setQf$heatInlt - )) - - rpt$rtioMoleWetH2o <- na.omit(data.frame(setQf$rtioMoleWetH2o, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00Mfm, - setQf$frtMfm, setQf$presAtmMfm, - setQf$tempMfm, setQf$sensMfm, - setQf$presInlt, setQf$pumpStor, - setQf$heatInlt - )) - - rpt$rtioMoleDryH2o <- na.omit(data.frame(setQf$rtioMoleDryH2o, setQf$rtioMoleWetH2o, - setQf$presCrdCo2, setQf$tempCrdCo2, - setQf$tempWbox, setQf$sensCrdCo2, - setQf$frt00Mfm, setQf$frtMfm, - setQf$presAtmMfm, setQf$tempMfm, - setQf$sensMfm, setQf$presInlt, - setQf$pumpStor, setQf$heatInlt - )) - - rpt$temp <- na.omit(data.frame(setQf$tempCrdCo2, setQf$sensCrdCo2)) - rpt$pres <- na.omit(data.frame(setQf$presCrdCo2, setQf$sensCrdCo2)) - rpt$presEnvHut <- na.omit(data.frame(setQf$presEnvHut)) - rpt$rhEnvHut <- na.omit(data.frame (setQf$rhEnvHut)) - rpt$tempEnvHut <- na.omit(data.frame (setQf$tempEnvHut)) - rpt$rtioMoleWetH2oEnvHut <- na.omit(data.frame (setQf$rtioMoleWetH2oEnvHut)) - }# close else statement - - }#close if statement of TypeMeas == "samp" - - #define qf which use only validation period - if (TypeMeas == "vali") { - #temporary list - tmp <- list() - #if not all idGas = NA or all qfRngTemp = NA - if (length(which(!is.na(idGas))) > 0){ - #grouping qulity flags that related to isoCo2 L1 sub-data product - tmp$rtioMoleWetCo2 <- data.frame(setQf$rtioMoleWetCo2, setQf$dlta13CCo2, - setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00MfcVali, - setQf$frtMfcVali, setQf$presAtmMfcVali, - setQf$tempMfcVali,setQf$sensMfcVali, - setQf$presValiRegInStor, idGas = idGas) - rpt$rtioMoleWetCo2 <- na.omit(tmp$rtioMoleWetCo2[which(tmp$rtioMoleWetCo2$idGas == 105 | (is.na(tmp$rtioMoleWetCo2$idGas) & tmp$rtioMoleWetCo2$qfSensStus == -1)),]) - - tmp$rtioMoleDryCo2<- data.frame(setQf$rtioMoleDryCo2, setQf$dlta13CCo2, - setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, + + ) + rpt$rtioMoleDry13CCo2 <- na.omit(tmp$rtioMoleDry13CCo2[which(tmp$rtioMoleDry13CCo2$idGas == 105 | (is.na(tmp$rtioMoleDry13CCo2$idGas) & tmp$rtioMoleDry13CCo2$qfSensStus == -1)),]) + + tmp$dlta13CCo2 <- data.frame(setQf$dlta13CCo2, setQf$rtioMoleWet12CCo2, + setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00MfcVali, - setQf$frtMfcVali, setQf$presAtmMfcVali, - setQf$tempMfcVali, setQf$sensMfcVali, - setQf$presValiRegInStor, idGas = idGas) - rpt$rtioMoleDryCo2 <- na.omit(tmp$rtioMoleDryCo2[which(tmp$rtioMoleDryCo2$idGas == 105 | (is.na(tmp$rtioMoleDryCo2$idGas) & tmp$rtioMoleDryCo2$qfSensStus == -1)),]) - - tmp$rtioMoleWet12CCo2 <- data.frame(setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00MfcVali, - setQf$frtMfcVali, setQf$presAtmMfcVali, - setQf$tempMfcVali, setQf$sensMfcVali, - setQf$presValiRegInStor, idGas = idGas) - rpt$rtioMoleWet12CCo2 <- na.omit(tmp$rtioMoleWet12CCo2[which(tmp$rtioMoleWet12CCo2$idGas == 105 | (is.na(tmp$rtioMoleWet12CCo2$idGas) & tmp$rtioMoleWet12CCo2$qfSensStus == -1)),]) - - tmp$rtioMoleDry12CCo2 <- data.frame(setQf$rtioMoleDry12CCo2, setQf$rtioMoleWet13CCo2, + setQf$sensCrdCo2, + #setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm, + #setQf$presInlt, setQf$pumpStor, + #setQf$heatInlt, + idGas = idGas + + ) + rpt$dlta13CCo2 <- na.omit(tmp$dlta13CCo2[which(tmp$dlta13CCo2$idGas == 105 | (is.na(tmp$dlta13CCo2$idGas) & tmp$dlta13CCo2$qfSensStus == -1)),]) + + tmp$rtioMoleWetH2o <- data.frame(setQf$rtioMoleWetH2o, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, + #setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm, + #setQf$presInlt, setQf$pumpStor, + #setQf$heatInlt, + idGas = idGas + + ) + rpt$rtioMoleWetH2o <- na.omit(tmp$rtioMoleWetH2o[which(tmp$rtioMoleWetH2o$idGas == 11 | (is.na(tmp$rtioMoleWetH2o$idGas) & tmp$rtioMoleWetH2o$qfSensStus == -1)),]) + + tmp$rtioMoleDryH2o <- data.frame(setQf$rtioMoleDryH2o, setQf$rtioMoleWetH2o, setQf$presCrdCo2, setQf$tempCrdCo2, - setQf$tempWbox, setQf$sensCrdCo2, - setQf$frt00MfcVali, setQf$frtMfcVali, - setQf$presAtmMfcVali, setQf$tempMfcVali, - setQf$sensMfcVali, setQf$presValiRegInStor, - idGas = idGas) - rpt$rtioMoleDry12CCo2 <- na.omit(tmp$rtioMoleDry12CCo2[which(tmp$rtioMoleDry12CCo2$idGas == 105 | (is.na(tmp$rtioMoleDry12CCo2$idGas) & tmp$rtioMoleDry12CCo2$qfSensStus == -1)),]) + setQf$tempWbox, setQf$sensCrdCo2, + #setQf$frt00Mfm, setQf$frtMfm, + #setQf$presAtmMfm, setQf$tempMfm, + setQf$sensMfm, + #setQf$presInlt, + #setQf$pumpStor, + #setQf$heatInlt, + idGas = idGas + ) + rpt$rtioMoleDryH2o <- na.omit(tmp$rtioMoleDryH2o[which(tmp$rtioMoleDryH2o$idGas == 11 | (is.na(tmp$rtioMoleDryH2o$idGas) & tmp$rtioMoleDryH2o$qfSensStus == -1)),]) + + rpt$temp <- na.omit(data.frame(setQf$tempCrdCo2, setQf$sensCrdCo2)) + rpt$pres <- na.omit(data.frame(setQf$presCrdCo2, setQf$sensCrdCo2)) + rpt$presEnvHut <- na.omit(data.frame(setQf$presEnvHut)) + rpt$rhEnvHut <- na.omit(data.frame (setQf$rhEnvHut)) + rpt$tempEnvHut <- na.omit(data.frame (setQf$tempEnvHut)) + rpt$rtioMoleWetH2oEnvHut <- na.omit(data.frame (setQf$rtioMoleWetH2oEnvHut)) + #remove idGas column + lapply(names(rpt), function(x){ + rpt[[x]] <<- rpt[[x]][, ! names(rpt[[x]]) %in% "idGas", drop = F] + }) + #remove tmp + rm(tmp) + } else { + #grouping qulity flags that related to isoCo2 L1 sub-data product + rpt$rtioMoleWetCo2 <- na.omit(data.frame(setQf$rtioMoleWetCo2, setQf$rtioMoleWet12CCo2, #setQf$dlta13CCo2, + setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, + #setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm#, + #setQf$presInlt, setQf$pumpStor + #setQf$heatInlt + )) + + rpt$rtioMoleDryCo2 <- na.omit(data.frame(setQf$rtioMoleDryCo2, setQf$rtioMoleDry12CCo2, #setQf$dlta13CCo2, + setQf$rtioMoleDry13CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, + #setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm#, + #setQf$presInlt, setQf$pumpStor + #setQf$heatInlt + )) + + rpt$rtioMoleWet12CCo2 <- na.omit(data.frame(setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, + #setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm#, + #setQf$presInlt, setQf$pumpStor + #setQf$heatInlt + )) + + rpt$rtioMoleDry12CCo2 <- na.omit(data.frame(setQf$rtioMoleDry12CCo2, setQf$rtioMoleWet12CCo2, + setQf$presCrdCo2, setQf$tempCrdCo2, + setQf$tempWbox, setQf$sensCrdCo2, + #setQf$frt00Mfm, setQf$frtMfm, + #setQf$presAtmMfm, setQf$tempMfm, + setQf$sensMfm#, setQf$presInlt, + #setQf$pumpStor + #setQf$heatInlt + )) + + rpt$rtioMoleWet13CCo2 <- na.omit(data.frame(setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, #setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm#, + #setQf$presInlt, setQf$pumpStor + #setQf$heatInlt + )) + + rpt$rtioMoleDry13CCo2 <- na.omit(data.frame(setQf$rtioMoleDry13CCo2, setQf$rtioMoleWet13CCo2, + setQf$presCrdCo2, setQf$tempCrdCo2, + setQf$tempWbox, setQf$sensCrdCo2, + #setQf$frt00Mfm, setQf$frtMfm, + #setQf$presAtmMfm, setQf$tempMfm, + setQf$sensMfm#, setQf$presInlt, + #setQf$pumpStor + #setQf$heatInlt + )) + + rpt$dlta13CCo2 <- na.omit(data.frame(setQf$dlta13CCo2, setQf$rtioMoleWet12CCo2, + setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2,# setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm#, + #setQf$presInlt, setQf$pumpStor + #setQf$heatInlt + )) + + rpt$rtioMoleWetH2o <- na.omit(data.frame(setQf$rtioMoleWetH2o, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, #setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm#, + #setQf$presInlt, setQf$pumpStor + #setQf$heatInlt + )) + + rpt$rtioMoleDryH2o <- na.omit(data.frame(setQf$rtioMoleDryH2o, setQf$rtioMoleWetH2o, + setQf$presCrdCo2, setQf$tempCrdCo2, + setQf$tempWbox, setQf$sensCrdCo2, + #setQf$frt00Mfm, setQf$frtMfm, + #setQf$presAtmMfm, setQf$tempMfm, + setQf$sensMfm#, setQf$presInlt, + #setQf$pumpStor + #setQf$heatInlt + )) + + rpt$temp <- na.omit(data.frame(setQf$tempCrdCo2, setQf$sensCrdCo2)) + rpt$pres <- na.omit(data.frame(setQf$presCrdCo2, setQf$sensCrdCo2)) + rpt$presEnvHut <- na.omit(data.frame(setQf$presEnvHut)) + rpt$rhEnvHut <- na.omit(data.frame (setQf$rhEnvHut)) + rpt$tempEnvHut <- na.omit(data.frame (setQf$tempEnvHut)) + rpt$rtioMoleWetH2oEnvHut <- na.omit(data.frame (setQf$rtioMoleWetH2oEnvHut)) + }# close else statement - tmp$rtioMoleWet13CCo2 <- data.frame(setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, + }#close if statement of TypeMeas == "samp" + + #define qf which use only validation period + if (TypeMeas == "vali") { + #temporary list + tmp <- list() + #if not all idGas = NA or all qfRngTemp = NA + if (length(which(!is.na(idGas))) > 0){ + #grouping qulity flags that related to isoCo2 L1 sub-data product + tmp$rtioMoleWetCo2 <- data.frame(setQf$rtioMoleWetCo2, setQf$rtioMoleWet12CCo2, #setQf$dlta13CCo2, + setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00MfcVali, - setQf$frtMfcVali, setQf$presAtmMfcVali, - setQf$tempMfcVali, setQf$sensMfcVali, - setQf$presValiRegInStor, idGas = idGas) - rpt$rtioMoleWet13CCo2 <- na.omit(tmp$rtioMoleWet13CCo2[which(tmp$rtioMoleWet13CCo2$idGas == 105 | (is.na(tmp$rtioMoleWet13CCo2$idGas) & tmp$rtioMoleWet13CCo2$qfSensStus == -1)),]) - - tmp$rtioMoleDry13CCo2 <- data.frame(setQf$rtioMoleDry13CCo2, setQf$rtioMoleWet13CCo2, - setQf$presCrdCo2, setQf$tempCrdCo2, - setQf$tempWbox, setQf$sensCrdCo2, - setQf$frt00MfcVali, setQf$frtMfcVali, - setQf$presAtmMfcVali, setQf$tempMfcVali, - setQf$sensMfcVali, setQf$presValiRegInStor, - idGas = idGas) - rpt$rtioMoleDry13CCo2 <- na.omit(tmp$rtioMoleDry13CCo2[which(tmp$rtioMoleDry13CCo2$idGas == 105 | (is.na(tmp$rtioMoleDry13CCo2$idGas) & tmp$rtioMoleDry13CCo2$qfSensStus == -1)),]) - - tmp$dlta13CCo2 <- data.frame(setQf$dlta13CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00MfcVali, - setQf$frtMfcVali, setQf$presAtmMfcVali, - setQf$tempMfcVali, setQf$sensMfcVali, - setQf$presValiRegInStor, idGas = idGas) - rpt$dlta13CCo2 <- na.omit(tmp$dlta13CCo2[which(tmp$dlta13CCo2$idGas == 105 | (is.na(tmp$dlta13CCo2$idGas) & tmp$dlta13CCo2$qfSensStus == -1)),]) - - tmp$rtioMoleWetH2o <- data.frame(setQf$rtioMoleWetH2o, setQf$presCrdCo2, + setQf$sensCrdCo2,# setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali, + #setQf$presValiRegInStor, + idGas = idGas) + rpt$rtioMoleWetCo2 <- na.omit(tmp$rtioMoleWetCo2[which(tmp$rtioMoleWetCo2$idGas == 105 | (is.na(tmp$rtioMoleWetCo2$idGas) & tmp$rtioMoleWetCo2$qfSensStus == -1)),]) + + tmp$rtioMoleDryCo2<- data.frame(setQf$rtioMoleDryCo2, setQf$rtioMoleDry12CCo2, #setQf$dlta13CCo2, + setQf$rtioMoleDry13CCo2, setQf$presCrdCo2, setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00MfcVali, - setQf$frtMfcVali, setQf$presAtmMfcVali, - setQf$tempMfcVali, setQf$sensMfcVali, - setQf$presValiRegInStor, idGas = idGas) - rpt$rtioMoleWetH2o <- na.omit(tmp$rtioMoleWetH2o[which(tmp$rtioMoleWetH2o$idGas == 11 | (is.na(tmp$rtioMoleWetH2o$idGas) & tmp$rtioMoleWetH2o$qfSensStus == -1)),]) - - tmp$rtioMoleDryH2o <- data.frame(setQf$rtioMoleDryH2o, setQf$rtioMoleWetH2o, - setQf$presCrdCo2, setQf$tempCrdCo2, - setQf$tempWbox, setQf$sensCrdCo2, - setQf$frt00MfcVali, setQf$frtMfcVali, - setQf$presAtmMfcVali, setQf$tempMfcVali, - setQf$sensMfcVali, setQf$presValiRegInStor, + setQf$sensCrdCo2, #setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali, + #setQf$presValiRegInStor, idGas = idGas) - rpt$rtioMoleDryH2o <- na.omit(tmp$rtioMoleDryH2o[which(tmp$rtioMoleDryH2o$idGas == 11 | (is.na(tmp$rtioMoleDryH2o$idGas) & tmp$rtioMoleDryH2o$qfSensStus == -1)),]) - - rpt$temp <- na.omit(data.frame(setQf$tempCrdCo2, setQf$sensCrdCo2)) - - rpt$pres <- na.omit(data.frame(setQf$presCrdCo2, setQf$sensCrdCo2)) - rpt$presEnvHut <- na.omit(data.frame(setQf$presEnvHut)) - rpt$rhEnvHut <- na.omit(data.frame (setQf$rhEnvHut)) - rpt$tempEnvHut <- na.omit(data.frame (setQf$tempEnvHut)) - rpt$rtioMoleWetH2oEnvHut <- na.omit(data.frame (setQf$rtioMoleWetH2oEnvHut)) - #remove idGas column - lapply(names(rpt), function(x){ - rpt[[x]] <<- rpt[[x]][, ! names(rpt[[x]]) %in% "idGas", drop = F] - }) - #remove tmp - rm(tmp) - } else { - #grouping qulity flags that related to isoCo2 L1 sub-data product - rpt$rtioMoleWetCo2 <- na.omit(data.frame(setQf$rtioMoleWetCo2, setQf$dlta13CCo2, - setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00MfcVali, - setQf$frtMfcVali, setQf$presAtmMfcVali, - setQf$tempMfcVali,setQf$sensMfcVali, - setQf$presValiRegInStor)) - - rpt$rtioMoleDryCo2 <- na.omit(data.frame(setQf$rtioMoleDryCo2, setQf$dlta13CCo2, - setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00MfcVali, - setQf$frtMfcVali, setQf$presAtmMfcVali, - setQf$tempMfcVali, setQf$sensMfcVali, - setQf$presValiRegInStor)) - - rpt$rtioMoleWet12CCo2 <- na.omit(data.frame(setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00MfcVali, - setQf$frtMfcVali, setQf$presAtmMfcVali, - setQf$tempMfcVali, setQf$sensMfcVali, - setQf$presValiRegInStor)) - - rpt$rtioMoleDry12CCo2 <- na.omit(data.frame(setQf$rtioMoleDry12CCo2, setQf$rtioMoleWet13CCo2, - setQf$presCrdCo2, setQf$tempCrdCo2, - setQf$tempWbox, setQf$sensCrdCo2, - setQf$frt00MfcVali, setQf$frtMfcVali, - setQf$presAtmMfcVali, setQf$tempMfcVali, - setQf$sensMfcVali, setQf$presValiRegInStor)) - - rpt$rtioMoleWet13CCo2 <- na.omit(data.frame(setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00MfcVali, - setQf$frtMfcVali, setQf$presAtmMfcVali, - setQf$tempMfcVali, setQf$sensMfcVali, - setQf$presValiRegInStor)) - - rpt$rtioMoleDry13CCo2 <- na.omit(data.frame(setQf$rtioMoleDry13CCo2, setQf$rtioMoleWet13CCo2, - setQf$presCrdCo2, setQf$tempCrdCo2, - setQf$tempWbox, setQf$sensCrdCo2, - setQf$frt00MfcVali, setQf$frtMfcVali, - setQf$presAtmMfcVali, setQf$tempMfcVali, - setQf$sensMfcVali, setQf$presValiRegInStor)) - - rpt$dlta13CCo2 <- na.omit(data.frame(setQf$dlta13CCo2, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00MfcVali, - setQf$frtMfcVali, setQf$presAtmMfcVali, - setQf$tempMfcVali, setQf$sensMfcVali, - setQf$presValiRegInStor)) - - rpt$rtioMoleWetH2o <- na.omit(data.frame(setQf$rtioMoleWetH2o, setQf$presCrdCo2, - setQf$tempCrdCo2, setQf$tempWbox, - setQf$sensCrdCo2, setQf$frt00MfcVali, - setQf$frtMfcVali, setQf$presAtmMfcVali, - setQf$tempMfcVali, setQf$sensMfcVali, - setQf$presValiRegInStor)) - - rpt$rtioMoleDryH2o <- na.omit(data.frame(setQf$rtioMoleDryH2o, setQf$rtioMoleWetH2o, - setQf$presCrdCo2, setQf$tempCrdCo2, - setQf$tempWbox, setQf$sensCrdCo2, - setQf$frt00MfcVali, setQf$frtMfcVali, - setQf$presAtmMfcVali, setQf$tempMfcVali, - setQf$sensMfcVali, setQf$presValiRegInStor)) - - rpt$temp <- na.omit(data.frame(setQf$tempCrdCo2, setQf$sensCrdCo2)) + rpt$rtioMoleDryCo2 <- na.omit(tmp$rtioMoleDryCo2[which(tmp$rtioMoleDryCo2$idGas == 105 | (is.na(tmp$rtioMoleDryCo2$idGas) & tmp$rtioMoleDryCo2$qfSensStus == -1)),]) + + tmp$rtioMoleWet12CCo2 <- data.frame(setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, #setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali, + #setQf$presValiRegInStor, + idGas = idGas) + rpt$rtioMoleWet12CCo2 <- na.omit(tmp$rtioMoleWet12CCo2[which(tmp$rtioMoleWet12CCo2$idGas == 105 | (is.na(tmp$rtioMoleWet12CCo2$idGas) & tmp$rtioMoleWet12CCo2$qfSensStus == -1)),]) + + tmp$rtioMoleDry12CCo2 <- data.frame(setQf$rtioMoleDry12CCo2, setQf$rtioMoleWet12CCo2, + setQf$presCrdCo2, setQf$tempCrdCo2, + setQf$tempWbox, setQf$sensCrdCo2, + #setQf$frt00MfcVali, setQf$frtMfcVali, + #setQf$presAtmMfcVali, setQf$tempMfcVali, + setQf$sensMfcVali, #setQf$presValiRegInStor, + idGas = idGas) + rpt$rtioMoleDry12CCo2 <- na.omit(tmp$rtioMoleDry12CCo2[which(tmp$rtioMoleDry12CCo2$idGas == 105 | (is.na(tmp$rtioMoleDry12CCo2$idGas) & tmp$rtioMoleDry12CCo2$qfSensStus == -1)),]) + + tmp$rtioMoleWet13CCo2 <- data.frame(setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, #setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali, + #setQf$presValiRegInStor, + idGas = idGas) + rpt$rtioMoleWet13CCo2 <- na.omit(tmp$rtioMoleWet13CCo2[which(tmp$rtioMoleWet13CCo2$idGas == 105 | (is.na(tmp$rtioMoleWet13CCo2$idGas) & tmp$rtioMoleWet13CCo2$qfSensStus == -1)),]) + + tmp$rtioMoleDry13CCo2 <- data.frame(setQf$rtioMoleDry13CCo2, setQf$rtioMoleWet13CCo2, + setQf$presCrdCo2, setQf$tempCrdCo2, + setQf$tempWbox, setQf$sensCrdCo2, + #setQf$frt00MfcVali, setQf$frtMfcVali, + #setQf$presAtmMfcVali, setQf$tempMfcVali, + setQf$sensMfcVali, #setQf$presValiRegInStor, + idGas = idGas) + rpt$rtioMoleDry13CCo2 <- na.omit(tmp$rtioMoleDry13CCo2[which(tmp$rtioMoleDry13CCo2$idGas == 105 | (is.na(tmp$rtioMoleDry13CCo2$idGas) & tmp$rtioMoleDry13CCo2$qfSensStus == -1)),]) + + tmp$dlta13CCo2 <- data.frame(setQf$dlta13CCo2, setQf$rtioMoleWet12CCo2, + setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, #setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali, + #setQf$presValiRegInStor, + idGas = idGas) + rpt$dlta13CCo2 <- na.omit(tmp$dlta13CCo2[which(tmp$dlta13CCo2$idGas == 105 | (is.na(tmp$dlta13CCo2$idGas) & tmp$dlta13CCo2$qfSensStus == -1)),]) + + tmp$rtioMoleWetH2o <- data.frame(setQf$rtioMoleWetH2o, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, #setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali, + #setQf$presValiRegInStor, + idGas = idGas) + rpt$rtioMoleWetH2o <- na.omit(tmp$rtioMoleWetH2o[which(tmp$rtioMoleWetH2o$idGas == 11 | (is.na(tmp$rtioMoleWetH2o$idGas) & tmp$rtioMoleWetH2o$qfSensStus == -1)),]) + + tmp$rtioMoleDryH2o <- data.frame(setQf$rtioMoleDryH2o, setQf$rtioMoleWetH2o, + setQf$presCrdCo2, setQf$tempCrdCo2, + setQf$tempWbox, setQf$sensCrdCo2, + #setQf$frt00MfcVali, setQf$frtMfcVali, + #setQf$presAtmMfcVali, setQf$tempMfcVali, + setQf$sensMfcVali, #setQf$presValiRegInStor, + idGas = idGas) + rpt$rtioMoleDryH2o <- na.omit(tmp$rtioMoleDryH2o[which(tmp$rtioMoleDryH2o$idGas == 11 | (is.na(tmp$rtioMoleDryH2o$idGas) & tmp$rtioMoleDryH2o$qfSensStus == -1)),]) + + rpt$temp <- na.omit(data.frame(setQf$tempCrdCo2, setQf$sensCrdCo2)) + + rpt$pres <- na.omit(data.frame(setQf$presCrdCo2, setQf$sensCrdCo2)) + rpt$presEnvHut <- na.omit(data.frame(setQf$presEnvHut)) + rpt$rhEnvHut <- na.omit(data.frame (setQf$rhEnvHut)) + rpt$tempEnvHut <- na.omit(data.frame (setQf$tempEnvHut)) + rpt$rtioMoleWetH2oEnvHut <- na.omit(data.frame (setQf$rtioMoleWetH2oEnvHut)) + #remove idGas column + lapply(names(rpt), function(x){ + rpt[[x]] <<- rpt[[x]][, ! names(rpt[[x]]) %in% "idGas", drop = F] + }) + #remove tmp + rm(tmp) + } else { + #grouping qulity flags that related to isoCo2 L1 sub-data product + rpt$rtioMoleWetCo2 <- na.omit(data.frame(setQf$rtioMoleWetCo2, setQf$rtioMoleWet12CCo2,#setQf$dlta13CCo2, + setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, #setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali#, + #setQf$presValiRegInStor + )) + + rpt$rtioMoleDryCo2 <- na.omit(data.frame(setQf$rtioMoleDryCo2, setQf$rtioMoleDry12CCo2,#setQf$dlta13CCo2, + setQf$rtioMoleDry13CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, #setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali#, + #setQf$presValiRegInStor + )) + + rpt$rtioMoleWet12CCo2 <- na.omit(data.frame(setQf$rtioMoleWet12CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, #setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali#, + #setQf$presValiRegInStor + )) + + rpt$rtioMoleDry12CCo2 <- na.omit(data.frame(setQf$rtioMoleDry12CCo2, setQf$rtioMoleWet12CCo2, + setQf$presCrdCo2, setQf$tempCrdCo2, + setQf$tempWbox, setQf$sensCrdCo2, + #setQf$frt00MfcVali, setQf$frtMfcVali, + #setQf$presAtmMfcVali, setQf$tempMfcVali, + setQf$sensMfcVali#, setQf$presValiRegInStor + )) + + rpt$rtioMoleWet13CCo2 <- na.omit(data.frame(setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, #setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali#, + #setQf$presValiRegInStor + )) + + rpt$rtioMoleDry13CCo2 <- na.omit(data.frame(setQf$rtioMoleDry13CCo2, setQf$rtioMoleWet13CCo2, + setQf$presCrdCo2, setQf$tempCrdCo2, + setQf$tempWbox, setQf$sensCrdCo2, + #setQf$frt00MfcVali, setQf$frtMfcVali, + #setQf$presAtmMfcVali, setQf$tempMfcVali, + setQf$sensMfcVali#, setQf$presValiRegInStor + )) + + rpt$dlta13CCo2 <- na.omit(data.frame(setQf$dlta13CCo2, setQf$rtioMoleWet12CCo2, + setQf$rtioMoleWet13CCo2, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, #setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali#, + #setQf$presValiRegInStor + )) + + rpt$rtioMoleWetH2o <- na.omit(data.frame(setQf$rtioMoleWetH2o, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, #setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali#, + #setQf$presValiRegInStor + )) + + rpt$rtioMoleDryH2o <- na.omit(data.frame(setQf$rtioMoleDryH2o, setQf$rtioMoleWetH2o, + setQf$presCrdCo2, setQf$tempCrdCo2, + setQf$tempWbox, setQf$sensCrdCo2, + #setQf$frt00MfcVali, setQf$frtMfcVali, + #setQf$presAtmMfcVali, setQf$tempMfcVali, + setQf$sensMfcVali#, setQf$presValiRegInStor + )) + + rpt$temp <- na.omit(data.frame(setQf$tempCrdCo2, setQf$sensCrdCo2)) + + rpt$pres <- na.omit(data.frame(setQf$presCrdCo2, setQf$sensCrdCo2)) + rpt$presEnvHut <- na.omit(data.frame(setQf$presEnvHut)) + rpt$rhEnvHut <- na.omit(data.frame (setQf$rhEnvHut)) + rpt$tempEnvHut <- na.omit(data.frame (setQf$tempEnvHut)) + rpt$rtioMoleWetH2oEnvHut <- na.omit(data.frame (setQf$rtioMoleWetH2oEnvHut)) + }#close else statement + }##close if statement of TypeMeas == "vali" + }#close if statment if isoCo2 is in dp01 + + #grouping quality flags related to the ch4Conc L1 data product + + if (dp01 %in% "ch4Conc") { + #define qf which use only sampling period + if (TypeMeas == "samp") { + #temporary list + tmp <- list() + #if not all idGas = NA or all qfRngTemp = NA + if (length(which(!is.na(idGas))) > 0){ + #grouping qulity flags that related to isoCo2 L1 sub-data product + tmp$rtioMoleWetCh4 <- data.frame(setQf$rtioMoleWetCh4, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, + #setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm, + #setQf$presInlt, setQf$pumpStor, + #setQf$heatInlt, + idGas = idGas + ) + rpt$rtioMoleWetCh4 <- na.omit(tmp$rtioMoleWetCh4[which(tmp$rtioMoleWetCh4$idGas == 25 | (is.na(tmp$rtioMoleWetCh4$idGas) & tmp$rtioMoleWetCh4$qfSensStus == -1)),]) + + tmp$rtioMoleDryCh4 <- data.frame(setQf$rtioMoleDryCh4, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, + #setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm, + #setQf$presInlt, setQf$pumpStor, + #setQf$heatInlt, + idGas = idGas + + ) + rpt$rtioMoleDryCh4 <- na.omit(tmp$rtioMoleDryCh4[which(tmp$rtioMoleDryCh4$idGas == 25 | (is.na(tmp$rtioMoleDryCh4$idGas) & tmp$rtioMoleDryCh4$qfSensStus == -1)),]) + + + + rpt$temp <- na.omit(data.frame(setQf$tempCrdCo2, setQf$sensCrdCo2)) + rpt$pres <- na.omit(data.frame(setQf$presCrdCo2, setQf$sensCrdCo2)) + rpt$presEnvHut <- na.omit(data.frame(setQf$presEnvHut)) + rpt$rhEnvHut <- na.omit(data.frame (setQf$rhEnvHut)) + rpt$tempEnvHut <- na.omit(data.frame (setQf$tempEnvHut)) + rpt$rtioMoleWetH2oEnvHut <- na.omit(data.frame (setQf$rtioMoleWetH2oEnvHut)) + #remove idGas column + lapply(names(rpt), function(x){ + rpt[[x]] <<- rpt[[x]][, ! names(rpt[[x]]) %in% "idGas", drop = F] + }) + #remove tmp + rm(tmp) + } else { + #grouping qulity flags that related to ch4Conc L1 sub-data product + rpt$rtioMoleWetCh4 <- na.omit(data.frame(setQf$rtioMoleWetCh4, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, + #setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm#, + #setQf$presInlt, setQf$pumpStor + #setQf$heatInlt + )) + + rpt$rtioMoleDryCh4 <- na.omit(data.frame(setQf$rtioMoleDryCh4, setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2, + #setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm#, + #setQf$presInlt, setQf$pumpStor + #setQf$heatInlt + )) + + + rpt$temp <- na.omit(data.frame(setQf$tempCrdCo2, setQf$sensCrdCo2)) + rpt$pres <- na.omit(data.frame(setQf$presCrdCo2, setQf$sensCrdCo2)) + rpt$presEnvHut <- na.omit(data.frame(setQf$presEnvHut)) + rpt$rhEnvHut <- na.omit(data.frame (setQf$rhEnvHut)) + rpt$tempEnvHut <- na.omit(data.frame (setQf$tempEnvHut)) + rpt$rtioMoleWetH2oEnvHut <- na.omit(data.frame (setQf$rtioMoleWetH2oEnvHut)) + }# close else statement - rpt$pres <- na.omit(data.frame(setQf$presCrdCo2, setQf$sensCrdCo2)) - rpt$presEnvHut <- na.omit(data.frame(setQf$presEnvHut)) - rpt$rhEnvHut <- na.omit(data.frame (setQf$rhEnvHut)) - rpt$tempEnvHut <- na.omit(data.frame (setQf$tempEnvHut)) - rpt$rtioMoleWetH2oEnvHut <- na.omit(data.frame (setQf$rtioMoleWetH2oEnvHut)) - }#close else statement - }##close if statement of TypeMeas == "vali" + }#close if statement of TypeMeas == "samp" + + #define qf which use only validation period + if (TypeMeas == "vali") { + #temporary list + tmp <- list() + #if not all idGas = NA or all qfRngTemp = NA + if (length(which(!is.na(idGas))) > 0){ + #grouping qulity flags that related to ch4Conc L1 sub-data product + tmp$rtioMoleWetCh4 <- data.frame(setQf$rtioMoleWetCh4, + setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2,# setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali, + #setQf$presValiRegInStor, + idGas = idGas) + rpt$rtioMoleWetCh4 <- na.omit(tmp$rtioMoleWetCh4[which(tmp$rtioMoleWetCh4$idGas == 25 | (is.na(tmp$rtioMoleWetCh4$idGas) & tmp$rtioMoleWetCh4$qfSensStus == -1)),]) + + tmp$rtioMoleDryCh4 <- data.frame(setQf$rtioMoleDryCh4, + setQf$presCrdCo2, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$sensCrdCo2,# setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali, + #setQf$presValiRegInStor, + idGas = idGas) + rpt$rtioMoleDryCh4 <- na.omit(tmp$rtioMoleDryCh4[which(tmp$rtioMoleDryCh4$idGas == 25 | (is.na(tmp$rtioMoleDryCh4$idGas) & tmp$rtioMoleDryCh4$qfSensStus == -1)),]) + + + rpt$temp <- na.omit(data.frame(setQf$tempCrdCo2, setQf$sensCrdCo2)) + rpt$pres <- na.omit(data.frame(setQf$presCrdCo2, setQf$sensCrdCo2)) + rpt$presEnvHut <- na.omit(data.frame(setQf$presEnvHut)) + rpt$rhEnvHut <- na.omit(data.frame (setQf$rhEnvHut)) + rpt$tempEnvHut <- na.omit(data.frame (setQf$tempEnvHut)) + rpt$rtioMoleWetH2oEnvHut <- na.omit(data.frame (setQf$rtioMoleWetH2oEnvHut)) + #remove idGas column + lapply(names(rpt), function(x){ + rpt[[x]] <<- rpt[[x]][, ! names(rpt[[x]]) %in% "idGas", drop = F] + }) + #remove tmp + rm(tmp) + } else { + #grouping qulity flags that related to ch4Conc L1 sub-data product + rpt$rtioMoleWetCh4 <- na.omit(data.frame(setQf$rtioMoleWetCh4, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$presCrdCo2, + setQf$sensCrdCo2, #setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali#, + #setQf$presValiRegInStor + )) + + rpt$rtioMoleDryCh4 <- na.omit(data.frame(setQf$rtioMoleDryCh4, + setQf$tempCrdCo2, setQf$tempWbox, + setQf$presCrdCo2, + setQf$sensCrdCo2, #setQf$frt00MfcVali, + #setQf$frtMfcVali, setQf$presAtmMfcVali, + #setQf$tempMfcVali, + setQf$sensMfcVali#, + #setQf$presValiRegInStor + )) + + rpt$temp <- na.omit(data.frame(setQf$tempCrdCo2, setQf$sensCrdCo2)) + + rpt$pres <- na.omit(data.frame(setQf$presCrdCo2, setQf$sensCrdCo2)) + rpt$presEnvHut <- na.omit(data.frame(setQf$presEnvHut)) + rpt$rhEnvHut <- na.omit(data.frame (setQf$rhEnvHut)) + rpt$tempEnvHut <- na.omit(data.frame (setQf$tempEnvHut)) + rpt$rtioMoleWetH2oEnvHut <- na.omit(data.frame (setQf$rtioMoleWetH2oEnvHut)) + }#close else statement + }##close if statement of TypeMeas == "vali" + }#close if statment if ch4Conc is in dp01 #remove setQf setQf <- NULL - }##close if statement of dp01 == isoCo2 - + }##close if statement of dp01 %in% c("isoCo2", "ch4Conc") + #isoH2o #################################################################################### if (dp01 == "isoH2o") { #check if data are exist @@ -1843,39 +2281,39 @@ if (MethMeas == "ecse") { #setQf for crdH2o setQf$rtioMoleDryH2o <- data.frame("qfRngRtioMoleDryH2o" = qfInp$crdH2o$qfRngRtioMoleDryH2o, - "qfStepRtioMoleDryH2o" = qfInp$crdH2o$qfStepRtioMoleDryH2o, - "qfPersRtioMoleDryH2o" = qfInp$crdH2o$qfPersRtioMoleDryH2o, - "qfCalRtioMoleDryH2o" = qfInp$crdH2o$qfCalRtioMoleDryH2o) + #"qfStepRtioMoleDryH2o" = qfInp$crdH2o$qfStepRtioMoleDryH2o, + "qfPersRtioMoleDryH2o" = qfInp$crdH2o$qfPersRtioMoleDryH2o) + #"qfCalRtioMoleDryH2o" = qfInp$crdH2o$qfCalRtioMoleDryH2o) setQf$rtioMoleWetH2o <- data.frame("qfRngRtioMoleWetH2o" = qfInp$crdH2o$qfRngRtioMoleWetH2o, - "qfStepRtioMoleWetH2o" = qfInp$crdH2o$qfStepRtioMoleWetH2o, - "qfPersRtioMoleWetH2o" = qfInp$crdH2o$qfPersRtioMoleWetH2o, - "qfCalRtioMoleWetH2o" = qfInp$crdH2o$qfCalRtioMoleWetH2o) + #"qfStepRtioMoleWetH2o" = qfInp$crdH2o$qfStepRtioMoleWetH2o, + "qfPersRtioMoleWetH2o" = qfInp$crdH2o$qfPersRtioMoleWetH2o) + #"qfCalRtioMoleWetH2o" = qfInp$crdH2o$qfCalRtioMoleWetH2o) setQf$dlta18OH2o <- data.frame("qfRngDlta18OH2o" = qfInp$crdH2o$qfRngDlta18OH2o, - "qfStepDlta18OH2o" = qfInp$crdH2o$qfStepDlta18OH2o, - "qfPersDlta18OH2o" = qfInp$crdH2o$qfPersDlta18OH2o, - "qfCalDlta18OH2o" = qfInp$crdH2o$qfCalDlta18OH2o) + #"qfStepDlta18OH2o" = qfInp$crdH2o$qfStepDlta18OH2o, + "qfPersDlta18OH2o" = qfInp$crdH2o$qfPersDlta18OH2o) + #"qfCalDlta18OH2o" = qfInp$crdH2o$qfCalDlta18OH2o) setQf$dlta2HH2o <- data.frame("qfRngDlta2HH2o" = qfInp$crdH2o$qfRngDlta2HH2o, - "qfStepDlta2HH2o" = qfInp$crdH2o$qfStepDlta2HH2o, - "qfPersDlta2HH2o" = qfInp$crdH2o$qfPersDlta2HH2o, - "qfCalDlta2HH2o" = qfInp$crdH2o$qfCalDlta2HH2o) + #"qfStepDlta2HH2o" = qfInp$crdH2o$qfStepDlta2HH2o, + "qfPersDlta2HH2o" = qfInp$crdH2o$qfPersDlta2HH2o) + #"qfCalDlta2HH2o" = qfInp$crdH2o$qfCalDlta2HH2o) setQf$presCrdH2o <- data.frame("qfRngPres" = qfInp$crdH2o$qfRngPres, - "qfStepPres" = qfInp$crdH2o$qfStepPres, - "qfPersPres" = qfInp$crdH2o$qfPersPres, - "qfCalPres" = qfInp$crdH2o$qfCalPres) + #"qfStepPres" = qfInp$crdH2o$qfStepPres, + "qfPersPres" = qfInp$crdH2o$qfPersPres) + #"qfCalPres" = qfInp$crdH2o$qfCalPres) setQf$tempCrdH2o <- data.frame("qfRngTemp" = qfInp$crdH2o$qfRngTemp, - "qfStepTemp" = qfInp$crdH2o$qfStepTemp, - "qfPersTemp" = qfInp$crdH2o$qfPersTemp, - "qfCalTemp" = qfInp$crdH2o$qfCalTemp) + #"qfStepTemp" = qfInp$crdH2o$qfStepTemp, + "qfPersTemp" = qfInp$crdH2o$qfPersTemp) + #"qfCalTemp" = qfInp$crdH2o$qfCalTemp) setQf$tempWbox <- data.frame("qfRngTempWbox" = qfInp$crdH2o$qfRngTempWbox, - "qfStepTempWbox" = qfInp$crdH2o$qfStepTempWbox, - "qfPersTempWbox" = qfInp$crdH2o$qfPersTempWbox, - "qfCalTempWbox" = qfInp$crdH2o$qfCalTempWbox) + #"qfStepTempWbox" = qfInp$crdH2o$qfStepTempWbox, + "qfPersTempWbox" = qfInp$crdH2o$qfPersTempWbox) + #"qfCalTempWbox" = qfInp$crdH2o$qfCalTempWbox) setQf$sensCrdH2o <- data.frame("qfSensStus" = qfInp$crdH2o$qfSensStus, "qfStusN2" = qfInp$crdH2o$qfStusN2) @@ -1973,38 +2411,42 @@ if (MethMeas == "ecse") { rpt$rtioMoleDryH2o <- na.omit(data.frame(setQf$rtioMoleDryH2o, setQf$rtioMoleWetH2o, setQf$presCrdH2o, setQf$tempCrdH2o, setQf$tempWbox, setQf$sensCrdH2o, - setQf$envHut, setQf$frt00Mfm, - setQf$frtMfm, setQf$presAtmMfm, - setQf$tempMfm, setQf$sensMfm, - setQf$presInlt, setQf$pumpStor, - setQf$heatInlt + #setQf$envHut, setQf$frt00Mfm, + #setQf$frtMfm, setQf$presAtmMfm, + #setQf$tempMfm, + setQf$sensMfm#, + #setQf$presInlt, setQf$pumpStor + #setQf$heatInlt )) rpt$rtioMoleWetH2o <- na.omit(data.frame(setQf$rtioMoleWetH2o, setQf$presCrdH2o, setQf$tempCrdH2o, setQf$tempWbox, - setQf$sensCrdH2o, setQf$envHut, - setQf$frt00Mfm, setQf$frtMfm, - setQf$presAtmMfm, setQf$tempMfm, - setQf$sensMfm, setQf$presInlt, - setQf$pumpStor, setQf$heatInlt + setQf$sensCrdH2o, #setQf$envHut, + #setQf$frt00Mfm, setQf$frtMfm, + #setQf$presAtmMfm, setQf$tempMfm, + setQf$sensMfm#, setQf$presInlt, + #setQf$pumpStor + #setQf$heatInlt )) rpt$dlta18OH2o <- na.omit(data.frame(setQf$dlta18OH2o, setQf$presCrdH2o, setQf$tempCrdH2o, setQf$tempWbox, - setQf$sensCrdH2o, setQf$envHut, - setQf$frt00Mfm, setQf$frtMfm, - setQf$presAtmMfm, setQf$tempMfm, - setQf$sensMfm, setQf$presInlt, - setQf$pumpStor, setQf$heatInlt + setQf$sensCrdH2o, #setQf$envHut, + #setQf$frt00Mfm, setQf$frtMfm, + #setQf$presAtmMfm, setQf$tempMfm, + setQf$sensMfm#, setQf$presInlt, + #setQf$pumpStor + #setQf$heatInlt )) rpt$dlta2HH2o <- na.omit(data.frame(setQf$dlta2HH2o, setQf$presCrdH2o, setQf$tempCrdH2o, setQf$tempWbox, - setQf$sensCrdH2o, setQf$envHut, - setQf$frt00Mfm, setQf$frtMfm, - setQf$presAtmMfm, setQf$tempMfm, - setQf$sensMfm, setQf$presInlt, - setQf$pumpStor, setQf$heatInlt + setQf$sensCrdH2o, #setQf$envHut, + #setQf$frt00Mfm, setQf$frtMfm, + #setQf$presAtmMfm, setQf$tempMfm, + setQf$sensMfm#, setQf$presInlt, + #setQf$pumpStor + #setQf$heatInlt )) rpt$pres <- na.omit(data.frame(setQf$presCrdH2o, setQf$sensCrdH2o$qfSensStus)) @@ -2021,26 +2463,31 @@ if (MethMeas == "ecse") { rpt$rtioMoleDryH2o <- na.omit(data.frame(setQf$rtioMoleDryH2o, setQf$rtioMoleWetH2o, setQf$presCrdH2o, setQf$tempCrdH2o, setQf$tempWbox, setQf$sensCrdH2o, - setQf$envHut, setQf$valiCrdH2o, - setQf$presValiRegInStor, setQf$presValiRegOutStor)) + #setQf$envHut, + setQf$valiCrdH2o#, + #setQf$presValiRegInStor, setQf$presValiRegOutStor + )) rpt$rtioMoleWetH2o <- na.omit(data.frame(setQf$rtioMoleWetH2o, setQf$presCrdH2o, setQf$tempCrdH2o, setQf$tempWbox, - setQf$sensCrdH2o, setQf$envHut, - setQf$valiCrdH2o, setQf$presValiRegInStor, - setQf$presValiRegOutStor)) + setQf$sensCrdH2o, #setQf$envHut, + setQf$valiCrdH2o#, setQf$presValiRegInStor, + #setQf$presValiRegOutStor + )) rpt$dlta18OH2o <- na.omit(data.frame(setQf$dlta18OH2o, setQf$presCrdH2o, setQf$tempCrdH2o, setQf$tempWbox, - setQf$sensCrdH2o, setQf$envHut, - setQf$valiCrdH2o, setQf$presValiRegInStor, - setQf$presValiRegOutStor)) + setQf$sensCrdH2o, #setQf$envHut, + setQf$valiCrdH2o#, setQf$presValiRegInStor, + #setQf$presValiRegOutStor + )) rpt$dlta2HH2o <- na.omit(data.frame(setQf$dlta2HH2o, setQf$presCrdH2o, setQf$tempCrdH2o, setQf$tempWbox, - setQf$sensCrdH2o, setQf$envHut, - setQf$valiCrdH2o, setQf$presValiRegInStor, - setQf$presValiRegOutStor)) + setQf$sensCrdH2o, #setQf$envHut, + setQf$valiCrdH2o#, setQf$presValiRegInStor, + #setQf$presValiRegOutStor + )) rpt$pres <- na.omit(data.frame(setQf$presCrdH2o, setQf$sensCrdH2o$qfSensStus, setQf$valiCrdH2o)) rpt$temp <- na.omit(data.frame(setQf$tempCrdH2o, setQf$sensCrdH2o$qfSensStus, setQf$valiCrdH2o)) @@ -2143,8 +2590,23 @@ if (MethMeas == "ecse") { }#close if statement of dp01 == tempAirLvl }# closed if statement of MethMeas == "ecse" +#Create name vector +nameVar <- names(rpt) + +#calculate qmAlpha, qmBeta, qfFinl +rpt <- lapply(names(rpt), function(x){ + nameQfNull <- grep("null", names(rpt[[x]]), ignore.case=TRUE, value = TRUE) + nameQfNullRmv <- grep(x, nameQfNull, ignore.case=TRUE, value = TRUE, invert = TRUE) + outQf <- rpt[[x]][,!names(rpt[[x]]) %in% nameQfNullRmv] + #Return output + return(outQf) + })#End lapply around removing other variables qfNull flags +#Reattribute variable names +names(rpt) <- nameVar + #return values return(rpt) #end function def.dp01.grp.qf() } + diff --git a/pack/eddy4R.qaqc/R/def.plau.R b/pack/eddy4R.qaqc/R/def.plau.R index 3507c57e..e6e12ad4 100644 --- a/pack/eddy4R.qaqc/R/def.plau.R +++ b/pack/eddy4R.qaqc/R/def.plau.R @@ -87,21 +87,34 @@ # applied eddy4R term name convention; replaced posQf by setQf # Natchaya P-Durden (2018-04-11) # applied eddy4R term name convention; replaced pos by idx or set +# Cove Sturtevant (2020-07-07) +# optimized time-based persistence test and removed point-based method, adding conversion to time-based method. +# also caught bug causing a potential lack of persistence test na flags when all values between 2 points are NA. +# Cove Sturtevant (2020-07-08) +# optimized gap test. WAY faster when there are a lot of gaps in the dataset +# Cove Sturtevant (2020-07-09) +# Additional optimization of persistence test achieved by turning time variable to numeric seconds past start +# Cove Sturtevant (2021-06-08) +# Fix bug causing looping (sometimes infinitely) over periods already flagged by persistence test. +# Fix bug truncating fractional seconds from time vector +# adjusted indexing statements to allow input of tibble data frame, which does not automatically drop the data +# frame when accessing a single column using [,]. +# Cove Sturtevant (2021-06-29) +# fix bug in gap test. The code effectively hard-coded the gap test threshold to 4. ############################################################################################## def.plau <- function ( data, # a data frame containing the data to be evaluated (do not include the time stamp vector here). Required input. - time = as.POSIXlt(seq.POSIXt(from=Sys.time(),by="sec",length.out=length(data[,1]))), # time vector corresponding with the rows in data, in Class "POSIXlt", which is a named list of vectors representing sec, min, hour,day,mon,year. Defaults to an evenly spaced time vector starting from execution by seconds. + time = as.POSIXlt(seq.POSIXt(from=Sys.time(),by="sec",length.out=nrow(data))), # time vector corresponding with the rows in data, in Class "POSIXlt", which is a named list of vectors representing sec, min, hour,day,mon,year. Defaults to an evenly spaced time vector starting from execution by seconds. RngMin = apply(data,2,min,na.rm=TRUE), # a numeric vector containing the minimum acceptable value for each variable in data, defaults to observed minimums RngMax = apply(data,2,max,na.rm=TRUE), # a numeric vector containing the maximum acceptable value for each variable in data, defaults to observed maximums DiffStepMax = apply(abs(apply(data,2,diff)),2,max,na.rm=TRUE), # a vector containing the maximum acceptable absolute difference between sequential data points for each variable in data - DiffPersMin = rep.int(0,length(data)), # a vector containing the minimum absolute change in value for each variable in data over the interval specified in WndwPers. Defaults to a vector of zeros. - WndwPers = 60*median(abs(diff(time)),na.rm=TRUE)*rep.int(1,length(data)), # a vector of class difftime specifying the time interval for each variable in data over which to test for the minimum absolute change in value specified in DiffPersMin. Defaults to 60 x median observed time difference. Class difftime can be generated using as.difftime. - TestNull = rep(FALSE,length(data)), # apply the null test? A logical vector of [TRUE or FALSE] of length equal to number of variables in data. Defaults to FALSE (no null values are flagged) - NumGap = rep(length(data[,1])+1,length(data)), # an integer greater than 0 specifying the number of consecutive NA values that constitute a gap + DiffPersMin = rep.int(0,ncol(data)), # a vector containing the minimum absolute change in value for each variable in data over the interval specified in WndwPers. Defaults to a vector of zeros. + WndwPers = 60*median(abs(diff(time)),na.rm=TRUE)*rep.int(1,ncol(data)), # a vector of class difftime specifying the time interval for each variable in data over which to test for the minimum absolute change in value specified in DiffPersMin. Defaults to 60 x median observed time difference. Class difftime can be generated using as.difftime. + TestNull = rep(FALSE,ncol(data)), # apply the null test? A logical vector of [TRUE or FALSE] of length equal to number of variables in data. Defaults to FALSE (no null values are flagged) + NumGap = rep(nrow(data)+1,ncol(data)), # an integer greater than 0 specifying the number of consecutive NA values that constitute a gap Vrbs = FALSE # FALSE = output the vector positions of the fail and na results, TRUE = output flag values for each test ) { - - + # Error Checking ---------------------------------------------------------- # Check data @@ -110,9 +123,9 @@ def.plau <- function ( } # Initial stats - numVar <- length(data) # Get number of variables + numVar <- ncol(data) # Get number of variables nameData <- names(data) # Get variable names - numData <- length(data[,1]) + numData <- nrow(data) # Check time @@ -190,8 +203,10 @@ def.plau <- function ( # intialize output of failed and na vector positions setQf <- vector("list",numVar) # Initialize output for each variable names(setQf) <- nameData - setDataReal <- lapply(data,FUN=function(var){which(!is.na(var))}) - setDataNa <- lapply(setDataReal,FUN=function(var){setdiff(1:numData,var)}) + dataNa <- lapply(data,FUN=function(var){is.na(var)})# logical + dataReal <- lapply(dataNa,FUN=function(var){!var}) # logical + setDataReal <- lapply(dataReal,FUN=function(var){which(var)}) # indices + setDataNa <- lapply(dataNa,FUN=function(var){which(var)}) # indices # For verbose option, initialize output if(Vrbs) { @@ -209,7 +224,7 @@ def.plau <- function ( # Do range test for(idxVar in 1:numVar) { setQf[[idxVar]]$setQfRng <- list(fail=numeric(0),na=numeric(0)) # initialize - setQf[[idxVar]]$setQfRng$fail <- which((data[,idxVar] < RngMin[idxVar]) | (data[,idxVar] > RngMax[idxVar])) + setQf[[idxVar]]$setQfRng$fail <- which((data[[idxVar]] < RngMin[idxVar]) | (data[[idxVar]] > RngMax[idxVar])) setQf[[idxVar]]$setQfRng$na <- setDataNa[[idxVar]] # For Verbose option, output actual flag values @@ -222,19 +237,26 @@ def.plau <- function ( # Do step test for(idxVar in 1:numVar) { + + diffDataIdx <- diff(data[[idxVar]]) + diffDataNaIdx <- is.na(diffDataIdx) + setNaDiffDataIdx <- which(diffDataNaIdx) + setRealDiffDataIdx <- which(!diffDataNaIdx) + + setQf[[idxVar]]$setQfStep <- list(fail=numeric(0),na=numeric(0)) # initialize - setQf[[idxVar]]$setQfStep$fail <- which((abs(diff(data[,idxVar])) > DiffStepMax[idxVar])) - setQf[[idxVar]]$setQfStep$fail <- unique(c(setQf[[idxVar]]$setQfStep$fail,setQf[[idxVar]]$setQfStep$fail+1)) - setQf[[idxVar]]$setQfStep$na <- which(is.na(diff(data[,idxVar])))+1 + setQf[[idxVar]]$setQfStep$fail <- which((abs(diffDataIdx) > DiffStepMax[idxVar])) + setQf[[idxVar]]$setQfStep$fail <- union(setQf[[idxVar]]$setQfStep$fail,setQf[[idxVar]]$setQfStep$fail+1) + setQf[[idxVar]]$setQfStep$na <- setNaDiffDataIdx+1 # If either of the first two values are NULL, flag NA (the first value is missed by the above code) - if(is.na(diff(data[,idxVar])[1])){setQf[[idxVar]]$setQfStep$na <- unique( - c(1,setQf[[idxVar]]$setQfStep$na))} + if(diffDataNaIdx[1]){ + setQf[[idxVar]]$setQfStep$na <- union(1,setQf[[idxVar]]$setQfStep$na) + } # If previous point is null, but next value is present, evaluate the step test with next value - setQf[[idxVar]]$setQfStep$na <- setdiff(setQf[[idxVar]]$setQfStep$na, - which(!is.na(diff(data[,idxVar])))) + setQf[[idxVar]]$setQfStep$na <- setdiff(setQf[[idxVar]]$setQfStep$na,setRealDiffDataIdx) # For Verbose option, output actual flag values if(Vrbs) { @@ -243,260 +265,174 @@ def.plau <- function ( } } - # Do persistence test with time-based windowing approach - if(class(WndwPers) == "difftime"){ + # Do persistence test + if(class(WndwPers) %in% c("numeric","integer")){ + timePers <- seq_len(numData) + } else { + # Convert time to seconds past start. It's faster. + timePers <- as.double(time-time[1], units = "secs") # Turn time to numeric seconds past start + WndwPers <- as.double(WndwPers,units='secs') + } + + for(idxVar in 1:numVar) { - for(idxVar in 1:numVar) { - - dataIdxVar <- data[[idxVar]] - DiffPersMinIdx <- DiffPersMin[idxVar] - WndwPersIdx <- WndwPers[idxVar] - - # Initialize - setQf[[idxVar]]$setQfPers <- list(fail=numeric(0),na=numeric(0)) - setQf[[idxVar]]$setQfPers$na <- which(is.na(dataIdxVar)) - - # Quit if all data are NA - if(length(setDataReal[[idxVar]]) <= 1){ - setQf[[idxVar]]$setQfPers$na <- 1:numData - if(Vrbs){qf[[idxVar]]$qfPers[setQf[[idxVar]]$setQfPers$na] <- -1} - next - } else if(DiffPersMinIdx <= 0){ - if(Vrbs){qf[[idxVar]]$qfPers[setQf[[idxVar]]$setQfPers$na] <- -1} + dataIdxVar <- data[[idxVar]] + dataNaIdx <- dataNa[[idxVar]] + dataRealIdx <- dataReal[[idxVar]] + DiffPersMinIdx <- DiffPersMin[idxVar] + WndwPersIdx <- WndwPers[idxVar] + + # Initialize + setQf[[idxVar]]$setQfPers <- list(fail=numeric(0),na=numeric(0)) + setQf[[idxVar]]$setQfPers$na <- setDataNa[[idxVar]] + + # Quit if all data are NA + if(length(setDataReal[[idxVar]]) <= 1){ + setQf[[idxVar]]$setQfPers$na <- 1:numData + if(Vrbs){qf[[idxVar]]$qfPers[setQf[[idxVar]]$setQfPers$na] <- -1} + next + } else if(DiffPersMinIdx <= 0){ + if(Vrbs){qf[[idxVar]]$qfPers[setQf[[idxVar]]$setQfPers$na] <- -1} + next + } + + # Start at the beginning, making sure we aren't on a null value + idxDataBgn <- setDataReal[[idxVar]][1] + + idxDataMin <- idxDataBgn # initialize index of running min + idxDataMax <- idxDataBgn # intialize index of running max + idxData <- 2 # intialize index position + + # Grab the data for these indices + timeIdxBgn <- timePers[idxDataBgn] + dataIdxMin <- dataIdxVar[idxDataMin] + dataIdxMax <- dataIdxVar[idxDataMax] + + while(idxData <= numData) { + + #If we hit NA, get to the next non-NA value + if(dataNaIdx[idxData]){ + idxData <- idxData +1 next + } + + # Is the value at this index the running max or min? + if(dataIdxVar[idxData] < dataIdxMin) { + idxDataMin <- idxData + dataIdxMin <- dataIdxVar[idxDataMin] + } else if(dataIdxVar[idxData] > dataIdxMax) { + idxDataMax <- idxData + dataIdxMax <- dataIdxVar[idxDataMax] } - # Start at the beginning, making sure we aren't on a null value - idxDataBgn <- setDataReal[[idxVar]][1] + + # Is diff between max and min at or larger than the persistence threshold? + if(dataIdxMax-dataIdxMin >= DiffPersMinIdx) { - idxDataMin <- idxDataBgn # initialize index of running min - idxDataMax <- idxDataBgn # intialize index of running max - idxData <- 2 # intialize index position + # We've hit the threshold, now check whether we are beyond the allowable time interval + if(timePers[idxData]-timeIdxBgn < WndwPersIdx) { + # Hooray! The data is not "stuck" + idxDataBgn <- min(c(idxDataMin,idxDataMax))+1 # set start of next window to the next point after the earlier of the running min and max - # Grab the data for these indices, improves CPU time - dataIdxMin <- dataIdxVar[idxDataMin] - dataIdxMax <- dataIdxVar[idxDataMax] - - while(idxData <= numData) { - - #If we hit NA, get to the next non-NA value - if(is.na(dataIdxVar[idxData])){ - idxData <- idxData +1 - next - } + # Make sure we aren't on a null value + while(dataNaIdx[idxDataBgn]){ + idxDataBgn <- idxDataBgn+1 + } - # Is the value at this index the running max or min? - if(dataIdxVar[idxData] < dataIdxMin) { - idxDataMin <- idxData + idxDataMin <- idxDataBgn # reset running minimum + idxDataMax <- idxDataBgn # reset running maximum + idxData <- idxDataBgn+1 # reset the next point to be evaluated + + # Grab the data for these indices, improves CPU time + timeIdxBgn <- timePers[idxDataBgn] dataIdxMin <- dataIdxVar[idxDataMin] - } else if(dataIdxVar[idxData] > dataIdxMax) { - idxDataMax <- idxData dataIdxMax <- dataIdxVar[idxDataMax] - } - - - # Is diff between max and min at or larger than the persistence threshold? - if(dataIdxMax-dataIdxMin >= DiffPersMinIdx) { - - # We've hit the threshold, now check whether we are beyond the allowable time interval - if(diff(time[c(idxDataBgn,idxData)]) < WndwPersIdx) { - # Hooray! The data is not "stuck" - idxDataBgn <- min(c(idxDataMin,idxDataMax))+1 # set start of next window to the next point after the lower of the running min and max - - # Make sure we aren't on a null value - while(is.na(dataIdxVar[idxDataBgn])){ - idxDataBgn <- idxDataBgn+1 - } - - idxDataMin <- idxDataBgn # reset running minimum - idxDataMax <- idxDataBgn # reset running maximum - idxData <- idxDataBgn+1 # reset the next point to be evaluated + + } else { + + # We might have a stuck sensor, but first let's check whether we blew the time threshold b/c + # all the data were NA prior to this point + if (sum(dataRealIdx[idxDataBgn:(idxData-1)]) <= 1) { - # Grab the data for these indices, improves CPU time - dataIdxMin <- dataIdxVar[idxDataMin] - dataIdxMax <- dataIdxVar[idxDataMax] + # Data were all NA between the starting index and the current point, mark the non-NA points + # as cannot evaluate if they haven't already been marked for test failure + setNaAdd <- setdiff(idxDataBgn:(idxData-1),setQf[[idxVar]]$setQfPers$fail) + setQf[[idxVar]]$setQfPers$na <- union(setQf[[idxVar]]$setQfPers$na,setNaAdd) } else { - # We might have a stuck sensor, but first let's check whether we blew the time threshold b/c - # all the data were NA prior to this point - if (sum(!is.na(idxDataBgn:(idxData-1))) <= 1) { - - # Data were all NA after starting index, mark as cannot evaluate - setQf[[idxVar]]$setQfPers$na <- union(setQf[[idxVar]]$setQfPers$na,idxDataBgn:(idxData-1)) - - } else { - - # Awe bummer, the sensor was stuck before this point. - setQf[[idxVar]]$setQfPers$fail <- unique(c(setQf[[idxVar]]$setQfPers$fail, - idxDataBgn:(idxData-1))) - - # Don't mark the NA values as fail - setQf[[idxVar]]$setQfPers$fail <- setdiff(setQf[[idxVar]]$setQfPers$fail, - setQf[[idxVar]]$setQfPers$na) - } + # Awe bummer, the sensor was stuck before this point. + setQf[[idxVar]]$setQfPers$fail <- union(setQf[[idxVar]]$setQfPers$fail, + idxDataBgn:(idxData-1)) - idxDataBgn <- idxData # restart the test from here - idxData <- idxDataBgn+1 # reset the next point to be evaluated - } - - } else if ((idxData == numData) && (time[idxData]-time[idxDataBgn] >= WndwPersIdx)) { - - # We didn't hit the threshold and we've reached the end of the data. We are also beyond the allowable - # time interval for the persistence test, so let's flag the data - setQf[[idxVar]]$setQfPers$fail <- unique(c(setQf[[idxVar]]$setQfPers$fail,idxDataBgn:idxData)) - - # Don't mark the NA values as fail - setQf[[idxVar]]$setQfPers$fail <- setdiff(setQf[[idxVar]]$setQfPers$fail,setQf[[idxVar]]$setQfPers$na) - - idxData <- idxData+1 # We're done - - } else { - - # We didn't pass the minimum acceptable change on this point, move to the next - idxData <- idxData+1 - } - } - # If we reached the end of the data but the last value was NA, we need to go back and evaluate the last - # non-NA value - idxData <- numData - if (is.na(dataIdxVar[idxData])) { - # Get to last non-NA point - idxData <- tail(setDataReal[[idxVar]],n=1) - - if (time[idxData]-time[idxDataBgn] >= WndwPersIdx) { - # We didn't hit the threshold for the final non-NA points and we were beyond the allowable - # time interval for the persistence test, so let's flag the end of the data - setQf[[idxVar]]$setQfPers$fail <- unique(c(setQf[[idxVar]]$setQfPers$fail,idxDataBgn:idxData)) - - # Don't mark the NA values as fail - setQf[[idxVar]]$setQfPers$fail <- setdiff(setQf[[idxVar]]$setQfPers$fail,setQf[[idxVar]]$setQfPers$na) + } - } else { - # We didn't hit the threshold for the final non-NA points, but we are not yet beyond the - # allowable time interval, so let's flag as unable to evaluate - setQf[[idxVar]]$setQfPers$na <- unique(c(setQf[[idxVar]]$setQfPers$na,idxDataBgn:idxData)) - } - } - - # For Verbose option, output actual flag values - if(Vrbs) { - qf[[idxVar]]$qfPers[setQf[[idxVar]]$setQfPers$fail] <- 1 - qf[[idxVar]]$qfPers[setQf[[idxVar]]$setQfPers$na] <- -1 - } - } - } - - - # Do persistence test with point-based windowing option - if(class(WndwPers) %in% c("numeric","integer")){ - for(idxVar in 1:numVar) { - - dataIdxVar <- data[[idxVar]] # Pull out this variable - DiffPersMinIdx <- DiffPersMin[idxVar] - - # Initialize - setQf[[idxVar]]$setQfPers <- list(fail=numeric(0),na=numeric(0)) - setQf[[idxVar]]$setQfPers$na <- which(is.na(dataIdxVar)) - tmpQfPers <- rep(0,numData) # Default the flag to OK - tmpQfPersNa <- rep(-1,numData) # Set up a variable to coincidentally mark whether the data were able to be evaluated + # Restart the test from here + idxDataBgn <- idxData + idxData <- idxDataBgn+1 - # Quit if all data are NA - if(length(setDataReal[[idxVar]]) <= 1){ - setQf[[idxVar]]$setQfPers$na <- 1:numData - if(Vrbs){qf[[idxVar]]$qfPers[setQf[[idxVar]]$setQfPers$na] <- -1} - next - } else if(DiffPersMinIdx <= 0){ - if(Vrbs){qf[[idxVar]]$qfPers[setQf[[idxVar]]$setQfPers$na] <- -1} - next - } - - # Loop thru the data, checking for persistence over the window size. Here, we default the data to ok, - # and if the persistence test fails over any window encompassing a particular data point, the flag - # for the point is set to 1 - for (idxData in setDataReal[[idxVar]][1]:(numData-WndwPers[idxVar]+1)){ + idxDataMin <- idxDataBgn # reset running minimum + idxDataMax <- idxDataBgn # reset running maximum + + # Grab the data for these indices, improves CPU time + timeIdxBgn <- timePers[idxDataBgn] + dataIdxMin <- dataIdxVar[idxDataMin] + dataIdxMax <- dataIdxVar[idxDataMax] + + } - # Initialize metrics - setData <- idxData:(idxData+WndwPers[idxVar]-1) # data range to eval - dataIdx <- dataIdxVar[setData] # data to eval - dataIdxBgn <- dataIdx[1] # 1st data point in eval range + } else if ((idxData == numData) && (timePers[idxData]-timeIdxBgn >= WndwPersIdx)) { - if(is.na(dataIdxBgn) || sum(!is.na(dataIdx)) < 2){next} + # We didn't hit the threshold and we've reached the end of the data. We are also beyond the allowable + # time interval for the persistence test, so let's flag the data + setQf[[idxVar]]$setQfPers$fail <- union(setQf[[idxVar]]$setQfPers$fail,idxDataBgn:idxData) - dataIdxEnd <- tail(dataIdx,n=1) # Last data point in eval range - diffMax <- max(dataIdx,na.rm=TRUE) - min(dataIdx,na.rm=TRUE) # max difference over the eval range + idxData <- idxData+1 # We're done - # If the last point is NA and we haven't passed the threshold, get to the next point that exceeds - # the threshold and flag everything up that point - if(is.na(dataIdxEnd) && diffMax < DiffPersMinIdx){ - - # Keep looking at data until we surpass the threshold, or reach the end of the data - setDataExt <- setData[1]:numData - setDataExt <- intersect(setDataExt,setDataReal[[idxVar]]) # filter out the NA indices - idxPass <- head(which(cummax(dataIdxVar[setDataExt])-cummin(dataIdxVar[setDataExt]) >= DiffPersMinIdx),n=1) # Find the first point at which we pass the threshold - - # If we found it, stage data up to, but not including this point, for flagging - if(length(idxPass) == 1){ - setData <- setData[1]:(setDataExt[idxPass]-1) - } - } - + } else { - # Check for passing the threshold, and flag appropriately - #if(sum(!is.na(dataIdx)) > 1 && !is.na(dataIdxBgn)){ - tmpQfPersNa[setData] <- 0 # Mark data as able to eval - - if(diffMax < DiffPersMinIdx){ - tmpQfPers[setData] <- 1 # Flag - } - #} - } # End loop through data for normal persistence testing - - # Now account for NA values - tmpQfPers[tmpQfPersNa==-1] <- -1 # Set flag to -1 for cannot eval - tmpQfPers[is.na(dataIdxVar)] <- -1 # Ensure that flag is -1 for NA points + # We didn't pass the minimum acceptable change on this point, move to the next + idxData <- idxData+1 + } + } + + # If we reached the end of the data but the last value was NA, we need to go back and evaluate the last + # non-NA value + idxData <- numData + if (dataNaIdx[idxData]) { + # Get to last non-NA point + idxData <- tail(setDataReal[[idxVar]],n=1) - # If the last point is NA, go back and see what we could not evaluate - if(is.na(dataIdxVar[numData])){ - - # Initialize - idxDataEnd <- tail(setDataReal[[idxVar]],n=1) - idxData <- idxDataEnd-1 + if (timePers[idxData]-timeIdxBgn >= WndwPersIdx) { + # We didn't hit the threshold for the final non-NA points and we were beyond the allowable + # time interval for the persistence test, so let's flag the end of the data + setQf[[idxVar]]$setQfPers$fail <- union(setQf[[idxVar]]$setQfPers$fail,idxDataBgn:idxData) - # Step backward through the data until we surpass the persistence threshold. - # All data after this point needs to be marked as cannot evaluate, unless it - # has already been flagged - while(idxData > (idxDataEnd-WndwPers[idxVar]+1) && tmpQfPers[idxDataEnd] != 1){ - setData <- idxData:idxDataEnd - dataIdx <- dataIdxVar[setData] - diffMax <- suppressWarnings(max(dataIdx,na.rm=TRUE) - min(dataIdx,na.rm=TRUE)) - if (diffMax < DiffPersMinIdx){ - tmpQfPers[setData] <- -1 - idxData <- idxData-1 - } else { - break - } - } # End while loop - } # End if statement checking last value for NA - - # Output the results in the desired format - if(Vrbs){ - qf[[idxVar]]$qfPers <- tmpQfPers } else { - setQf[[idxVar]]$setQfPers$fail <- which(tmpQfPers == 1) - setQf[[idxVar]]$setQfPers$na <- which(tmpQfPers == -1) + # We didn't hit the threshold for the final non-NA points, but we are not yet beyond the + # allowable time interval, so let's flag as unable to evaluate + setQf[[idxVar]]$setQfPers$na <- union(setQf[[idxVar]]$setQfPers$na,idxDataBgn:idxData) } - - } # End for loop around variables - } # End if statement checking for numeric WndwPers value - + } + + # Don't mark the NA values as fail + setQf[[idxVar]]$setQfPers$fail <- setdiff(setQf[[idxVar]]$setQfPers$fail,setQf[[idxVar]]$setQfPers$na) + + + # For Verbose option, output actual flag values + if(Vrbs) { + qf[[idxVar]]$qfPers[setQf[[idxVar]]$setQfPers$fail] <- 1 + qf[[idxVar]]$qfPers[setQf[[idxVar]]$setQfPers$na] <- -1 + } + } + # Do Null test for(idxVar in 1:numVar) { setQf[[idxVar]]$setQfNull <- list(fail=numeric(0),na=numeric(0)) # initialize if(TestNull[idxVar]) { - setQf[[idxVar]]$setQfNull$fail <- which(is.na(data[,idxVar])) + setQf[[idxVar]]$setQfNull$fail <- setDataNa[[idxVar]] } # For Verbose option, output actual flag values @@ -511,48 +447,27 @@ def.plau <- function ( # Do Gap test for(idxVar in 1:numVar) { + # Initialize setQf[[idxVar]]$setQfGap <- list(fail=numeric(0),na=numeric(0)) # initialize - setNull <- setDataNa[[idxVar]] # find NA values - setGap <- setNull # Start out thinking every Null is a gap, we'll whittle it down below - diffSetNull <- diff(setNull) # difference between null position + # Do a rolling count of NAs with a window the length of the gap + dataNAIdx <- dataNa[[idxVar]] + NumGapIdx <- NumGap[idxVar] + numNaWndwGap <- RcppRoll::roll_sum(x=dataNAIdx,n=NumGapIdx,by=1,align='left',na.rm=FALSE) - # Only evaluate this if we need to - if (length(setNull) >= NumGap[idxVar]) { - - # Go thru each NA value to determine if it is part of a set >= NumGap[idxVar] - for (idxNull in 1:length(setNull)) { - - # Isolate positions within the NumGap[idxVar] range that are consecutive - diffSetNullSelf <- c(diffSetNull[1:idxNull-1],1,diffSetNull[idxNull:length(diffSetNull)])# Fill in the position difference vector with a value of 1 for idxNull itself - setNullPre <- seq(from=idxNull-NumGap[idxVar],to=idxNull-1,by=1) - setNullPre <- rev(setNullPre[(setNullPre > 0) & (setNullPre <= numData)]) - numNullPre <- which(diffSetNullSelf[setNullPre]!=1)[1]-1 # number of consecutive nulls prior to this Null - if (is.na(numNullPre)) { - numNullPre <- length(setNullPre) - } - setNullPost <- seq(from=idxNull,to=idxNull+NumGap[idxVar]-1,by=1) - setNullPost <- setNullPost[(setNullPost > 0) & (setNullPost <= numData)] - numNullPost <- which(diffSetNullSelf[setNullPost]!=1)[1]-1 # number of consecutive nulls including and after this Null - if (is.na(numNullPost)) { - numNullPost <- length(setNullPost) - } - - if (numNullPre+numNullPost < NumGap[idxVar]) { - # This position is not within a gap, so remove it from out list - setGap <- setdiff(setGap,setNull[idxNull]) - } - } - - setQf[[idxVar]]$setQfGap$fail <- setGap - + # Which of the windows has all gaps? Mark the test failure + setGapBgn <- which(numNaWndwGap == NumGapIdx) + qfGap <- rep(0,numData) + for(idxGapBgn in setGapBgn){ + qfGap[idxGapBgn:(idxGapBgn+NumGapIdx-1)] <- 1 } - # For Verbose option, output actual flag values + setQf[[idxVar]]$setQfGap$fail <- which(qfGap == 1) + if(Vrbs) { - qf[[idxVar]]$qfGap[setQf[[idxVar]]$setQfGap$fail] <- 1 - qf[[idxVar]]$qfGap[setQf[[idxVar]]$setQfGap$na] <- -1 + qf[[idxVar]]$qfGap <- qfGap } + } # Return results diff --git a/pack/eddy4R.qaqc/R/def.plot.dp01.qfqm.R b/pack/eddy4R.qaqc/R/def.plot.dp01.qfqm.R index df4908ce..c17339d5 100644 --- a/pack/eddy4R.qaqc/R/def.plot.dp01.qfqm.R +++ b/pack/eddy4R.qaqc/R/def.plot.dp01.qfqm.R @@ -45,6 +45,8 @@ # Changed term 'Pos' to 'Set' for multiple indices # David Durden (2019-02-05) # Fixing bug with Rmisc::multiplot and POSIXlt +# David Durden (2021-07-15) +# Fixing bug with R4.0.3 update where grob widths didn't match up ############################################################################################## @@ -220,7 +222,7 @@ def.plot.dp01.qfqm <- function ( grobNa <- ggplot2::ggplotGrob(plotNa) # grab the grob for this plot for later manipulation # Adjust grobs and combine to align plots (since we want the legend only for the middle plot) - colGrobLeg <- which(!(grobFail$widths %in% grobPass$widths)) # where the legend width is located in grobFail + colGrobLeg <- which(!(as.character(grobFail$widths) %in% as.character(grobPass$widths))) # where the legend width is located in grobFail distWdthLeg <- grobFail$widths[colGrobLeg] # Get legend width grobPass <- gtable::gtable_add_cols(grobPass, distWdthLeg, colGrobLeg[1]-1) # Add an entry for the legend width matching grobFail grobNa <- gtable::gtable_add_cols(grobNa, distWdthLeg, colGrobLeg[1]-1) # Add an entry for the legend width matching grobFail @@ -249,7 +251,7 @@ def.plot.dp01.qfqm <- function ( grobFinl <- ggplot2::ggplotGrob(plotFinl) # grab the grob for this plot for later manipulation # Adjust grobs and combine to align plots (since we want the legend only for the top plot) - colGrobLeg <- which(!(grobAlphBeta$widths %in% grobFinl$widths)) # where the legend width is located in grobFail + colGrobLeg <- which(!(as.character(grobAlphBeta$widths) %in% as.character(grobFinl$widths))) # where the legend width is located in grobFail distWdthLeg <- grobAlphBeta$widths[colGrobLeg] # Get legend width grobFinl <- gtable::gtable_add_cols(grobFinl, distWdthLeg, colGrobLeg[1]-1) # Add an entry for the legend width matching grobAlphBeta diff --git a/pack/eddy4R.qaqc/R/def.qf.ecte.R b/pack/eddy4R.qaqc/R/def.qf.ecte.R index eedd3951..dfbed865 100644 --- a/pack/eddy4R.qaqc/R/def.qf.ecte.R +++ b/pack/eddy4R.qaqc/R/def.qf.ecte.R @@ -38,13 +38,15 @@ # edited term name # Natchaya P-Durden (2018-04-03) # update @param format +# Dave Durden (2021-08-23) +# update to new ECTE dp01 names ############################################################################################## def.qf.ecte <- function( TimeBgn, TimeEnd, Freq = 20, - Sens = c("soni","irga","irgaMfcSamp","soniAmrs")[1], + Sens = c("soni","irgaTurb","mfcSampTurb","amrs")[1], PcntQf = 0.05 ) { @@ -79,21 +81,21 @@ def.qf.ecte <- function( varName <- c("veloSoni", "veloXaxs","veloYaxs","veloZaxs","tempSoni") qfNameSens <- c("qfSoniUnrs", "qfSoniData", "qfSoniTrig", "qfSoniComm", "qfSoniCode", "qfSoniTemp", "qfSoniSgnlPoor", "qfSoniSgnlHigh","qfSoniSgnlLow" )} - if(Sens == "irga"){ + if(Sens == "irgaTurb"){ varName <- c("asrpCo2", "asrpH2o", "densMoleCo2", "densMoleH2o", "tempRefe", "presAtm","presDiff", "potCool", "rtioMoleDryCo2", "rtioMoleDryH2o", "tempIn","tempOut", "powrH2oSamp", "powrH2oRefe", "powrCo2Samp", "powrCo2Refe","ssiCo2", "ssiH2o","tempMean", "presSum") qfNameSens <- c("qfIrgaHead", "qfIrgaTempOut", "qfIrgaTempIn", "qfIrgaAux", "qfIrgaPres", "qfIrgaChop", "qfIrgaDetc", "qfIrgaPll","qfIrgaSync", "qfIrgaAgc" )} - if(Sens == "irgaMfcSamp"){ + if(Sens == "mfcSampTurb"){ varName <- c("presAtm", "temp", "frt", "frt00", "frtSet00") qfNameSens <- NULL} - if(Sens == "soniAmrs"){ + if(Sens == "amrs"){ varName <- c("angXaxs", "angYaxs", "angZaxs", "accXaxs", "accYaxs", "accZaxs", "accXaxsDiff", "accYaxsDiff", "accZaxsDiff", "avelXaxs", "avelYaxs", "avelZaxs") qfNameSens <- c("qfAmrsVal", "qfAmrsFilt", "qfAmrsVelo", "qfAmrsRng")} ################################################################################################################################ # Name of prefixes for qaqc plausibility tests - qfNamePlau <- c("qfRng","qfStep", "qfPers", "qfCal" ) + qfNamePlau <- c("qfRng","qfStep", "qfPers", "qfNull", "qfSpk" ) #varNameUp <- paste(toupper(substr(varName, 1, 1)), substr(varName, 2, nchar(varName)), sep="") diff --git a/pack/eddy4R.qaqc/R/def.qf.finl.R b/pack/eddy4R.qaqc/R/def.qf.finl.R index 7811aa4b..41e215d8 100644 --- a/pack/eddy4R.qaqc/R/def.qf.finl.R +++ b/pack/eddy4R.qaqc/R/def.qf.finl.R @@ -49,6 +49,8 @@ # Added exclusion of Null test failures from qmBeta (to avoid double-counting) # Natchaya P-Durden (2018-04-03) # update @param format +# David Durden (2021-08-18) +# Removing quality indicators (qi) from qfFinal determination ############################################################################################## def.qf.finl <- function ( qf, @@ -75,6 +77,9 @@ def.qf.finl <- function ( stop("Input parameter Thsh must be a numeric vector of length 1") } + #Remove quality indicators from being qfFinal determination + qf <- qf[,grep("^qi", x = names(qf), invert = TRUE)] + # Compute Quality metric --------------------------------------------------- #compute qfAlpha and qfBeta qf[,"qfAlph"] <- apply(qf, 1, function(x) ifelse(any(x==1),1,0)) diff --git a/pack/eddy4R.qaqc/R/def.qf.rmv.data.R b/pack/eddy4R.qaqc/R/def.qf.rmv.data.R index 45664bee..0b0f98d8 100644 --- a/pack/eddy4R.qaqc/R/def.qf.rmv.data.R +++ b/pack/eddy4R.qaqc/R/def.qf.rmv.data.R @@ -48,6 +48,8 @@ # replaced dfQf by inpQf # Natchaya P-Durden (2019-03-12) # added qfRmv into the function parameter list +# Dave Durden (2020-04-24) +# Adding qfNull to output ############################################################################################## @@ -71,6 +73,16 @@ def.qf.rmv.data <- function( #List of variables to check for flags to remove bad data rpt$listVar <- base::names(inpData[!base::names(inpData) %in% c("time", "idx")]) + #Create a NULL flag based on NaN's in original data + rpt$qfNull <- as.data.frame(sapply(rpt$listVar, function(x){ + qfTmp <- base::rep_len(0L, length.out = length(inpData[[x]])) + setNull <- which(is.na(inpData[[x]])) + qfTmp[setNull] <- 1L + return(qfTmp) + })) + #Correct the Null flag names + names(rpt$qfNull) <- paste0("qfNull",toupper(substring(names(rpt$qfNull),1,1)),substring(names(rpt$qfNull),2,nchar(names(rpt$qfNull)))) + #If a sensor (Sens) is included, check for sensor specific flags to perform filtering of data if(!base::is.null(Sens) && base::length(Sens) == 1){ diff --git a/pack/eddy4R.qaqc/R/def.qi.qf.R b/pack/eddy4R.qaqc/R/def.qi.qf.R new file mode 100644 index 00000000..f78e41ca --- /dev/null +++ b/pack/eddy4R.qaqc/R/def.qi.qf.R @@ -0,0 +1,58 @@ +############################################################################################## +#' @title Definition function: Quality Indicators + +#' @author +#' David Durden \email{eddy4R.info@gmail.com} + +#' @description +#' Function definition. Convert quality flags (qf) to quality indicators (qi) by changing names. Performed for the entire set of input data. + +#' @param qf A data frame or list of data.frames containing quality flags, class integer. Each column contains the quality flag values [-1,0,1] for that flag. Note: This is the Vrbs output from def.plau, def.dspk.wndw, and def.dspk.br86. See def.conv.qf.vrbs for converting from non-verbose to verbose output. + +#' @return A dataframe containing quality indicators of failed (1), pass (0), and NA (-1) for each of the individual flag defined by input \code{qf}. + +#' @references +#' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007. \cr +#' NEON Algorithm Theoretical Basis Document: Quality Flags and Quality Metrics for TIS Data Products (NEON.DOC.001113) \cr +#' Smith, D.E., Metzger, S., and Taylor, J.R.: A transparent and transferable framework for tracking quality information in large datasets. PLoS ONE, 9(11), e112249.doi:10.1371/journal.pone.0112249, 2014. \cr + +#' @keywords NEON QAQC, quality flags, quality metrics + +#' @examples +#' qfA <- c(0,0,0,0,0,1,1,1,1,1,0,0,0,0,1) +#' qfB <- c(0,0,-1,-1,-1,1,1,0,0,0,0,0,0,0,0) +#' qfC <- c(0,1,1,0,0,0,0,0,0,0,0,0,-1,-1,-1) +#' qfD <- data.frame(qfE = c(0,1,1,0,0,0,0,0,0,0,0,0,-1,-1,-1), qfF = c(0,1,1,0,0,0,0,-1,0,0,0,0,-1,-1,-1)) +#' test<-list() +#' test$dfQf <- data.frame(qfA,qfB,qfC) +#' test$listQf <- list(test$dfQf,qfD) +#' test$qi<- def.qi.qf(qf=test$listQf) + +#' @seealso +#' \code{\link[eddy4R.qaqc]{wrap.qfqm.dp01}} +#' \code{\link[eddy4R.qaqc]{def.dp01.grp.qf}} +#' \code{\link[eddy4R.qaqc]{wrap.dp01.qfqm.eddy}} +#' \code{\link[eddy4R.qaqc]{def.qf.finl.R}} +#' \code{\link[eddy4R.qaqc]{def.qm.R}} + +#' @export + +# changelog and author contributions / copyrights +# David Durden (2021-08-24) +# original creation + +############################################################################################## +def.qi.qf <- function ( + qf # A data.frame or list of data.frames containing quality flags, class integer. Each column contains the quality flag values [-1,0,1] for that flag. + +) { + + #Check if input is a list of data.frames, if TRUE convert to single data frame + if(base::is.list(qf)){qf <- base::cbind.data.frame(qf)} + + #convert quality flags to quality indicators + names(qf) <- base::gsub(pattern = "^qf", replacement = "qi", x = names(qf)) + + #Return output data.frame with updated names + return(qf) +}#End def.qi.qf \ No newline at end of file diff --git a/pack/eddy4R.qaqc/R/wrap.dp01.qfqm.R b/pack/eddy4R.qaqc/R/wrap.dp01.qfqm.R index d7b37004..e0b98218 100644 --- a/pack/eddy4R.qaqc/R/wrap.dp01.qfqm.R +++ b/pack/eddy4R.qaqc/R/wrap.dp01.qfqm.R @@ -238,10 +238,10 @@ wrap.dp01.qfqm <- function ( # Find data locations in window setData <- base::which((time >= timeAgrBgn[idxAgr]) & (time < timeAgrBgn[idxAgr]+WndwAgr)) - numDataAgr <- base::length(setData) + numDataIdxAgr <- base::length(setData) # If there is no data to process move to next aggregation window - if (numDataAgr == 0) { + if (numDataIdxAgr == 0) { next } diff --git a/pack/eddy4R.qaqc/R/wrap.dp01.qfqm.ecse.R b/pack/eddy4R.qaqc/R/wrap.dp01.qfqm.ecse.R index c0b4292f..418fd5b1 100644 --- a/pack/eddy4R.qaqc/R/wrap.dp01.qfqm.ecse.R +++ b/pack/eddy4R.qaqc/R/wrap.dp01.qfqm.ecse.R @@ -17,6 +17,7 @@ #' @param lvlValv Measurement level of irgaValvLvl, crdCo2ValvLvl, or crdH2oValvLvl. Defaults to NULL. Of type character. [-] #' @param lvlValvAux Location of valvAux which apply to only dp01 equal to "co2Stor" or "h2oStor". Defaults to NULL. Of type character. [-] #' @param lvlCrdH2oValvVali Measurement level of crdH2oValvVali which apply to only dp01 equal to "isoH2o". Defaults to NULL. Of type character. [-] +#' @param lvlCrdCo2Valv Horizontal and vertical location for crdCo2 valve. Defaults to NULL. Of type character. [-] #' @param data A list of data frame containing the input dp0p data that related to dp01 which qfqm are being calculated. Of class integer". [User defined] #' @param qfInp A list of data frame containing the input quality flag data that related to dp01 are being grouped. Of class integer". [-] #' @param TypeMeas A vector of class "character" containing the name of measurement type (sampling or validation), TypeMeas = c("samp", "vali"). Defaults to "samp". [-] @@ -75,9 +76,26 @@ # pull in the qf from presInlt # Natchaya P-Durden (2019-05-23) # pull in the qf from pumpStor, presValiRegInStor and presValiRegOutStor +# Natchaya P-Durden (2020-03-12) +# In irgaCo2 an irgaH2o, not include the period when crdCo2 take over to measure at that level +# and irga have to move to measure next level +# Natchaya P-Durden (2020-03-25) +# added lvlCrdCo2Valv to the function's parameter +# David Durden (2020-05-15) +# failsafe for crd kickoff removal causing no data for entire day +# David Durden (2020-05-21) +# bug fix for qmBeta causing differing number of values between data and qfqm +# David Durden (2020-07-23) +# bug fix for valve issues where looks like consistently Stor data thrown off by Crd +# Chris Florian (2021-02-26) +# adding ch4Conc to dp01 list +# Chris Florian (2021-06-03) +# adjusting data input for crdCh4 validation time periods +# Chris Florian (2021-06-07) +# updating list to remove the CH4 reference values from qfqm ############################################################################################## wrap.dp01.qfqm.ecse <- function( - dp01 = c("co2Stor", "h2oStor", "tempAirLvl", "tempAirTop", "isoCo2", "isoH2o")[1], + dp01 = c("co2Stor", "h2oStor", "tempAirLvl", "tempAirTop", "isoCo2", "isoH2o", "ch4Conc")[1], RptExpd = FALSE, lvl, lvlMfcSampStor = NULL, @@ -86,6 +104,7 @@ wrap.dp01.qfqm.ecse <- function( lvlValv = NULL, lvlValvAux = NULL, lvlCrdH2oValvVali = NULL, + lvlCrdCo2Valv = NULL, data = list(), qfInp = list(), TypeMeas = c("samp", "vali")[1], @@ -213,6 +232,53 @@ wrap.dp01.qfqm.ecse <- function( rpt[[idxAgr]]$timeEnd[[idxVar]] <- wrk$idx$timeEnd[idxAgr] } + #get rid of period that crdCo2 take over and irga have to move to measure other level + #and number of sample less than 10% (120-120*0.1) + if (dp01 == "co2Stor") {numSamp <- sum(!is.na(wrk$data$rtioMoleDryCo2[wrk$idx$idxBgn[idxAgr]:wrk$idx$idxEnd[idxAgr]]))} + if (dp01 == "h2oStor") {numSamp <- sum(!is.na(wrk$data$rtioMoleDryH2o[wrk$idx$idxBgn[idxAgr]:wrk$idx$idxEnd[idxAgr]]))} + if (data$crdCo2ValvLvl[[lvlCrdCo2Valv]]$lvlCrdCo2[wrk$idx$idxEnd[idxAgr]] == lvlIrga & numSamp < 108){ + rpt[[idxAgr]] <- NULL + } + + #Remove any empty lists in case valve issues + if(length(wrk$idx$idxBgn) == idxAgr) rpt <- Filter(Negate(is.null), rpt) + + #Check if after removing data for valve kickooff if no data remains + if(length(wrk$idx$idxBgn) == idxAgr && length(rpt) == 0){ + rpt[[1]] <- list() + #idxStat <- NameQf[1] + rpt[[1]]$qmAlph <- as.data.frame(matrix(0, nrow = 1, ncol = ncol(wrk$data))) + rpt[[1]]$qmBeta <- as.data.frame(matrix(1, nrow = 1, ncol = ncol(wrk$data))) + rpt[[1]]$qfFinl <- as.data.frame(matrix(1, nrow = 1, ncol = ncol(wrk$data))) + rpt[[1]]$qfSciRevw <- as.data.frame(matrix(0, nrow = 1, ncol = ncol(wrk$data))) + #change data type + rpt[[1]]$qfFinl[,1:ncol(wrk$data)] <- sapply(rpt[[1]]$qfFinl[,1:ncol(wrk$data)], as.integer) + rpt[[1]]$qfSciRevw[,1:ncol(wrk$data)] <- sapply(rpt[[1]]$qfSciRevw[,1:ncol(wrk$data)], as.integer) + + for(idxQf in NameQf){ + #assign name to each column + names(rpt[[1]][[idxQf]]) <- names(wrk$data) + #not report lvlIrga + rpt[[1]][[idxQf]] <- rpt[[1]][[idxQf]][which(!(names(rpt[[1]][[idxQf]]) %in% c("lvlIrga")))] + }; rm(idxQf) + + #add both time begin and time end to rpt + rpt[[1]]$timeBgn <- list() + rpt[[1]]$timeEnd <- list() + + #output time for dp01 + for(idxVar in names(wrk$data)[which(!(names(wrk$data) %in% c("lvlIrga")))]){ + rpt[[1]]$timeBgn[[idxVar]] <- data$time[1] + rpt[[1]]$timeEnd[[idxVar]] <- data$time[length(data$time)] + #unit + attributes(rpt[[1]]$qmAlph[[idxVar]])$unit <- "-" + attributes(rpt[[1]]$qmBeta[[idxVar]])$unit <- "-" + attributes(rpt[[1]]$qfFinl[[idxVar]])$unit <- "NA" + attributes(rpt[[1]]$qfSciRevw[[idxVar]])$unit <- "NA" + + }; rm(idxVar) + + }#end of failsafe to check if no measurement data at all in the whole day after valve kickoff removal #}# end of there is at least one data }; rm(idxAgr) @@ -263,9 +329,31 @@ wrap.dp01.qfqm.ecse <- function( wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$irgaStor$qfRngTemp, CritTime = 60) #delete row if last timeBgn and timeEnd is NA wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] + #replace last idxEnd > 86400 by 86400 + wrk$idx$idxEnd <- ifelse(wrk$idx$idxEnd > 86400, 86400, wrk$idx$idxEnd) #if last timeEnd is NA, replce that time to the last time value in data$time wrk$idx$timeEnd <- as.POSIXct(ifelse(is.na(wrk$idx$timeEnd), data$time[length(data$time)], wrk$idx$timeEnd), origin = "1970-01-01", tz = "UTC") + #get rid of period that crdCo2 take over and irga have to move to measure other level + #and number of sample less than 10% (120-120*0.1) + tmpWrkIdx <- list() + for (idxAgr in 1:length(wrk$idx$idxEnd)){ + if (dp01 == "co2Stor") {numSamp <- sum(!is.na(wrk$data$rtioMoleDryCo2[wrk$idx$idxBgn[idxAgr]:wrk$idx$idxEnd[idxAgr]]))} + if (dp01 == "h2oStor") {numSamp <- sum(!is.na(wrk$data$rtioMoleDryH2o[wrk$idx$idxBgn[idxAgr]:wrk$idx$idxEnd[idxAgr]]))} + if (data$crdCo2ValvLvl[[lvlCrdCo2Valv]]$lvlCrdCo2[wrk$idx$idxEnd[idxAgr]] == lvlIrga & numSamp < 108){ + tmpWrkIdx[[idxAgr]] <- idxAgr + + } + } + + #combine idxAgr which will be removed + tmpRmv <- do.call(cbind,tmpWrkIdx) + #remove those idxAgr from wrk$idx + if(!is.null(tmpRmv)) wrk$idx <- wrk$idx[-c(tmpRmv),] + + #If statement to check if all values were removed + if(nrow(wrk$idx) > 0){ + whrSamp <- wrk$idx$idxBgn[1]:wrk$idx$idxEnd[1] if (length (wrk$idx$idxBgn) > 1 ){ for(ii in 2:length (wrk$idx$idxBgn)){ @@ -279,7 +367,8 @@ wrap.dp01.qfqm.ecse <- function( wrk$qfqm[[idxSens]][wrk$data$lvlIrga != lvlIrga, 1:length(wrk$qfqm[[idxSens]])] <- -1 #replace all qf that not belong to that measurement level by NaN wrk$qfqm[[idxSens]][-whrSamp, 1:length(wrk$qfqm[[idxSens]])] <- NaN - } + } + }#End if statement after qfRmv } @@ -632,7 +721,7 @@ wrap.dp01.qfqm.ecse <- function( #if there is at least one measurement if(length(which(!is.na(wrk$qfqm$crdCo2$qfRngTemp))) > 0){ #determine the index of each measurement - wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 60) + wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 240) #delete row if last timeBgn and timeEnd is NA wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] #replace last idxEnd > 86400 by 86400 @@ -719,7 +808,7 @@ wrap.dp01.qfqm.ecse <- function( if(length(which(!is.na(wrk$qfqm$crdCo2$qfRngTemp))) > 0){ #determine the index of each measurement - wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 60) + wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 240) #delete row if last timeBgn and timeEnd is NA wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] #if last timeEnd is NA, replce that time to the last time value in data$time @@ -820,7 +909,7 @@ wrap.dp01.qfqm.ecse <- function( #if there is at least one measurement if(length(which(!is.na(wrk$qfqm$crdCo2$qfRngTemp))) > 0){ #determine the end time of each measurement - wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 60) + wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 240) #delete row if last timeBgn and timeEnd is NA wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] #if last timeEnd is NA, replce that time to the last time value in data$time @@ -910,7 +999,7 @@ wrap.dp01.qfqm.ecse <- function( #if there is at least one measurement if(length(which(!is.na(wrk$qfqm$crdCo2$qfRngTemp))) > 0){ # #determine the end time of each measurement - wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 60) + wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 240) #delete row if last timeBgn and timeEnd is NA wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] #if last timeEnd is NA, replce that time to the last time value in data$time @@ -967,6 +1056,382 @@ wrap.dp01.qfqm.ecse <- function( }#end of TypeMeas %in% "vali" if statement }##end of dp01 if statement + + #calculate dp01 for "ch4Conc" ######################################################################################## + if (dp01 %in% c("ch4Conc")){ + #during sampling period + if (TypeMeas %in% "samp"){ + #assign lvlIrga for each measurement level + if (lvl == "000_010") {lvlCrdCo2 <- "lvl01"} + if (lvl == "000_020") {lvlCrdCo2 <- "lvl02"} + if (lvl == "000_030") {lvlCrdCo2 <- "lvl03"} + if (lvl == "000_040") {lvlCrdCo2 <- "lvl04"} + if (lvl == "000_050") {lvlCrdCo2 <- "lvl05"} + if (lvl == "000_060") {lvlCrdCo2 <- "lvl06"} + if (lvl == "000_070") {lvlCrdCo2 <- "lvl07"} + if (lvl == "000_080") {lvlCrdCo2 <- "lvl08"} + #assign lvlMfm + lvlMfm <- paste0("700_", strsplit(lvl, "_")[[1]][2]) + + #input the whole day data + wrk$data <- data.frame(stringsAsFactors = FALSE, + "pres" = data$crdCo2[[lvl]]$pres, + "presEnvHut" = data$envHut[[lvlEnvHut]]$pres, + "rhEnvHut" = data$envHut[[lvlEnvHut]]$rh, + "rtioMoleDryCh4" = data$crdCo2[[lvl]]$rtioMoleDryCh4, + "rtioMoleWetCh4" = data$crdCo2[[lvl]]$rtioMoleWetCh4, + "rtioMoleWetH2oEnvHut" = data$envHut[[lvlEnvHut]]$rtioMoleWetH2o, + "temp" = data$crdCo2[[lvl]]$temp, + "tempEnvHut" = data$envHut[[lvlEnvHut]]$temp, + "lvlCrdCo2" = data$crdCo2ValvLvl[[lvlValv]]$lvlCrdCo2 + + ) + + #input the whole day qfqm + wrk$qfqm <- list() + #subset only + wrk$qfqm$crdCo2 <- qfInp$crdCo2[[lvl]] + wrk$qfqm$envHut <- qfInp$envHut[[lvlEnvHut]] + wrk$qfqm$mfm <- qfInp$mfm[[lvlMfm]] + if ("presInlt" %in% names(qfInp)) wrk$qfqm$presInlt <- qfInp$presInlt[[lvl]] + if ("pumpStor" %in% names(qfInp)) wrk$qfqm$pumpStor <- qfInp$pumpStor[[lvlMfm]] + + if (PrdMeas == PrdAgr) { + #PrdAgr <- 9 + #9 minutely sampling data + #idxLvLPrdAgr <- paste0(lvl, "_", sprintf("%02d", PrdAgr), "m") + #rpt[[dp01]][[idxLvLPrdAgr]] <- list() + + #if there is at least one measurement + if(length(which(!is.na(wrk$qfqm$crdCo2$qfRngTemp))) > 0){ + #determine the index of each measurement + wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 60) + #delete row if last timeBgn and timeEnd is NA + wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] + #replace last idxEnd > 86400 by 86400 + wrk$idx$idxEnd <- ifelse(wrk$idx$idxEnd > 86400, 86400, wrk$idx$idxEnd) + #if last timeEnd is NA, replce that time to the last time value in data$time + wrk$idx$timeEnd <- as.POSIXct(ifelse(is.na(wrk$idx$timeEnd), data$time[length(data$time)], wrk$idx$timeEnd), origin = "1970-01-01", tz = "UTC") + #idxAgr2 <- 0 + for (idxAgr in 1:length(wrk$idx$idxBgn)){ + #idxAgr <- 25 + #get data for each idxAgr + wrk$inpMask$data <- list() + wrk$inpMask$data <- wrk$data[wrk$idx$idxBgn[idxAgr]:wrk$idx$idxEnd[idxAgr],] + #wrk$inpMask for qfqm + wrk$inpMask$qfqm <- list() + lapply(names(wrk$qfqm), function (x) wrk$inpMask$qfqm[[x]] <<- wrk$qfqm[[x]][wrk$idx$idxBgn[idxAgr]:wrk$idx$idxEnd[idxAgr],] ) + + for (idxSens in names(wrk$inpMask$qfqm)){ + #replace qfqm with -1 when valve switch to measure to next level before schedule time (9 min) + wrk$inpMask$qfqm[[idxSens]][wrk$inpMask$data$lvlCrdCo2 != lvlCrdCo2, 1:length(wrk$inpMask$qfqm[[idxSens]])] <- -1 + } + + #qfqm processing + rpt[[idxAgr]] <- eddy4R.qaqc::wrap.dp01.qfqm.eddy( + qfInp = wrk$inpMask$qfqm, + MethMeas = "ecse", + TypeMeas = "samp", + RptExpd = FALSE, + dp01 = dp01, + idGas = wrk$inpMask$data$idGas + ) + #grab and add both time begin and time end to rpt + rpt[[idxAgr]]$timeBgn <- list() + rpt[[idxAgr]]$timeEnd <- list() + + #output time for dp01 + for(idxVar in names(wrk$data)[which(!(names(wrk$data) %in% c("idGas", "lvlCrdCo2")))]){ + rpt[[idxAgr]]$timeBgn[[idxVar]] <- wrk$idx$timeBgn[idxAgr] + rpt[[idxAgr]]$timeEnd[[idxVar]] <- wrk$idx$timeEnd[idxAgr] + }; rm(idxVar) + + #}# end of there is at least one data + + }; rm(idxAgr) + } else { + + rpt[[1]] <- list() + #idxStat <- NameQf[1] + rpt[[1]]$qmAlph <- as.data.frame(matrix(0, nrow = 1, ncol = ncol(wrk$data))) + rpt[[1]]$qmBeta <- as.data.frame(matrix(1, nrow = 1, ncol = ncol(wrk$data))) + rpt[[1]]$qfFinl <- as.data.frame(matrix(1, nrow = 1, ncol = ncol(wrk$data))) + rpt[[1]]$qfSciRevw <- as.data.frame(matrix(0, nrow = 1, ncol = ncol(wrk$data))) + #change data type + rpt[[1]]$qfFinl[,1:ncol(wrk$data)] <- sapply(rpt[[1]]$qfFinl[,1:ncol(wrk$data)], as.integer) + rpt[[1]]$qfSciRevw[,1:ncol(wrk$data)] <- sapply(rpt[[1]]$qfSciRevw[,1:ncol(wrk$data)], as.integer) + + for(idxQf in NameQf){ + #assign name to each column + names(rpt[[1]][[idxQf]]) <- names(wrk$data) + #not report lvlIrga + rpt[[1]][[idxQf]] <- rpt[[1]][[idxQf]][which(!(names(rpt[[1]][[idxQf]]) %in% c("idGas", "lvlCrdCo2")))] + }; rm(idxQf) + + #add both time begin and time end to rpt + rpt[[1]]$timeBgn <- list() + rpt[[1]]$timeEnd <- list() + + #output time for dp01 + for(idxVar in names(wrk$data)[which(!(names(wrk$data) %in% c("idGas", "lvlCrdCo2")))]){ + rpt[[1]]$timeBgn[[idxVar]] <- data$time[1] + rpt[[1]]$timeEnd[[idxVar]] <- data$time[length(data$time)] + #unit + attributes(rpt[[1]]$qmAlph[[idxVar]])$unit <- "-" + attributes(rpt[[1]]$qmBeta[[idxVar]])$unit <- "-" + attributes(rpt[[1]]$qfFinl[[idxVar]])$unit <- "NA" + attributes(rpt[[1]]$qfSciRevw[[idxVar]])$unit <- "NA" + }; rm(idxVar) + + }#end of if no measurement data at all in the whole day + } #end of PrdAgr + + if (PrdMeas != PrdAgr) { + #PrdAgr <- 30 + #if there is at least one measurement + if(length(which(!is.na(wrk$qfqm$crdCo2$qfRngTemp))) > 0){ + + #determine the index of each measurement + wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 60) + #delete row if last timeBgn and timeEnd is NA + wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] + #if last timeEnd is NA, replce that time to the last time value in data$time + wrk$idx$timeEnd <- as.POSIXct(ifelse(is.na(wrk$idx$timeEnd), data$time[length(data$time)], wrk$idx$timeEnd), origin = "1970-01-01", tz = "UTC") + whrSamp <- wrk$idx$idxBgn[1]:wrk$idx$idxEnd[1] + if (length (wrk$idx$idxBgn) > 1 ){ + for(ii in 2:length (wrk$idx$idxBgn)){ + whrSamp <- c(whrSamp, wrk$idx$idxBgn[ii]:wrk$idx$idxEnd[ii]) + } + } + wrk$data[-whrSamp, 1:16] <- NaN + + for (idxSens in names(wrk$qfqm)){ + #replace qfqm with -1 when valve switch to measure to next level before schedule time (9 min) + wrk$qfqm[[idxSens]][wrk$data$lvlCrdCo2 != lvlCrdCo2, 1:length(wrk$qfqm[[idxSens]])] <- -1 + #replace all qf that not belong to that measurement level by NaN + wrk$qfqm[[idxSens]][-whrSamp, 1:length(wrk$qfqm[[idxSens]])] <- NaN + } + } + + for(idxAgr in c(1:length(idxTime[[paste0(PrdAgr, "min")]]$Bgn))) { + #idxAgr <- 48 + #get data for each idxAgr + wrk$inpMask$data <- list() + idxLvLPrdAgr <- paste0(lvl, "_", sprintf("%02d", PrdAgr), "m") + + wrk$inpMask$data <- wrk$data[idxTime[[paste0(PrdAgr, "min")]]$Bgn[idxAgr]:idxTime[[paste0(PrdAgr, "min")]]$End[idxAgr],] + # for qfqm + wrk$inpMask$qfqm <- list() + lapply(names(wrk$qfqm), function (x) wrk$inpMask$qfqm[[x]] <<- wrk$qfqm[[x]][idxTime[[paste0(PrdAgr, "min")]]$Bgn[idxAgr]:idxTime[[paste0(PrdAgr, "min")]]$End[idxAgr],]) + + #qfqm processing + rpt[[idxAgr]] <- eddy4R.qaqc::wrap.dp01.qfqm.eddy( + qfInp = wrk$inpMask$qfqm, + MethMeas = "ecse", + TypeMeas = "samp", + RptExpd = RptExpd, + dp01 = dp01, + idGas = wrk$inpMask$data$idGas + ) + + #grab and add both time begin and time end to rpt + rpt[[idxAgr]]$timeBgn <- list() + rpt[[idxAgr]]$timeEnd <- list() + + for(idxVar in names(wrk$data)[which(!(names(wrk$data) %in% c("idGas", "lvlCrdCo2")))]){ + rpt[[idxAgr]]$timeBgn[[idxVar]] <- data$time[idxTime[[paste0(PrdAgr, "min")]]$Bgn[idxAgr]] + rpt[[idxAgr]]$timeEnd[[idxVar]] <- data$time[idxTime[[paste0(PrdAgr, "min")]]$End[idxAgr]] + } + + }; #rm(idxAgr) + + }#end of PrdAgr == 30 + }#end of TypeMeas %in% "samp" + + #during validation period + if (TypeMeas %in% "vali"){ + #assign lvlPresValiRegInStor for each gas + if (lvl == "co2Arch") lvlPresValiRegInStor <- "710_000" + if (lvl == "co2Low") lvlPresValiRegInStor <- "712_000" + if (lvl == "co2Med") lvlPresValiRegInStor <- "713_000" + if (lvl == "co2High") lvlPresValiRegInStor <- "714_000" + #input the whole day data + wrk$data <- data.frame(stringsAsFactors = FALSE, + "idGas" = data$crdCo2[[lvl]]$idGas, + "pres" = data$crdCo2[[lvl]]$pres, + "presEnvHut" = data$envHut[[lvlEnvHut]]$pres, + "rhEnvHut" = data$envHut[[lvlEnvHut]]$rh, + "rtioMoleDryCh4" = data$crdCo2[[lvl]]$rtioMoleDryCo2, + "rtioMoleDryCh4Refe" = data$crdCo2[[lvl]]$rtioMoleDryCo2Refe, + "rtioMoleWetCh4" = data$crdCo2[[lvl]]$rtioMoleWetCo2, + "rtioMoleWetH2oEnvHut" = data$envHut[[lvlEnvHut]]$rtioMoleWetH2o, + "temp" = data$crdCo2[[lvl]]$temp, + "tempEnvHut" = data$envHut[[lvlEnvHut]]$temp + + ) + #input the whole day qfqm + wrk$qfqm <- list() + wrk$qfqm$crdCo2 <- qfInp$crdCo2[[lvl]] + wrk$qfqm$envHut <- qfInp$envHut[[lvlEnvHut]] + wrk$qfqm$mfcValiStor <- qfInp$mfcValiStor[[lvlMfcValiStor]] + if ("presValiRegInStor" %in% names(qfInp)) wrk$qfqm$presValiRegInStor <- qfInp$presValiRegInStor[[lvlPresValiRegInStor]] + + if (PrdMeas == PrdAgr) { + #PrdAgr <- 9 + #9 minutely sampling data + #idxLvLPrdAgr <- paste0(lvl, "_", sprintf("%02d", PrdAgr), "m") + #rpt[[dp01]][[idxLvLPrdAgr]] <- list() + + #if there is at least one measurement + if(length(which(!is.na(wrk$qfqm$crdCo2$qfRngTemp))) > 0){ + #determine the end time of each measurement + wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 60) + #delete row if last timeBgn and timeEnd is NA + wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] + #if last timeEnd is NA, replce that time to the last time value in data$time + wrk$idx$timeEnd <- as.POSIXct(ifelse(is.na(wrk$idx$timeEnd), data$time[length(data$time)], wrk$idx$timeEnd), origin = "1970-01-01", tz = "UTC") + + #idxAgr2 <- 0 + for (idxAgr in 1:length(wrk$idx$idxBgn)){ + #idxAgr <- 1 + #determine input data for each idxAgr + wrk$inpMask$data <- list() + wrk$inpMask$data <- wrk$data[wrk$idx$idxBgn[idxAgr]:wrk$idx$idxEnd[idxAgr],] + #wrk$inpMask for qfqm + wrk$inpMask$qfqm <- list() + lapply(names(wrk$qfqm), function (x) wrk$inpMask$qfqm[[x]] <<- wrk$qfqm[[x]][wrk$idx$idxBgn[idxAgr]:wrk$idx$idxEnd[idxAgr],] ) + + #qfqm processing + rpt[[idxAgr]] <- eddy4R.qaqc::wrap.dp01.qfqm.eddy( + qfInp = wrk$inpMask$qfqm, + MethMeas = "ecse", + TypeMeas = "vali", + RptExpd = FALSE, + dp01 = dp01, + idGas = wrk$inpMask$data$idGas + ) + + #grab and add both time begin and time end to rpt + rpt[[idxAgr]]$timeBgn <- list() + rpt[[idxAgr]]$timeEnd <- list() + + #output time for qf dp01; do not output reference gas + for(idxVar in names(wrk$data)[which(!(names(wrk$data) %in% c("rtioMoleDryCh4Refe", "idGas")))]){ + rpt[[idxAgr]]$timeBgn[[idxVar]] <- wrk$idx$timeBgn[idxAgr] + rpt[[idxAgr]]$timeEnd[[idxVar]] <- wrk$idx$timeEnd[idxAgr] + }; rm(idxVar) + + }#; rm(idxAgr) + + } else { + + rpt[[1]] <- list() + + if(lvl %in% c("co2Arch")){ + rpt[[1]]$qmAlph <- as.data.frame(matrix(NaN, nrow = 1, ncol = ncol(wrk$data))) + rpt[[1]]$qmBeta <- as.data.frame(matrix(NaN, nrow = 1, ncol = ncol(wrk$data))) + rpt[[1]]$qfFinl <- as.data.frame(matrix(NaN, nrow = 1, ncol = ncol(wrk$data))) + rpt[[1]]$qfSciRevw <- as.data.frame(matrix(0, nrow = 1, ncol = ncol(wrk$data))) + #change data type + rpt[[1]]$qfSciRevw[,1:ncol(wrk$data)] <- sapply(rpt[[1]]$qfSciRevw[,1:ncol(wrk$data)], as.integer) + }else{ + rpt[[1]]$qmAlph <- as.data.frame(matrix(0, nrow = 1, ncol = ncol(wrk$data))) + rpt[[1]]$qmBeta <- as.data.frame(matrix(1, nrow = 1, ncol = ncol(wrk$data))) + rpt[[1]]$qfFinl <- as.data.frame(matrix(1, nrow = 1, ncol = ncol(wrk$data))) + rpt[[1]]$qfSciRevw <- as.data.frame(matrix(0, nrow = 1, ncol = ncol(wrk$data))) + #change data type + rpt[[1]]$qfFinl[,1:ncol(wrk$data)] <- sapply(rpt[[1]]$qfFinl[,1:ncol(wrk$data)], as.integer) + rpt[[1]]$qfSciRevw[,1:ncol(wrk$data)] <- sapply(rpt[[1]]$qfSciRevw[,1:ncol(wrk$data)], as.integer) + } + + for(idxQf in NameQf){ + #assign name to each column + names(rpt[[1]][[idxQf]]) <- names(wrk$data) + #not report lvlIrga + rpt[[1]][[idxQf]] <- rpt[[1]][[idxQf]][which(!(names(rpt[[1]][[idxQf]]) %in% c("rtioMoleDryCh4Refe", "idGas")))] + }; rm(idxQf) + + #add both time begin and time end to rpt + rpt[[1]]$timeBgn <- list() + rpt[[1]]$timeEnd <- list() + + #output time for dp01 + for(idxVar in names(wrk$data)[which(!(names(wrk$data) %in% c("rtioMoleDryCh4Refe", "idGas")))]){ + rpt[[1]]$timeBgn[[idxVar]] <- data$time[1] + rpt[[1]]$timeEnd[[idxVar]] <- data$time[length(data$time)] + #unit + attributes(rpt[[1]]$qmAlph[[idxVar]])$unit <- "-" + attributes(rpt[[1]]$qmBeta[[idxVar]])$unit <- "-" + attributes(rpt[[1]]$qfFinl[[idxVar]])$unit <- "NA" + attributes(rpt[[1]]$qfSciRevw[[idxVar]])$unit <- "NA" + }; rm(idxVar) + + }#end of if no measurement data at all in the whole day + + } #end of PrdAgr == 9 + + if (PrdMeas != PrdAgr) { + #PrdAgr <- 30 + #if there is at least one measurement + if(length(which(!is.na(wrk$qfqm$crdCo2$qfRngTemp))) > 0){ + # #determine the end time of each measurement + wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 60) + #delete row if last timeBgn and timeEnd is NA + wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] + #if last timeEnd is NA, replce that time to the last time value in data$time + wrk$idx$timeEnd <- as.POSIXct(ifelse(is.na(wrk$idx$timeEnd), data$time[length(data$time)], wrk$idx$timeEnd), origin = "1970-01-01", tz = "UTC") + + whrSamp <- wrk$idx$idxBgn[1]:wrk$idx$idxEnd[1] + if (length (wrk$idx$idxBgn) > 1 ){ + for(ii in 2:length (wrk$idx$idxBgn)){ + whrSamp <- c(whrSamp, wrk$idx$idxBgn[ii]:wrk$idx$idxEnd[ii]) + } + } + wrk$data[-whrSamp, ] <- NaN + for (idxSens in names(wrk$qfqm)){ + #replace all qf that not belong to that measurement level by NaN + wrk$qfqm[[idxSens]][-whrSamp, 1:length(wrk$qfqm[[idxSens]])] <- NaN + } + } + + for(idxAgr in c(1:length(idxTime[[paste0(PrdAgr, "min")]]$Bgn))) { + #idxAgr <- 1 + + ## grab data at the selected mask data + # for data + wrk$inpMask$data <- list() + idxLvLPrdAgr <- paste0(lvl, "_", sprintf("%02d", PrdAgr), "m") + wrk$inpMask$data <- wrk$data[idxTime[[paste0(PrdAgr, "min")]]$Bgn[idxAgr]:idxTime[[paste0(PrdAgr, "min")]]$End[idxAgr],] + + # for qfqm + wrk$inpMask$qfqm <- list() + lapply(names(wrk$qfqm), function (x) wrk$inpMask$qfqm[[x]] <<- wrk$qfqm[[x]][idxTime[[paste0(PrdAgr, "min")]]$Bgn[idxAgr]:idxTime[[paste0(PrdAgr, "min")]]$End[idxAgr],]) + + #qfqm processing + rpt[[idxAgr]] <- eddy4R.qaqc::wrap.dp01.qfqm.eddy( + qfInp = wrk$inpMask$qfqm, + MethMeas = "ecse", + TypeMeas = "vali", + RptExpd = RptExpd, + dp01 = dp01, + idGas = wrk$inpMask$data$idGas + ) + + #grab and add both time begin and time end to rpt + rpt[[idxAgr]]$timeBgn <- list() + rpt[[idxAgr]]$timeEnd <- list() + + #output time for qf dp01; do not output reference gas + for(idxVar in names(wrk$data)[which(!(names(wrk$data) %in% c("rtioMoleDryCh4Refe", "idGas")))]){ + rpt[[idxAgr]]$timeBgn[[idxVar]] <- data$time[idxTime[[paste0(PrdAgr, "min")]]$Bgn[idxAgr]] + rpt[[idxAgr]]$timeEnd[[idxVar]] <- data$time[idxTime[[paste0(PrdAgr, "min")]]$End[idxAgr]] + }; rm(idxVar) + + }; #rm(idxAgr) + } #end of PrdAgr == 30 + }#end of TypeMeas %in% "vali" if statement + }##end of dp01 if statement + #calculate dp01 for "isoH2o" ######################################################################################## if (dp01 %in% c("isoH2o")){ #during sampling period @@ -1370,6 +1835,8 @@ wrap.dp01.qfqm.ecse <- function( }#end of dp01 if statement + #remove NULL list from rpt + rpt <- rpt[!sapply(rpt, is.null)] #return results return(rpt) diff --git a/pack/eddy4R.qaqc/R/wrap.dp01.qfqm.eddy.R b/pack/eddy4R.qaqc/R/wrap.dp01.qfqm.eddy.R index 44b3fb01..fa997ed3 100644 --- a/pack/eddy4R.qaqc/R/wrap.dp01.qfqm.eddy.R +++ b/pack/eddy4R.qaqc/R/wrap.dp01.qfqm.eddy.R @@ -7,12 +7,12 @@ #' @description Wrapper function. Calculate quality metrics, alpha and beta quality metrics, and final quality flag for the NEON eddy-covariance turbulent and storage exchange L1 data products. -#' @param qfInp A list of data frame containing the input quality flag data that related to L1 data products are being grouped. Of class integer". [-] -#' @param MethMeas A vector of class "character" containing the name of measurement method (eddy-covariance turbulent exchange or storage exchange), MethMeas = c("ecte", "ecse"). Defaults to "ecse". [-] +#' @param qfInp A list of data frame containing the input quality flag data that related to L1 data products are being grouped. Of class integer". [-] +#' @param MethMeas A vector of class "character" containing the name of measurement method (eddy-covariance turbulent exchange or storage exchange), MethMeas = c("ecte", "ecse"). Defaults to "ecse". [-] #' @param TypeMeas A vector of class "character" containing the name of measurement type (sampling or validation), TypeMeas = c("samp", "ecse"). Defaults to "samp". [-] #' @param RptExpd A logical parameter that determines if the full quality metric \code{qm} is output in the returned list (defaults to FALSE). #' @param dp01 A vector of class "character" containing the name of NEON ECTE and ECSE L1 data products which the flags are being grouped, \cr -#' c("envHut", "co2Turb", "h2oTurb", "isoCo2", "isoH2o", "soni", "soniAmrs", "tempAirLvl", "tempAirTop"). Defaults to "co2Turb". [-] +#' c("envHut", "co2Turb", "h2oTurb", "isoCo2", "isoH2o", "soni", "soniAmrs", "tempAirLvl", "tempAirTop"). Defaults to "co2Turb". [-] #' @param \code{idGas} A data frame contianing gas ID for isoCo2 measurement. Need to provide when dp01 = "isoCo2". Default to NULL. [-] #' @return A list of: \cr @@ -27,7 +27,7 @@ #' @keywords NEON QAQC, quality flags, quality metrics -#' @examples +#' @examples #' #generate the fake quality flags for each sensor #' TimeBgn <- "2016-04-24 02:00:00.000" #' TimeEnd <- "2016-04-24 02:29:59.950" @@ -57,7 +57,7 @@ # revised original function to wrap.neon.dp01.qfqm () # added ECSE quality flags # Dave Durden (2017-04-24) -# Changed output to dataframes, added switch +# Changed output to dataframes, added switch # for expanded output and updated the output data type. # Natchaya P-Durden (2017-08-02) # added idGas and replaced isopCo2 and isopH2o by isoCo2 and isoH2o @@ -70,14 +70,20 @@ # rename function from wrap.neon.dp01.qfqm() to wrap.dp01.qfqm.eddy() # Natchaya P-Durden (2019-07-23) # adding one row with qf = -1 in the empty dataframe to eliminate code break in def.qf.finl() +# Chris Florian (2021-04-09) +# adding ch4Conc to the dp01 list +# Chris Florian (2021-11-08) +# adding logic to allow for more missing data due to variable picarro sampling frequency +# Chris Florian (2021-11-12) +# setting the ch4Conc qmBeta weighting to 0.2, this is the lowest weighing we can assign while also flagging intervals that are fully missing ############################################################################################## wrap.dp01.qfqm.eddy <- function( qfInp = list(), MethMeas = c("ecte", "ecse")[1], - TypeMeas = c("samp", "vali")[1], + TypeMeas = c("samp", "vali")[1], RptExpd = FALSE, - dp01 = c("envHut", "co2Turb", "h2oTurb", "co2Stor", "h2oStor", "isoCo2", "isoH2o", "soni", "soniAmrs", "tempAirLvl", "tempAirTop")[1], + dp01 = c("envHut", "co2Turb", "h2oTurb", "co2Stor", "h2oStor", "isoCo2", "ch4Conc", "isoH2o", "soni", "soniAmrs", "tempAirLvl", "tempAirTop")[1], idGas =NULL ) { @@ -86,13 +92,20 @@ wrap.dp01.qfqm.eddy <- function( tmp <- list() #grouping qf inp <- eddy4R.qaqc::def.dp01.grp.qf(qfInp = qfInp, MethMeas = MethMeas, TypeMeas = TypeMeas, dp01=dp01, idGas = idGas) - + #adding one row with qf = -1 in the empty dataframe to eliminate code break in def.qf.finl lapply(names(inp), function(x) if (nrow(inp[[x]]) == 0) inp[[x]][1,] <<- -1) - + #calculate qmAlpha, qmBeta, qfFinl - tmp <- lapply(inp, FUN = eddy4R.qaqc::def.qf.finl) + #set qmBeta weight lower for methane to account for lower sampling frequency, use default 2:1 weighing for all other data products + if (dp01 %in% c("ch4Conc")){ + tmp <- lapply(inp, FUN = eddy4R.qaqc::def.qf.finl, WghtAlphBeta=c(2,0.2)) + } else { + tmp <- lapply(inp, FUN = eddy4R.qaqc::def.qf.finl) + } + + #assign default qfSciRevw lapply(names(tmp), function(x) tmp[[x]]$qfqm$qfSciRevw <<- 0) #Only report expanded quality metrics if producing expanded file @@ -103,28 +116,28 @@ wrap.dp01.qfqm.eddy <- function( lapply(names(tmp), function(x) rpt$qm[[x]] <<- tmp[[x]]$qm) #Add units to the output of expanded quality metrics for(idxVar in names(rpt$qm)){ - lapply(names(rpt$qm[[idxVar]]), function(x) attr(rpt$qm[[idxVar]][[x]], which = "unit") <<- "-") + lapply(names(rpt$qm[[idxVar]]), function(x) attr(rpt$qm[[idxVar]][[x]], which = "unit") <<- "-") } } - + #assign return results for basic results lapply(names(tmp), function(x) rpt$qmAlph[[x]] <<- tmp[[x]]$qfqm$qmAlph) lapply(names(tmp), function(x) rpt$qmBeta[[x]] <<- tmp[[x]]$qfqm$qmBeta) lapply(names(tmp), function(x) rpt$qfFinl[[x]] <<- as.integer(tmp[[x]]$qfqm$qfFinl)) lapply(names(tmp), function(x) rpt$qfSciRevw[[x]] <<- as.integer(tmp[[x]]$qfqm$qfSciRevw)) - + # Convert output to dataframe's - rpt$qmAlph <- data.frame(t(rpt$qmAlph), row.names = NULL) - rpt$qmBeta <- data.frame(t(rpt$qmBeta), row.names = NULL) - rpt$qfFinl <- data.frame(t(rpt$qfFinl), row.names = NULL) - rpt$qfSciRevw <- data.frame(t(rpt$qfSciRevw), row.names = NULL) - + rpt$qmAlph <- base::rbind.data.frame(rpt$qmAlph) + rpt$qmBeta <- base::rbind.data.frame(rpt$qmBeta) + rpt$qfFinl <- base::rbind.data.frame(rpt$qfFinl) + rpt$qfSciRevw <- base::rbind.data.frame(rpt$qfSciRevw) + lapply(names(rpt$qmAlph), function(x) attr(rpt$qmAlph[[x]], which = "unit") <<- "-") lapply(names(rpt$qmBeta), function(x) attr(rpt$qmBeta[[x]], which = "unit") <<- "-") lapply(names(rpt$qfFinl), function(x) attr(rpt$qfFinl[[x]], which = "unit") <<- "NA") lapply(names(rpt$qfSciRevw), function(x) attr(rpt$qfSciRevw[[x]], which = "unit") <<- "NA") #return results return(rpt) - + } -# end function wrap.neon.ecte.dp01.qfqm() \ No newline at end of file +# end function wrap.neon.ecte.dp01.qfqm() diff --git a/pack/eddy4R.qaqc/R/wrap.qf.rmv.data.R b/pack/eddy4R.qaqc/R/wrap.qf.rmv.data.R index 899fb35e..a3714e42 100644 --- a/pack/eddy4R.qaqc/R/wrap.qf.rmv.data.R +++ b/pack/eddy4R.qaqc/R/wrap.qf.rmv.data.R @@ -41,6 +41,10 @@ # replaced dfQf by inpQf # Natchaya P-Durden (2019-03-12) # added Sens and qfRmv into the function parameter list +# David Durden (2020-07-14) +# added despiking routine and output qfSpk and qfNull +# David Durden (2021-10-7) +# not removing identified spike in presAtm data stream as patch for 2021 data release ############################################################################################## @@ -68,12 +72,43 @@ wrap.qf.rmv.data <- function( #Apply names to the output list base::names(outList) <- Sens + + #Despiking routine +base::lapply(Sens, function(x){ + #x <- Sens[1] #for testing + #print(x) + varDspk <- names(outList[[x]]$inpData)[!names(outList[[x]]$inpData) %in% c("time","idx","frtSet00")] + base::lapply(varDspk, function(y){ + #print(y) + #y <- "asrpCo2" + tmp <- eddy4R.qaqc::def.dspk.br86( + # input data, univariate vector of integers or numerics + dataInp = outList[[x]]$inpData[[y]][], + # filter width + WndwFilt = as.numeric(ramattribs(inpList$data[[x]][[y]])$`Dspk$Br86$NumWndw`), + # initial number/step size of histogram bins + NumBin = as.numeric(ramattribs(inpList$data[[x]][[y]])$`Dspk$Br86$NumBin`), + # resolution threshold + ThshReso = as.numeric(ramattribs(inpList$data[[x]][[y]])$`Dspk$Br86$MaxReso`) + ) #Remove high frequency data that is flagged by sensor specific flags or plausibility tests flags + if(!(x == "irgaTurb" & y == "presAtm")){ + outList[[x]]$inpData[[y]] <<- tmp$dataOut + } #Not removing qfSpk for irgaTurb presAtm TODO: fix once we have a better solution + + #Create flag name + nameQf <- base::paste0("qfSpk",base::toupper(base::substring(y,1,1)),base::substring(y,2,base::nchar(y))) + #Output despiking flag + outList[[x]]$qfSpk[[nameQf]] <<- tmp$qfSpk + + }) #End lapply for variables + })#End lapply around sensors #Applying the bad quality flags to the reported output data base::lapply(base::names(outList), function(x) { #Outputting the data ffdf's rpt$data[[x]] <<- ff::as.ffdf(outList[[x]]$inpData) rpt$data[[x]] <<- eddy4R.base::def.unit.var(samp = rpt$data[[x]], refe = inpList$data[[x]]) #Copy units + rpt$qfqm[[x]] <<- ff::as.ffdf(base::cbind(rpt$qfqm[[x]][], outList[[x]]$qfNull, as.data.frame(outList[[x]]$qfSpk))) }) #If verbose is true write out all the information about the quality flags applied to the raw data diff --git a/pack/eddy4R.qaqc/R/wrap.qfqm.rpt.R b/pack/eddy4R.qaqc/R/wrap.qfqm.rpt.R index 5e205410..0b336bf2 100644 --- a/pack/eddy4R.qaqc/R/wrap.qfqm.rpt.R +++ b/pack/eddy4R.qaqc/R/wrap.qfqm.rpt.R @@ -11,6 +11,7 @@ #' @param Dp A vector of data products to check the quality report for. Defaults to NULL which will test all dp's found in HDF5 file. Of type character. [-] #' @param Vrbs A logical parameter that determines if all failed quality times should be reported (defaults to FALSE). +#' @param MethQm A character string that determines which quality metric to pull ("Pass", "Fail", "Na"). #' @return A list for all dp's provided including either summary or verbose quality report: \cr #' \code{qm} quality metrics (percent) passed sub-data products if Vrbs = TRUE. [percent] \cr @@ -33,20 +34,24 @@ # changelog and author contributions / copyrights # David Durden (2019-04-30) +# Original creation +# David Durden (2020-03-18) +# Adding argument to grab quality metric desired for submetrics ############################################################################################## # start function wrap.qfqm.rpt() wrap.qfqm.rpt <- function( - File, + FileName, Dp = NULL, - Vrbs = FALSE + Vrbs = FALSE, + MethQm = c("Pass", "Fail","Na")[1] ) { #Check if dp's are provided for quality report, otherwise check all dp's in qfqm portion of HDF5 file if(is.null(Dp)){ # List of objects in the HDF5 file - listObj <- rhdf5::h5ls(file = fileName, datasetinfo = FALSE) + listObj <- rhdf5::h5ls(file = FileName, datasetinfo = FALSE) #Vector of data product names in the qfqm tabs in the HDF5 file Dp <- listObj[grep(pattern = "/qfqm$", x = listObj$group),"name"] } # End of if statement is.null(dp) if statement @@ -55,7 +60,7 @@ if(is.null(Dp)){ qfqmVarBase <- c("timeBgn","timeEnd","qfFinl","qfSciRevw","qmAlph","qmBeta") #Extract all the groups, data, and attributes from the HDF5 file -listHdf5 <- eddy4R.base::def.hdf5.extr(FileInp = fileName) +listHdf5 <- eddy4R.base::def.hdf5.extr(FileInp = FileName) #Initialize lists qfqm <- list() @@ -74,7 +79,7 @@ for (idxDp in Dp) { rpt <- list() #Check if any final quality flags (qfFinl) are tripped - if(sum(x$qfFinl) > 0) { + if(sum(x$qfFinl, na.rm = TRUE) > 0) { #Subset rows where the final quality flag is failed rpt <- x[x$qfFinl == 1,] @@ -83,7 +88,7 @@ for (idxDp in Dp) { qfqmVarBaseSub <- dplyr::intersect(qfqmVarBase, base::names(rpt)) #Determine all qf expanded variables - qfqmVarExp <- base::grep(pattern = "Pass", x = dplyr::setdiff(base::names(rpt),qfqmVarBaseSub), value = TRUE) + qfqmVarExp <- base::grep(pattern = MethQm, x = dplyr::setdiff(base::names(rpt),qfqmVarBaseSub), value = TRUE) #Rearrange the data with the base variables at the front of the data.frame rpt <- rpt[,c(qfqmVarBaseSub,qfqmVarExp)] ############################################################################################################# @@ -100,7 +105,8 @@ for (idxDp in Dp) { qmMean <- as.data.frame(t(colMeans(rpt[,grep(pattern = "qm", names(rpt))], na.rm = TRUE))) #Convert units to percent - if(length(qmMean) > 0) qmMean <- eddy4R.base::def.unit.conv(qmMean, unitFrom = "-", unitTo = "%", MethGc = FALSE) + #if(length(qmMean) > 0) qmMean <- eddy4R.base::def.unit.conv(qmMean, unitFrom = "-", unitTo = "%", MethGc = FALSE) + if(length(qmMean) > 0) qmMean <- qmMean * 100 #Check if qmAlph and qmBeta are present to order the quality metrics if("qmAlph" %in% names(qmMean) & "qmBeta" %in% names(qmMean)){ diff --git a/pack/eddy4R.qaqc/man/def.agr.file.dp00.Rd b/pack/eddy4R.qaqc/man/def.agr.file.dp00.Rd index 644a7dbe..ce7b6fb1 100644 --- a/pack/eddy4R.qaqc/man/def.agr.file.dp00.Rd +++ b/pack/eddy4R.qaqc/man/def.agr.file.dp00.Rd @@ -4,8 +4,15 @@ \alias{def.agr.file.dp00} \title{Definition function: Combine individual L0 data streams from file} \usage{ -def.agr.file.dp00(dirFile = ".", nameFile, nameVar, unitVar, Freq, - FmtTime = "\%d-\%b-\%Y \%I.\%M.\%OS \%p", Tz = "GMT") +def.agr.file.dp00( + dirFile = ".", + nameFile, + nameVar, + unitVar, + Freq, + FmtTime = "\%d-\%b-\%Y \%I.\%M.\%OS \%p", + Tz = "GMT" +) } \arguments{ \item{dirFile}{Optional. A character string indicating the directory in which the files to combine are stored. Default is the current working directory.} diff --git a/pack/eddy4R.qaqc/man/def.dp01.grp.qf.Rd b/pack/eddy4R.qaqc/man/def.dp01.grp.qf.Rd index 4c6a5b4a..f29f81c9 100644 --- a/pack/eddy4R.qaqc/man/def.dp01.grp.qf.Rd +++ b/pack/eddy4R.qaqc/man/def.dp01.grp.qf.Rd @@ -4,10 +4,14 @@ \alias{def.dp01.grp.qf} \title{Definition function: Grouping the quality flags for each of NEON ECTE and ECSE L1 data product} \usage{ -def.dp01.grp.qf(qfInp = list(), MethMeas = c("ecte", "ecse")[1], - TypeMeas = c("samp", "vali")[1], dp01 = c("envHut", "co2Turb", - "h2oTurb", "co2Stor", "h2oStor", "isoCo2", "isoH2o", "soni", "amrs", - "tempAirLvl", "tempAirTop")[1], idGas = NULL) +def.dp01.grp.qf( + qfInp = list(), + MethMeas = c("ecte", "ecse")[1], + TypeMeas = c("samp", "vali")[1], + dp01 = c("envHut", "co2Turb", "h2oTurb", "co2Stor", "h2oStor", "isoCo2", "ch4Conc", + "isoH2o", "soni", "amrs", "tempAirLvl", "tempAirTop")[1], + idGas = NULL +) } \arguments{ \item{qfInp}{A list of data frame containing the input quality flag data that related to L1 data products are being grouped. Of class integer". [-]} @@ -17,7 +21,7 @@ def.dp01.grp.qf(qfInp = list(), MethMeas = c("ecte", "ecse")[1], \item{TypeMeas}{A vector of class "character" containing the name of measurement type (sampling or validation), TypeMeas = c("samp", "ecse"). Defaults to "samp". [-]} \item{dp01}{A vector of class "character" containing the name of NEON ECTE and ECSE L1 data products which the flags are being grouped, \cr -c("envHut", "co2Turb", "h2oTurb", "isoCo2", "isoH2o", "soni", "amrs", "tempAirLvl", "tempAirTop"). Defaults to "co2Turb". [-]} +c("envHut", "co2Turb", "h2oTurb", "isoCo2", "ch4Conc", isoH2o", "soni", "amrs", "tempAirLvl", "tempAirTop"). Defaults to "co2Turb". [-]} \item{idGas}{A data frame contianing gas ID for isoCo2 measurement. Need to provide when dp01 = "isoCo2". Default to NULL. [-]} } diff --git a/pack/eddy4R.qaqc/man/def.dspk.br86.Rd b/pack/eddy4R.qaqc/man/def.dspk.br86.Rd index 1f9a8e83..07df4dea 100644 --- a/pack/eddy4R.qaqc/man/def.dspk.br86.Rd +++ b/pack/eddy4R.qaqc/man/def.dspk.br86.Rd @@ -4,8 +4,13 @@ \alias{def.dspk.br86} \title{Definition function: Median filter de-spiking} \usage{ -def.dspk.br86(dataInp, WndwFilt = 9, NumBin = 2, ThshReso = 10, - FracRealMin = 0.025) +def.dspk.br86( + dataInp, + WndwFilt = 9, + NumBin = 2, + ThshReso = 10, + FracRealMin = 0.025 +) } \arguments{ \item{dataInp}{Required. A univariate vector of integers or numerics of Input data} diff --git a/pack/eddy4R.qaqc/man/def.dspk.wndw.Rd b/pack/eddy4R.qaqc/man/def.dspk.wndw.Rd index aed2e582..f360abe7 100644 --- a/pack/eddy4R.qaqc/man/def.dspk.wndw.Rd +++ b/pack/eddy4R.qaqc/man/def.dspk.wndw.Rd @@ -4,11 +4,15 @@ \alias{def.dspk.wndw} \title{Definition function: Determine spike locations using window-based statistics} \usage{ -def.dspk.wndw(data, Trt = list(AlgClas = c("mean", "median")[2], - NumPtsWndw = c(11, 101)[2], NumPtsSlid = 1, ThshStd = c(3.5, 20)[2], - NaFracMax = 0.1, Infl = 0, IterMax = Inf, NumPtsGrp = c(4, 10)[2], NaTrt - = c("approx", "omit")[2]), Cntl = list(NaOmit = c(TRUE, FALSE)[2], Prnt - = c(TRUE, FALSE)[1], Plot = c(TRUE, FALSE)[1]), Vrbs = FALSE) +def.dspk.wndw( + data, + Trt = list(AlgClas = c("mean", "median")[2], NumPtsWndw = c(11, 101)[2], NumPtsSlid = + 1, ThshStd = c(3.5, 20)[2], NaFracMax = 0.1, Infl = 0, IterMax = Inf, NumPtsGrp = + c(4, 10)[2], NaTrt = c("approx", "omit")[2]), + Cntl = list(NaOmit = c(TRUE, FALSE)[2], Prnt = c(TRUE, FALSE)[1], Plot = c(TRUE, + FALSE)[1]), + Vrbs = FALSE +) } \arguments{ \item{data}{Required input. A data frame or matrix containing the data to be evaluated.} diff --git a/pack/eddy4R.qaqc/man/def.plau.Rd b/pack/eddy4R.qaqc/man/def.plau.Rd index 130b1681..35fded23 100644 --- a/pack/eddy4R.qaqc/man/def.plau.Rd +++ b/pack/eddy4R.qaqc/man/def.plau.Rd @@ -4,14 +4,18 @@ \alias{def.plau} \title{Definition function: Plausibility tests (Range, Step, Persistence, Null, Gap)} \usage{ -def.plau(data, time = as.POSIXlt(seq.POSIXt(from = Sys.time(), by = - "sec", length.out = length(data[, 1]))), RngMin = apply(data, 2, min, - na.rm = TRUE), RngMax = apply(data, 2, max, na.rm = TRUE), +def.plau( + data, + time = as.POSIXlt(seq.POSIXt(from = Sys.time(), by = "sec", length.out = nrow(data))), + RngMin = apply(data, 2, min, na.rm = TRUE), + RngMax = apply(data, 2, max, na.rm = TRUE), DiffStepMax = apply(abs(apply(data, 2, diff)), 2, max, na.rm = TRUE), - DiffPersMin = rep.int(0, length(data)), WndwPers = 60 * - median(abs(diff(time)), na.rm = TRUE) * rep.int(1, length(data)), - TestNull = rep(FALSE, length(data)), NumGap = rep(length(data[, 1]) + - 1, length(data)), Vrbs = FALSE) + DiffPersMin = rep.int(0, ncol(data)), + WndwPers = 60 * median(abs(diff(time)), na.rm = TRUE) * rep.int(1, ncol(data)), + TestNull = rep(FALSE, ncol(data)), + NumGap = rep(nrow(data) + 1, ncol(data)), + Vrbs = FALSE +) } \arguments{ \item{data}{Required input. A data frame containing the data to be evaluated (do not include the time stamp vector here).} diff --git a/pack/eddy4R.qaqc/man/def.plot.dp01.qfqm.Rd b/pack/eddy4R.qaqc/man/def.plot.dp01.qfqm.Rd index 10c2fdc9..4f1c66c3 100644 --- a/pack/eddy4R.qaqc/man/def.plot.dp01.qfqm.Rd +++ b/pack/eddy4R.qaqc/man/def.plot.dp01.qfqm.Rd @@ -4,10 +4,12 @@ \alias{def.plot.dp01.qfqm} \title{Definition function: Plot quality flags and quality metrics (basic L1 data products)} \usage{ -def.plot.dp01.qfqm(dataDp01, WndwTime = c(min(dataDp01$timeAgrBgn), - max(dataDp01$timeAgrBgn)), NameQmPlot = sub("Pass", "", - names(dataDp01$dataAgr[[1]][grep("Pass", - names(dataDp01$dataAgr[[1]]))]))) +def.plot.dp01.qfqm( + dataDp01, + WndwTime = c(min(dataDp01$timeAgrBgn), max(dataDp01$timeAgrBgn)), + NameQmPlot = sub("Pass", "", names(dataDp01$dataAgr[[1]][grep("Pass", + names(dataDp01$dataAgr[[1]]))])) +) } \arguments{ \item{dataDp01}{Required input. A list output from wrap.dp01.qfqm.R of: \cr diff --git a/pack/eddy4R.qaqc/man/def.qf.ecte.Rd b/pack/eddy4R.qaqc/man/def.qf.ecte.Rd index d34344c1..314200d4 100644 --- a/pack/eddy4R.qaqc/man/def.qf.ecte.Rd +++ b/pack/eddy4R.qaqc/man/def.qf.ecte.Rd @@ -4,8 +4,13 @@ \alias{def.qf.ecte} \title{Definition function: Create fake flags for ECTE sensors} \usage{ -def.qf.ecte(TimeBgn, TimeEnd, Freq = 20, Sens = c("soni", "irga", - "irgaMfcSamp", "soniAmrs")[1], PcntQf = 0.05) +def.qf.ecte( + TimeBgn, + TimeEnd, + Freq = 20, + Sens = c("soni", "irgaTurb", "mfcSampTurb", "amrs")[1], + PcntQf = 0.05 +) } \arguments{ \item{TimeBgn}{is the beginning time of the period to generate flags.} diff --git a/pack/eddy4R.qaqc/man/def.qf.irga.vali.Rd b/pack/eddy4R.qaqc/man/def.qf.irga.vali.Rd index 0ebc7a60..91896d5e 100644 --- a/pack/eddy4R.qaqc/man/def.qf.irga.vali.Rd +++ b/pack/eddy4R.qaqc/man/def.qf.irga.vali.Rd @@ -4,8 +4,11 @@ \alias{def.qf.irga.vali} \title{Definition function: Validation flag for IRGA} \usage{ -def.qf.irga.vali(data, Sens = c("irgaMfcSamp", "valvValiNemaTurb")[1], - qfGas = NULL) +def.qf.irga.vali( + data, + Sens = c("irgaMfcSamp", "valvValiNemaTurb")[1], + qfGas = NULL +) } \arguments{ \item{data}{A data.frame containing the L0p input IRGA sampling mass flow controller data or the IRGA validation soleniod valves data at native resolution. Of type numeric or integer. [User-defined]} diff --git a/pack/eddy4R.qaqc/man/def.qf.rmv.data.Rd b/pack/eddy4R.qaqc/man/def.qf.rmv.data.Rd index 685a4157..8b79b2d5 100644 --- a/pack/eddy4R.qaqc/man/def.qf.rmv.data.Rd +++ b/pack/eddy4R.qaqc/man/def.qf.rmv.data.Rd @@ -4,8 +4,14 @@ \alias{def.qf.rmv.data} \title{Definition function: to remove high frequency data points that have failed quality flags} \usage{ -def.qf.rmv.data(inpData, inpQf, Sens = NULL, qfRmv = NULL, - Vrbs = FALSE, TypeData = c("integer", "real")[1]) +def.qf.rmv.data( + inpData, + inpQf, + Sens = NULL, + qfRmv = NULL, + Vrbs = FALSE, + TypeData = c("integer", "real")[1] +) } \arguments{ \item{inpData}{Input data.frame for data to be removed from based on quality flags} diff --git a/pack/eddy4R.qaqc/man/def.qi.qf.Rd b/pack/eddy4R.qaqc/man/def.qi.qf.Rd new file mode 100644 index 00000000..c47950a0 --- /dev/null +++ b/pack/eddy4R.qaqc/man/def.qi.qf.Rd @@ -0,0 +1,47 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/def.qi.qf.R +\name{def.qi.qf} +\alias{def.qi.qf} +\title{Definition function: Quality Indicators} +\usage{ +def.qi.qf(qf) +} +\arguments{ +\item{qf}{A data frame or list of data.frames containing quality flags, class integer. Each column contains the quality flag values [-1,0,1] for that flag. Note: This is the Vrbs output from def.plau, def.dspk.wndw, and def.dspk.br86. See def.conv.qf.vrbs for converting from non-verbose to verbose output.} +} +\value{ +A dataframe containing quality indicators of failed (1), pass (0), and NA (-1) for each of the individual flag defined by input \code{qf}. +} +\description{ +Function definition. Convert quality flags (qf) to quality indicators (qi) by changing names. Performed for the entire set of input data. +} +\examples{ +qfA <- c(0,0,0,0,0,1,1,1,1,1,0,0,0,0,1) +qfB <- c(0,0,-1,-1,-1,1,1,0,0,0,0,0,0,0,0) +qfC <- c(0,1,1,0,0,0,0,0,0,0,0,0,-1,-1,-1) +qfD <- data.frame(qfE = c(0,1,1,0,0,0,0,0,0,0,0,0,-1,-1,-1), qfF = c(0,1,1,0,0,0,0,-1,0,0,0,0,-1,-1,-1)) +test<-list() +test$dfQf <- data.frame(qfA,qfB,qfC) +test$listQf <- list(test$dfQf,qfD) +test$qi<- def.qi.qf(qf=test$listQf) +} +\references{ +License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007. \cr +NEON Algorithm Theoretical Basis Document: Quality Flags and Quality Metrics for TIS Data Products (NEON.DOC.001113) \cr +Smith, D.E., Metzger, S., and Taylor, J.R.: A transparent and transferable framework for tracking quality information in large datasets. PLoS ONE, 9(11), e112249.doi:10.1371/journal.pone.0112249, 2014. \cr +} +\seealso{ +\code{\link[eddy4R.qaqc]{wrap.qfqm.dp01}} +\code{\link[eddy4R.qaqc]{def.dp01.grp.qf}} +\code{\link[eddy4R.qaqc]{wrap.dp01.qfqm.eddy}} +\code{\link[eddy4R.qaqc]{def.qf.finl.R}} +\code{\link[eddy4R.qaqc]{def.qm.R}} +} +\author{ +David Durden \email{eddy4R.info@gmail.com} +} +\keyword{NEON} +\keyword{QAQC,} +\keyword{flags,} +\keyword{metrics} +\keyword{quality} diff --git a/pack/eddy4R.qaqc/man/wrap.dp01.qfqm.Rd b/pack/eddy4R.qaqc/man/wrap.dp01.qfqm.Rd index 1c4c872d..fdefb660 100644 --- a/pack/eddy4R.qaqc/man/wrap.dp01.qfqm.Rd +++ b/pack/eddy4R.qaqc/man/wrap.dp01.qfqm.Rd @@ -4,14 +4,17 @@ \alias{wrap.dp01.qfqm} \title{Wrapper function: Generate basic L1 data product, including descriptive statics, quality metrics, and final quality flag} \usage{ -wrap.dp01.qfqm(data, time = base::as.POSIXlt(base::seq.POSIXt(from = - base::Sys.time(), by = "sec", length.out = base::length(data[, 1]))), +wrap.dp01.qfqm( + data, + time = base::as.POSIXlt(base::seq.POSIXt(from = base::Sys.time(), by = "sec", + length.out = base::length(data[, 1]))), setQf = base::vector("list", base::length(data)), - qf = base::vector("list", base::length(data)), WndwAgr = 1800 * - stats::median(base::abs(base::diff(time)), na.rm = TRUE), - TimeBgn = NULL, TimeEnd = NULL, - NameQfExcl = base::as.list(base::character(length = - base::length(data)))) + qf = base::vector("list", base::length(data)), + WndwAgr = 1800 * stats::median(base::abs(base::diff(time)), na.rm = TRUE), + TimeBgn = NULL, + TimeEnd = NULL, + NameQfExcl = base::as.list(base::character(length = base::length(data))) +) } \arguments{ \item{data}{Required input. A data frame containing the L0' (calibrated raw) data evaluated (do not include the time stamp vector here).} diff --git a/pack/eddy4R.qaqc/man/wrap.dp01.qfqm.ecse.Rd b/pack/eddy4R.qaqc/man/wrap.dp01.qfqm.ecse.Rd index 079a2a65..3d6a44a1 100644 --- a/pack/eddy4R.qaqc/man/wrap.dp01.qfqm.ecse.Rd +++ b/pack/eddy4R.qaqc/man/wrap.dp01.qfqm.ecse.Rd @@ -4,12 +4,25 @@ \alias{wrap.dp01.qfqm.ecse} \title{Wrapper function: Preprocessing and calculating quality metrics, alpha and beta quality metrics, and final quality flag for the NEON eddy-covariance stroage exchange L1 data products} \usage{ -wrap.dp01.qfqm.ecse(dp01 = c("co2Stor", "h2oStor", "tempAirLvl", - "tempAirTop", "isoCo2", "isoH2o")[1], RptExpd = FALSE, lvl, - lvlMfcSampStor = NULL, lvlMfcValiStor = NULL, lvlEnvHut = NULL, - lvlValv = NULL, lvlValvAux = NULL, lvlCrdH2oValvVali = NULL, - data = list(), qfInp = list(), TypeMeas = c("samp", "vali")[1], - PrdMeas, PrdAgr, idxTime = list()) +wrap.dp01.qfqm.ecse( + dp01 = c("co2Stor", "h2oStor", "tempAirLvl", "tempAirTop", "isoCo2", "isoH2o", + "ch4Conc")[1], + RptExpd = FALSE, + lvl, + lvlMfcSampStor = NULL, + lvlMfcValiStor = NULL, + lvlEnvHut = NULL, + lvlValv = NULL, + lvlValvAux = NULL, + lvlCrdH2oValvVali = NULL, + lvlCrdCo2Valv = NULL, + data = list(), + qfInp = list(), + TypeMeas = c("samp", "vali")[1], + PrdMeas, + PrdAgr, + idxTime = list() +) } \arguments{ \item{dp01}{A vector of class "character" containing the name of NEON ECSE dp01 which descriptive statistics are being calculated, \cr @@ -31,6 +44,8 @@ c("co2Stor", "h2oStor", "tempAirLvl", "tempAirTop", "isoCo2", "isoH2o"). Default \item{lvlCrdH2oValvVali}{Measurement level of crdH2oValvVali which apply to only dp01 equal to "isoH2o". Defaults to NULL. Of type character. [-]} +\item{lvlCrdCo2Valv}{Horizontal and vertical location for crdCo2 valve. Defaults to NULL. Of type character. [-]} + \item{data}{A list of data frame containing the input dp0p data that related to dp01 which qfqm are being calculated. Of class integer". [User defined]} \item{qfInp}{A list of data frame containing the input quality flag data that related to dp01 are being grouped. Of class integer". [-]} diff --git a/pack/eddy4R.qaqc/man/wrap.dp01.qfqm.eddy.Rd b/pack/eddy4R.qaqc/man/wrap.dp01.qfqm.eddy.Rd index c1ebf009..0a9a1df5 100644 --- a/pack/eddy4R.qaqc/man/wrap.dp01.qfqm.eddy.Rd +++ b/pack/eddy4R.qaqc/man/wrap.dp01.qfqm.eddy.Rd @@ -4,11 +4,15 @@ \alias{wrap.dp01.qfqm.eddy} \title{Wrapper function: Calculate quality metrics, alpha and beta quality metrics, and final quality flag for the NEON eddy-covariance turbulent and stroage exchange L1 data products} \usage{ -wrap.dp01.qfqm.eddy(qfInp = list(), MethMeas = c("ecte", "ecse")[1], - TypeMeas = c("samp", "vali")[1], RptExpd = FALSE, - dp01 = c("envHut", "co2Turb", "h2oTurb", "co2Stor", "h2oStor", - "isoCo2", "isoH2o", "soni", "soniAmrs", "tempAirLvl", "tempAirTop")[1], - idGas = NULL) +wrap.dp01.qfqm.eddy( + qfInp = list(), + MethMeas = c("ecte", "ecse")[1], + TypeMeas = c("samp", "vali")[1], + RptExpd = FALSE, + dp01 = c("envHut", "co2Turb", "h2oTurb", "co2Stor", "h2oStor", "isoCo2", "ch4Conc", + "isoH2o", "soni", "soniAmrs", "tempAirLvl", "tempAirTop")[1], + idGas = NULL +) } \arguments{ \item{qfInp}{A list of data frame containing the input quality flag data that related to L1 data products are being grouped. Of class integer". [-]} diff --git a/pack/eddy4R.qaqc/man/wrap.qf.rmv.data.Rd b/pack/eddy4R.qaqc/man/wrap.qf.rmv.data.Rd index d370e020..28bfaf5c 100644 --- a/pack/eddy4R.qaqc/man/wrap.qf.rmv.data.Rd +++ b/pack/eddy4R.qaqc/man/wrap.qf.rmv.data.Rd @@ -4,8 +4,13 @@ \alias{wrap.qf.rmv.data} \title{Wrapper function: to remove high frequency data points that have failed quality flags} \usage{ -wrap.qf.rmv.data(inpList, Sens = NULL, qfRmv = NULL, Vrbs = FALSE, - MethMeas = c("ecte", "ecse")[1]) +wrap.qf.rmv.data( + inpList, + Sens = NULL, + qfRmv = NULL, + Vrbs = FALSE, + MethMeas = c("ecte", "ecse")[1] +) } \arguments{ \item{inpList}{List consisting of \code{ff::ffdf} file-backed objects, in the format provided by function \code{eddy4R.base::wrap.hdf5.read()}. Of types numeric and integer.} diff --git a/pack/eddy4R.qaqc/man/wrap.qfqm.rpt.Rd b/pack/eddy4R.qaqc/man/wrap.qfqm.rpt.Rd index 79b3ba6e..50238df1 100644 --- a/pack/eddy4R.qaqc/man/wrap.qfqm.rpt.Rd +++ b/pack/eddy4R.qaqc/man/wrap.qfqm.rpt.Rd @@ -4,14 +4,21 @@ \alias{wrap.qfqm.rpt} \title{Wrapper function: QFQM quality report for eddy-covariance dp04 HDF5 files} \usage{ -wrap.qfqm.rpt(File, Dp = NULL, Vrbs = FALSE) +wrap.qfqm.rpt( + FileName, + Dp = NULL, + Vrbs = FALSE, + MethQm = c("Pass", "Fail", "Na")[1] +) } \arguments{ -\item{File}{character vector for the full path to the HDF5 file to perform the quality report [-]} - \item{Dp}{A vector of data products to check the quality report for. Defaults to NULL which will test all dp's found in HDF5 file. Of type character. [-]} \item{Vrbs}{A logical parameter that determines if all failed quality times should be reported (defaults to FALSE).} + +\item{MethQm}{A character string that determines which quality metric to pull ("Pass", "Fail", "Na").} + +\item{File}{character vector for the full path to the HDF5 file to perform the quality report [-]} } \value{ A list for all dp's provided including either summary or verbose quality report: \cr diff --git a/pack/eddy4R.stor/DESCRIPTION b/pack/eddy4R.stor/DESCRIPTION index 73fbc2a9..a419d98c 100644 --- a/pack/eddy4R.stor/DESCRIPTION +++ b/pack/eddy4R.stor/DESCRIPTION @@ -15,4 +15,4 @@ Suggests: License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 Encoding: UTF-8 LazyData: true -RoxygenNote: 6.1.1 +RoxygenNote: 7.1.1 diff --git a/pack/eddy4R.stor/R/def.itpl.time.R b/pack/eddy4R.stor/R/def.itpl.time.R index c5b09b6d..bd8c5108 100644 --- a/pack/eddy4R.stor/R/def.itpl.time.R +++ b/pack/eddy4R.stor/R/def.itpl.time.R @@ -32,6 +32,18 @@ # original creation # Ke Xu (2018-07-01) # apply eddy4R terms: from gap to Wndw +# Natchaya P-Durden (2020-03-17) +# remove na value before applying linear interpolation if like that maxgap will not work +# Natchaya P-Durden (2020-04-01) +# added failsafe for not to break the zoo::na.approx function when timeFrac are duplicate +# Natchaya P-Durden (2020-04-24) +# added failsafe replace NaN in numSamp with zero +# David Durden (2020-07-10) +# added failsafe to make sure the time and data lengths are the same for cases with setLgth = 2 +# David Durden(2020-07-24) +# Failsafe if multiple lines exist, but removed due to NaN +# David Durden(2020-12-24) +# Added numSamp > 1 since na.omit below would remove later since variance can not be calculated ############################################################################################################## #Start of function call ############################################################################################################## @@ -66,6 +78,9 @@ def.itpl.time <- function( #convert to POSIXct, so the full date and time can be stored in as accessed as a single vector timeInp <- as.POSIXlt(dataInp$timeBgn, format="%Y-%m-%dT%H:%M:%OSZ", tz="UTC") + #failsafe replace NaN in numSamp with zero + dataInp$numSamp[is.na(dataInp$numSamp)] <- 0 + #if(idxDp == "co2Stor" | idxDp == "h2oStor"){ #timeBgn + numSamp/2/* 1/1Hz timeInp <- as.POSIXlt(timeInp + dataInp$numSamp/2*1/1, format="%Y-%m-%d %H:%M:%OS", tz="UTC") @@ -85,7 +100,7 @@ def.itpl.time <- function( #data: determine which datapoints to assess - setLgth <- length(which(!is.na(dataInp$mean))) + setLgth <- length(which(!is.na(dataInp$mean) & dataInp$numSamp > 1)) #Added numSamp > 1 since na.omit below would remove later since variance can not be calculated #less than 2 values (minimum required by approx() function) @@ -95,12 +110,24 @@ def.itpl.time <- function( #interpolate actual data } else { - + #Failsafe for not to break the zoo::na.approx function + #if only two data are available and as.integer(dataInp$timeFrac * 60) are the same value, add 1 to timeFrac of the 2nd value + tmpTimeFrac <- as.integer(dataInp$timeFrac * 60) + if (setLgth == 2 & tmpTimeFrac[1]==tmpTimeFrac[2]) tmpTimeFrac[2] <- tmpTimeFrac[2]+1 if(methItpl == "linear"){ + #remove na value if like that maxgap will not work + dataInp <- na.omit(dataInp) + #make sure use the right inpTime before interpolating + if (setLgth == 2 && length(tmpTimeFrac) == setLgth){ + inpTime <- tmpTimeFrac + }else{ + inpTime <- as.integer(dataInp$timeFrac * 60) + if (setLgth == 2 & inpTime[1]==inpTime[2]) inpTime[2] <- inpTime[2]+1 #Failsafe if multiple lines exist, but removed due to NaN + } #End if for setLgth == 2 and length of tmpTime == to setLgth rpt <- zoo::na.approx(object=as.vector(dataInp$mean), x=#dataInp$timeFrac - as.integer(dataInp$timeFrac * 60) + inpTime , xout=as.integer(timeFracOut * 60) - , method = "linear", maxgap=(WndwMax/60), na.rm=FALSE, rule=1, f=0) + , method = "linear", maxgap=(WndwMax/60), na.rm=FALSE, rule=1, f=0,ties = mean) } diff --git a/pack/eddy4R.stor/R/wrap.dp01.ecse.R b/pack/eddy4R.stor/R/wrap.dp01.ecse.R index cf88be7c..0bbd1832 100644 --- a/pack/eddy4R.stor/R/wrap.dp01.ecse.R +++ b/pack/eddy4R.stor/R/wrap.dp01.ecse.R @@ -14,6 +14,7 @@ #' @param lvlEnvHut Measurement level of envHut. Defaults to NULL. Of type character. [-] #' @param lvlValv Measurement level of irgaValvLvl, crdCo2ValvLvl, or crdH2oValvLvl. Defaults to NULL. Of type character. [-] #' @param lvlCrdH2oValvVali Measurement level of crdH2oValvVali which apply to only dp01 equal to "isoH2o". Defaults to NULL. Of type character. [-] +#' @param lvlCrdCo2Valv Horizontal and vertical location for crdCo2 valve. Defaults to NULL. Of type character. [-] #' @param data A list of data frame containing the input dp0p data that related to dp01 which descriptive statistics are being calculated. Of class integer". [User defined] #' @param qfInp A list of data frame containing the input quality flag data that related to dp01 are being grouped. Of class integer". [-] #' @param TypeMeas A vector of class "character" containing the name of measurement type (sampling or validation), TypeMeas = c("samp", "vali"). Defaults to "samp". [-] @@ -64,14 +65,32 @@ # rename function from wrap.neon.dp01.ecse() to wrap.dp01.ecse() # Natchaya P-Durden (2019-01-31) # using injNum instate of qfRngTmp to determine missing data +# Natchaya P-Durden (2020-03-12) +# In irgaCo2 an irgaH2o, not include the period when crdCo2 take over to measure at that level +# and irga have to move to measure next level +# Natchaya P-Durden (2020-03-25) +# added lvlCrdCo2Valv to the function's parameter +# David Durden (2020-05-15) +# failsafe for crd kickoff removal causing no data for entire day +# David Durden (2020-07-23) +# bug fix for valve issues where looks like consistently Stor data thrown off by Crd +# Chris Florian (2021-02-24) +# adding ch4Conc L1 data processing steps +# Chris Florian (2021-06-09) +# adding rtioMoleDryCh4Refe data to validation time period +# Chris Florian +# updating the columns used to select data from 15 to 8 for ch4Conc to get rid of extra dataframes +# Rich Fiorella (2022-09-20) +# based on analysis done for ATM ISO TWG, changing isoCo2 to 6 minute product from 9 minute ############################################################################################## wrap.dp01.ecse <- function( - dp01 = c("co2Stor", "h2oStor", "tempAirLvl", "tempAirTop", "isoCo2", "isoH2o")[1], + dp01 = c("co2Stor", "h2oStor", "tempAirLvl", "tempAirTop", "isoCo2", "isoH2o", "ch4Conc")[1], lvl, lvlMfcSampStor = NULL, lvlEnvHut = NULL, lvlValv = NULL, lvlCrdH2oValvVali = NULL, + lvlCrdCo2Valv = NULL, data = list(), qfInp = list(), TypeMeas = c("samp", "vali")[1], @@ -149,11 +168,13 @@ wrap.dp01.ecse <- function( wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$irgaStor$qfRngTemp, CritTime = 60) #delete row if last timeBgn and timeEnd is NA wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] + #replace last idxEnd > 86400 by 86400 + wrk$idx$idxEnd <- ifelse(wrk$idx$idxEnd > 86400, 86400, wrk$idx$idxEnd) #if last timeEnd is NA, replce that time to the last time value in data$time wrk$idx$timeEnd <- as.POSIXct(ifelse(is.na(wrk$idx$timeEnd), data$time[length(data$time)], wrk$idx$timeEnd), origin = "1970-01-01", tz = "UTC") #idxAgr2 <- 0 for (idxAgr in 1:length(wrk$idx$idxBgn)){ - #idxAgr <- 4 + #idxAgr <- 3 #only use the middle 2 min after the first 1 min wrk$inpMask$data <- list() @@ -195,6 +216,45 @@ wrap.dp01.ecse <- function( rpt[[idxAgr]]$timeEnd[[idxVar]] <- wrk$idx$timeEnd[idxAgr] } + #check if this period is the period that crdCo2 take over and irga have to move to measure other level + #and number of sample less than 10% (120-120*0.1) + if (dp01 == "co2Stor") {numSamp <- rpt[[idxAgr]]$numSamp$rtioMoleDryCo2} + if (dp01 == "h2oStor") {numSamp <- rpt[[idxAgr]]$numSamp$rtioMoleDryH2o} + if (data$crdCo2ValvLvl[[lvlCrdCo2Valv]]$lvlCrdCo2[wrk$idx$idxEnd[idxAgr]] == lvlIrga & numSamp < 108){ + rpt[[idxAgr]] <- NULL + } + + #Remove any empty lists in case valve issues + if(length(wrk$idx$idxBgn) == idxAgr) rpt <- Filter(Negate(is.null), rpt) + + #Check if after removing data for valve kickooff if no data remains + if(length(wrk$idx$idxBgn) == idxAgr && length(rpt) == 0){ + rpt[[1]] <- list() + + for(idxStat in NameStat){ + #idxStat <- NameStat[1] + rpt[[1]][[idxStat]] <- as.data.frame(matrix(NaN, nrow = 1, ncol = ncol(wrk$data))) + #assign name to each column + names(rpt[[1]][[idxStat]]) <- names(wrk$data) + #not report lvlIrga + rpt[[1]][[idxStat]] <- rpt[[1]][[idxStat]][which(!(names(rpt[[1]][[idxStat]]) %in% c("lvlIrga")))] + + }; rm(idxStat) + #add both time begin and time end to rpt + rpt[[1]]$timeBgn <- list() + rpt[[1]]$timeEnd <- list() + + #output time for dp01 + for(idxVar in names(wrk$data)[which(!(names(wrk$data) %in% c("lvlIrga")))]){ + rpt[[1]]$timeBgn[[idxVar]] <- data$time[1] + rpt[[1]]$timeEnd[[idxVar]] <- data$time[length(data$time)] + #unit + attributes(rpt[[1]]$mean[[idxVar]])$unit <- attributes(wrk$data[[idxVar]])$unit + + }; rm(idxVar) + + }#end failsafe of if no measurement data at all in the whole day once valve issue removed + #}# end of there is at least one data }; rm(idxAgr) @@ -236,6 +296,8 @@ wrap.dp01.ecse <- function( wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$irgaStor$qfRngTemp, CritTime = 60) #delete row if last timeBgn and timeEnd is NA wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] + #replace last idxEnd > 86400 by 86400 + wrk$idx$idxEnd <- ifelse(wrk$idx$idxEnd > 86400, 86400, wrk$idx$idxEnd) #if last timeEnd is NA, replce that time to the last time value in data$time wrk$idx$timeEnd <- as.POSIXct(ifelse(is.na(wrk$idx$timeEnd), data$time[length(data$time)], wrk$idx$timeEnd), origin = "1970-01-01", tz = "UTC") @@ -574,16 +636,16 @@ wrap.dp01.ecse <- function( #subset only wrk$qfqm$crdCo2 <- qfInp$crdCo2[[lvl]] - if (PrdMeas == PrdAgr) { - #PrdAgr <- 9 - #9 minutely sampling data + if (PrdMeas == PrdAgr) { + #PrdAgr <- 6 # RF - now a 6 minute product based on TWG input! + #6 minutely sampling data #idxLvLPrdAgr <- paste0(lvl, "_", sprintf("%02d", PrdAgr), "m") #rpt[[dp01]][[idxLvLPrdAgr]] <- list() #if there is at least one measurement if(length(which(!is.na(wrk$qfqm$crdCo2$qfRngTemp))) > 0){ #determine the index of each measurement - wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 60) + wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 240) #delete row if last timeBgn and timeEnd is NA wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] #if last timeEnd is NA, replce that time to the last time value in data$time @@ -668,7 +730,7 @@ wrap.dp01.ecse <- function( if(length(which(!is.na(wrk$qfqm$crdCo2$qfRngTemp))) > 0){ #determine the index of each measurement - wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 60) + wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 240) #delete row if last timeBgn and timeEnd is NA wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] #if last timeEnd is NA, replce that time to the last time value in data$time @@ -778,7 +840,7 @@ wrap.dp01.ecse <- function( #if there is at least one measurement if(length(which(!is.na(wrk$qfqm$crdCo2$qfRngTemp))) > 0){ #determine the end time of each measurement - wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 60) + wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 240) #delete row if last timeBgn and timeEnd is NA wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] #if last timeEnd is NA, replce that time to the last time value in data$time @@ -851,6 +913,374 @@ wrap.dp01.ecse <- function( + if (PrdMeas != PrdAgr) { + #PrdAgr <- 30 + #if there is at least one measurement + if(length(which(!is.na(wrk$qfqm$crdCo2$qfRngTemp))) > 0){ + # #determine the end time of each measurement + wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 240) + #delete row if last timeBgn and timeEnd is NA + wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] + #if last timeEnd is NA, replce that time to the last time value in data$time + wrk$idx$timeEnd <- as.POSIXct(ifelse(is.na(wrk$idx$timeEnd), data$time[length(data$time)], wrk$idx$timeEnd), origin = "1970-01-01", tz = "UTC") + + whrSamp <- wrk$idx$idxBgn[1]:wrk$idx$idxEnd[1] + if (length (wrk$idx$idxBgn) > 1 ){ + for(ii in 2:length (wrk$idx$idxBgn)){ + whrSamp <- c(whrSamp, wrk$idx$idxBgn[ii]:wrk$idx$idxEnd[ii]) + } + } + wrk$data[-whrSamp, ] <- NaN + } else {#end of if no measurement data at all in the whole day + tmpAttr <- list() + for (idxData in c("presEnvHut", "rhEnvHut", "rtioMoleWetH2oEnvHut", "tempEnvHut")){ + #defined attributes + tmpAttr[[idxData]] <- attributes(wrk$data[[idxData]]) + wrk$data[[idxData]] <- NaN + attributes(wrk$data[[idxData]]) <- tmpAttr[[idxData]] + } + } + + for(idxAgr in c(1:length(idxTime[[paste0(PrdAgr, "min")]]$Bgn))) { + #idxAgr <- 1 + + ## grab data at the selected mask data + # for data + wrk$inpMask$data <- list() + idxLvLPrdAgr <- paste0(lvl, "_", sprintf("%02d", PrdAgr), "m") + wrk$inpMask$data <- wrk$data[idxTime[[paste0(PrdAgr, "min")]]$Bgn[idxAgr]:idxTime[[paste0(PrdAgr, "min")]]$End[idxAgr],] + + #call wrap.dp01.R to calculate descriptive statistics + rpt[[idxAgr]] <- eddy4R.base::wrap.dp01( + # assign data: data.frame or list of type numeric or integer + data = wrk$inpMask$data#, + ) + + #units: + for (idxVar in names(rpt[[1]]$mean)){ + #idxVar <- names(rpt[[1]]$mean)[1] + attributes(rpt[[1]]$mean[[idxVar]])$unit <- attributes(wrk$data[[idxVar]])$unit + } + + #grab and add both time begin and time end to rpt + rpt[[idxAgr]]$timeBgn <- list() + rpt[[idxAgr]]$timeEnd <- list() + + #output time for dp01 + for(idxVar in names(wrk$data)){ + # rpt[[idxAgr2]]$timeBgn[[idxVar]] <- data$time[(whrEnd[idxAgr] - 20 - 2*60+1)] + # rpt[[idxAgr2]]$timeEnd[[idxVar]] <- data$time[(whrEnd[idxAgr] - 20)] + rpt[[idxAgr]]$timeBgn[[idxVar]] <- data$time[idxTime[[paste0(PrdAgr, "min")]]$Bgn[idxAgr]] + rpt[[idxAgr]]$timeEnd[[idxVar]] <- data$time[idxTime[[paste0(PrdAgr, "min")]]$End[idxAgr]] + }; rm(idxVar) + + }; #rm(idxAgr) + } #end of PrdAgr == 30 + }#end of TypeMeas %in% "vali" if statement + }##end of dp01 if statement + + + #calculate dp01 for "ch4Conc" ######################################################################################## + if (dp01 %in% c("ch4Conc")){ + #during sampling period + if (TypeMeas %in% "samp"){ + #assign lvlCrdCo2 for each measurement level + if (lvl == "000_010") {lvlCrdCo2 <- "lvl01"} + if (lvl == "000_020") {lvlCrdCo2 <- "lvl02"} + if (lvl == "000_030") {lvlCrdCo2 <- "lvl03"} + if (lvl == "000_040") {lvlCrdCo2 <- "lvl04"} + if (lvl == "000_050") {lvlCrdCo2 <- "lvl05"} + if (lvl == "000_060") {lvlCrdCo2 <- "lvl06"} + if (lvl == "000_070") {lvlCrdCo2 <- "lvl07"} + if (lvl == "000_080") {lvlCrdCo2 <- "lvl08"} + + #input the whole day data + wrk$data <- data.frame(stringsAsFactors = FALSE, + "rtioMoleWetCh4" = data$crdCo2[[lvl]]$rtioMoleWetCh4, + "rtioMoleDryCh4" = data$crdCo2[[lvl]]$rtioMoleDryCh4, + "temp" = data$crdCo2[[lvl]]$temp, + "pres" = data$crdCo2[[lvl]]$pres, + "presEnvHut" = data$envHut[[lvlEnvHut]]$pres, + "rhEnvHut" = data$envHut[[lvlEnvHut]]$rh, + "tempEnvHut" = data$envHut[[lvlEnvHut]]$temp, + "rtioMoleWetH2oEnvHut" = data$envHut[[lvlEnvHut]]$rtioMoleWetH2o, + "lvlCrdCo2" = data$crdCo2ValvLvl[[lvlValv]]$lvlCrdCo2 + ) + + #input the whole day qfqm + wrk$qfqm <- list() + #subset only + wrk$qfqm$crdCo2 <- qfInp$crdCo2[[lvl]] + + if (PrdMeas == PrdAgr) { + #PrdAgr <- 9 + #9 minutely sampling data + #idxLvLPrdAgr <- paste0(lvl, "_", sprintf("%02d", PrdAgr), "m") + #rpt[[dp01]][[idxLvLPrdAgr]] <- list() + + #if there is at least one measurement + if(length(which(!is.na(wrk$qfqm$crdCo2$qfRngTemp))) > 0){ + #determine the index of each measurement + wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 60) + #delete row if last timeBgn and timeEnd is NA + wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] + #if last timeEnd is NA, replce that time to the last time value in data$time + wrk$idx$timeEnd <- as.POSIXct(ifelse(is.na(wrk$idx$timeEnd), data$time[length(data$time)], wrk$idx$timeEnd), origin = "1970-01-01", tz = "UTC") + #idxAgr2 <- 0 + for (idxAgr in 1:length(wrk$idx$idxBgn)){ + #idxAgr <- 25 + #get data for each idxAgr + wrk$inpMask$data <- list() + wrk$inpMask$data <- wrk$data[wrk$idx$idxBgn[idxAgr]:wrk$idx$idxEnd[idxAgr],] + + #replace data with NaN when valve switch to measure to next level before schedule time (9 min) + wrk$inpMask$data$presEnvHut <- ifelse(wrk$inpMask$data$lvlCrdCo2 == lvlCrdCo2, wrk$inpMask$data$presEnvHut, NaN) + wrk$inpMask$data$rhEnvHut <- ifelse(wrk$inpMask$data$lvlCrdCo2 == lvlCrdCo2, wrk$inpMask$data$rhEnvHut, NaN) + wrk$inpMask$data$rtioMoleWetH2oEnvHut <- ifelse(wrk$inpMask$data$lvlCrdCo2 == lvlCrdCo2, wrk$inpMask$data$rtioMoleWetH2oEnvHut, NaN) + wrk$inpMask$data$tempEnvHut <- ifelse(wrk$inpMask$data$lvlCrdCo2 == lvlCrdCo2, wrk$inpMask$data$tempEnvHut, NaN) + + #get rid of lvlCrdCo2 + wrk$inpMask$data <- wrk$inpMask$data[,-which(names(wrk$inpMask$data) == "lvlCrdCo2")] + + #dp01 processing + rpt[[idxAgr]] <- eddy4R.base::wrap.dp01( + # assign data: data.frame or list of type numeric or integer + data = wrk$inpMask$data#, + # if data is a list, which list entries should be processed into Level 1 data products? + # defaults to NULL which expects data to be a data.frame + #idx = + #names(wrk$inpMask$data[[dp01]]) #"000_010_02m" + ) + + #units: + for (idxVar in names(rpt[[1]]$mean)){ + #idxVar <- names(rpt[[1]]$mean)[1] + attributes(rpt[[1]]$mean[[idxVar]])$unit <- attributes(wrk$data[[idxVar]])$unit + } + + #grab and add both time begin and time end to rpt + rpt[[idxAgr]]$timeBgn <- list() + rpt[[idxAgr]]$timeEnd <- list() + + #output time for dp01 + for(idxVar in names(wrk$data)[which(!(names(wrk$data) %in% c("lvlCrdCo2")))]){ + rpt[[idxAgr]]$timeBgn[[idxVar]] <- wrk$idx$timeBgn[idxAgr] + rpt[[idxAgr]]$timeEnd[[idxVar]] <- wrk$idx$timeEnd[idxAgr] + }; rm(idxVar) + + #}# end of there is at least one data + + }; rm(idxAgr) + } else { + + rpt[[1]] <- list() + + for(idxStat in NameStat){ + #idxStat <- NameStat[1] + rpt[[1]][[idxStat]] <- as.data.frame(matrix(NaN, nrow = 1, ncol = ncol(wrk$data))) + #assign name to each column + names(rpt[[1]][[idxStat]]) <- names(wrk$data) + #not report lvlCrdCo2 + rpt[[1]][[idxStat]] <- rpt[[1]][[idxStat]][which(!(names(rpt[[1]][[idxStat]]) %in% c("lvlCrdCo2")))] + + }; rm(idxStat) + #add both time begin and time end to rpt + rpt[[1]]$timeBgn <- list() + rpt[[1]]$timeEnd <- list() + + #output time for dp01 + for(idxVar in names(wrk$data)[which(!(names(wrk$data) %in% c("lvlCrdCo2")))]){ + rpt[[1]]$timeBgn[[idxVar]] <- data$time[1] + rpt[[1]]$timeEnd[[idxVar]] <- data$time[length(data$time)] + #unit + attributes(rpt[[1]]$mean[[idxVar]])$unit <- attributes(wrk$data[[idxVar]])$unit + + }; rm(idxVar) + + }#end of if no measurement data at all in the whole day + } #end of PrdAgr + + if (PrdMeas != PrdAgr) { + #PrdAgr <- 30 + #if there is at least one measurement + if(length(which(!is.na(wrk$qfqm$crdCo2$qfRngTemp))) > 0){ + + #determine the index of each measurement + wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 60) + #delete row if last timeBgn and timeEnd is NA + wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] + #if last timeEnd is NA, replce that time to the last time value in data$time + wrk$idx$timeEnd <- as.POSIXct(ifelse(is.na(wrk$idx$timeEnd), data$time[length(data$time)], wrk$idx$timeEnd), origin = "1970-01-01", tz = "UTC") + + whrSamp <- wrk$idx$idxBgn[1]:wrk$idx$idxEnd[1] + if (length (wrk$idx$idxBgn) > 1 ){ + for(ii in 2:length (wrk$idx$idxBgn)){ + whrSamp <- c(whrSamp, wrk$idx$idxBgn[ii]:wrk$idx$idxEnd[ii]) + } + } + + tmpAttr <- list() + for (idxData in c("presEnvHut", "rhEnvHut", "rtioMoleWetH2oEnvHut", "tempEnvHut")){ + #defined attributes + tmpAttr[[idxData]] <- attributes(wrk$data[[idxData]]) + #replace idxData data with NaN when irga got kick out to measure the new measurement level + wrk$data[[idxData]] <- ifelse(wrk$data$lvlCrdCo2 == lvlCrdCo2, wrk$data[[idxData]], NaN) + } + wrk$data[-whrSamp, 1:8] <- NaN #the range of columns represents the varibles included except the valve data + #added attributes + for (idxData in c("presEnvHut", "rhEnvHut", "rtioMoleWetH2oEnvHut", "tempEnvHut")){ + attributes(wrk$data[[idxData]]) <- tmpAttr[[idxData]] + } + } else {#if there are no data at all in wrk$data$temp + wrk$data[,1:8] <- NaN #the range of columns represents the varibles included except the valve data + } + + + for(idxAgr in c(1:length(idxTime[[paste0(PrdAgr, "min")]]$Bgn))) { + #idxAgr <- 48 + + + ## grab data at the selected mask data + # for data + wrk$inpMask$data <- list() + idxLvLPrdAgr <- paste0(lvl, "_", sprintf("%02d", PrdAgr), "m") + + wrk$inpMask$data <- wrk$data[idxTime[[paste0(PrdAgr, "min")]]$Bgn[idxAgr]:idxTime[[paste0(PrdAgr, "min")]]$End[idxAgr],] + #get rid of lvlCrdCo2 + wrk$inpMask$data <- wrk$inpMask$data[,-which(names(wrk$inpMask$data) == "lvlCrdCo2")] + + #call wrap.dp01.R to calculate descriptive statistics + rpt[[idxAgr]] <- eddy4R.base::wrap.dp01( + # assign data: data.frame or list of type numeric or integer + data = wrk$inpMask$data#, + # if data is a list, which list entries should be processed into Level 1 data products? + # defaults to NULL which expects data to be a data.frame + #idx = + #names(wrk$inpMask$data[[dp01]]) #"000_010_02m" + ) + + #units: + for (idxVar in names(rpt[[1]]$mean)){ + #idxVar <- names(rpt[[1]]$mean)[1] + attributes(rpt[[1]]$mean[[idxVar]])$unit <- attributes(wrk$data[[idxVar]])$unit + } + + #grab and add both time begin and time end to rpt + rpt[[idxAgr]]$timeBgn <- list() + rpt[[idxAgr]]$timeEnd <- list() + + for(idxVar in names(wrk$data)[which(!(names(wrk$data) %in% c("lvlCrdCo2")))]){ + rpt[[idxAgr]]$timeBgn[[idxVar]] <- data$time[idxTime[[paste0(PrdAgr, "min")]]$Bgn[idxAgr]] + rpt[[idxAgr]]$timeEnd[[idxVar]] <- data$time[idxTime[[paste0(PrdAgr, "min")]]$End[idxAgr]] + } + + }; #rm(idxAgr) + + }#end of PrdAgr == 30 + + }#end of TypeMeas %in% "samp" if statement + + #during validation period + if (TypeMeas %in% "vali"){ + #input the whole day data + wrk$data <- data.frame(stringsAsFactors = FALSE, + "rtioMoleWetCh4" = data$crdCo2[[lvl]]$rtioMoleWetCh4, + "rtioMoleDryCh4" = data$crdCo2[[lvl]]$rtioMoleDryCh4, + "rtioMoleDryCh4Refe" = data$crdCo2[[lvl]]$rtioMoleDryCh4Refe, + "temp" = data$crdCo2[[lvl]]$temp, + "pres" = data$crdCo2[[lvl]]$pres, + "presEnvHut" = data$envHut[[lvlEnvHut]]$pres, + "rhEnvHut" = data$envHut[[lvlEnvHut]]$rh, + "tempEnvHut" = data$envHut[[lvlEnvHut]]$temp, + "rtioMoleWetH2oEnvHut" = data$envHut[[lvlEnvHut]]$rtioMoleWetH2o + ) + + #input the whole day qfqm + wrk$qfqm <- list() + wrk$qfqm$crdCo2 <- qfInp$crdCo2[[lvl]] + + if (PrdMeas == PrdAgr) { + #PrdAgr <- 9 + #9 minutely sampling data + #idxLvLPrdAgr <- paste0(lvl, "_", sprintf("%02d", PrdAgr), "m") + #rpt[[dp01]][[idxLvLPrdAgr]] <- list() + + #if there is at least one measurement + if(length(which(!is.na(wrk$qfqm$crdCo2$qfRngTemp))) > 0){ + #determine the end time of each measurement + wrk$idx <- eddy4R.base::def.idx.agr(time = data$time, PrdAgr = (PrdMeas*60), FreqLoca = 1, MethIdx = "specBgn", data = wrk$qfqm$crdCo2$qfRngTemp, CritTime = 60) + #delete row if last timeBgn and timeEnd is NA + wrk$idx <- wrk$idx[rowSums(is.na(wrk$idx)) != 2,] + #if last timeEnd is NA, replce that time to the last time value in data$time + wrk$idx$timeEnd <- as.POSIXct(ifelse(is.na(wrk$idx$timeEnd), data$time[length(data$time)], wrk$idx$timeEnd), origin = "1970-01-01", tz = "UTC") + + #idxAgr2 <- 0 + for (idxAgr in 1:length(wrk$idx$idxBgn)){ + #idxAgr <- 1 + #determine input data for each idxAgr + wrk$inpMask$data <- list() + wrk$inpMask$data <- wrk$data[wrk$idx$idxBgn[idxAgr]:wrk$idx$idxEnd[idxAgr],] + + #calculate dp01 + rpt[[idxAgr]] <- eddy4R.base::wrap.dp01( + # assign data: data.frame or list of type numeric or integer + data = wrk$inpMask$data#, + # if data is a list, which list entries should be processed into Level 1 data products? + # defaults to NULL which expects data to be a data.frame + #idx = + #names(wrk$inpMask$data[[dp01]]) #"000_010_02m" + ) + + #units: + for (idxVar in names(rpt[[1]]$mean)){ + #idxVar <- names(rpt[[1]]$mean)[1] + attributes(rpt[[1]]$mean[[idxVar]])$unit <- attributes(wrk$data[[idxVar]])$unit + } + + #grab and add both time begin and time end to rpt + rpt[[idxAgr]]$timeBgn <- list() + rpt[[idxAgr]]$timeEnd <- list() + + #output time for dp01 + for(idxVar in names(wrk$data)){ + # rpt[[idxAgr2]]$timeBgn[[idxVar]] <- data$time[(whrEnd[idxAgr] - 20 - 2*60+1)] + # rpt[[idxAgr2]]$timeEnd[[idxVar]] <- data$time[(whrEnd[idxAgr] - 20)] + rpt[[idxAgr]]$timeBgn[[idxVar]] <- wrk$idx$timeBgn[idxAgr] + rpt[[idxAgr]]$timeEnd[[idxVar]] <- wrk$idx$timeEnd[idxAgr] + }; rm(idxVar) + + }#; rm(idxAgr) + + } else { + + rpt[[1]] <- list() + + for(idxStat in NameStat){ + #idxStat <- NameStat[1] + rpt[[1]][[idxStat]] <- as.data.frame(matrix(NaN, nrow = 1, ncol = ncol(wrk$data))) + #assign name to each column + names(rpt[[1]][[idxStat]]) <- names(wrk$data) + + }; rm(idxStat) + #add both time begin and time end to rpt + rpt[[1]]$timeBgn <- list() + rpt[[1]]$timeEnd <- list() + + #output time for dp01 + for(idxVar in names(wrk$data)){ + rpt[[1]]$timeBgn[[idxVar]] <- data$time[1] + rpt[[1]]$timeEnd[[idxVar]] <- data$time[length(data$time)] + #unit + attributes(rpt[[1]]$mean[[idxVar]])$unit <- attributes(wrk$data[[idxVar]])$unit + + }; rm(idxVar) + + }#end of if no measurement data at all in the whole day + + } #end of PrdAgr == 9 + + + if (PrdMeas != PrdAgr) { #PrdAgr <- 30 #if there is at least one measurement @@ -917,6 +1347,7 @@ wrap.dp01.ecse <- function( }#end of TypeMeas %in% "vali" if statement }##end of dp01 if statement + #calculate dp01 for "isoH2o" ######################################################################################## if (dp01 %in% c("isoH2o")){ #during sampling period @@ -1313,6 +1744,8 @@ wrap.dp01.ecse <- function( }#end of dp01 if statement + #remove NULL list from rpt + rpt <- rpt[!sapply(rpt, is.null)] #return results return(rpt) diff --git a/pack/eddy4R.stor/R/wrap.prd.day.ecse.R b/pack/eddy4R.stor/R/wrap.prd.day.ecse.R index 8bde978b..3fb56e9f 100644 --- a/pack/eddy4R.stor/R/wrap.prd.day.ecse.R +++ b/pack/eddy4R.stor/R/wrap.prd.day.ecse.R @@ -36,6 +36,8 @@ # update input parameters in wrap.qf.rmv.data function # Natchaya P-Durden (2019-05-06) # adding logic to determine qfFrt00 from mfm +# Chris Florian (2021-06-02) +# updating qfRmv to not exclude rtioMoleWetH2o data that are below the low humidity threshold ############################################################################################## wrap.prd.day.ecse <- function( @@ -57,7 +59,7 @@ wrap.prd.day.ecse <- function( } #Removing high frequency flagged data #Applying the bad quality flags to the reported output data - rpt <- eddy4R.qaqc::wrap.qf.rmv.data(inpList = inpList, Vrbs = FALSE, MethMeas = "ecse", Sens = NULL, qfRmv = c("qfCal", "qfRh", "qfTemp")) + rpt <- eddy4R.qaqc::wrap.qf.rmv.data(inpList = inpList, Vrbs = FALSE, MethMeas = "ecse", Sens = NULL, qfRmv = c("qfCal", "qfRh", "qfTemp", "qfLowRtioMoleWetH2o")) # perform de-spiking diff --git a/pack/eddy4R.stor/man/def.time.rate.diff.Rd b/pack/eddy4R.stor/man/def.time.rate.diff.Rd index f643b8cd..62d74816 100644 --- a/pack/eddy4R.stor/man/def.time.rate.diff.Rd +++ b/pack/eddy4R.stor/man/def.time.rate.diff.Rd @@ -4,8 +4,14 @@ \alias{def.time.rate.diff} \title{Definition function: calculate time rate of change for ecse dp02 data and qfqm} \usage{ -def.time.rate.diff(dataInp, numDate, PrdWndwAgr, PrdIncrAgr, Date, - qfqmFlag = FALSE) +def.time.rate.diff( + dataInp, + numDate, + PrdWndwAgr, + PrdIncrAgr, + Date, + qfqmFlag = FALSE +) } \arguments{ \item{\code{dataInp}}{Input data.} diff --git a/pack/eddy4R.stor/man/wrap.dp01.ecse.Rd b/pack/eddy4R.stor/man/wrap.dp01.ecse.Rd index 52059d2d..1e8518e0 100644 --- a/pack/eddy4R.stor/man/wrap.dp01.ecse.Rd +++ b/pack/eddy4R.stor/man/wrap.dp01.ecse.Rd @@ -4,11 +4,22 @@ \alias{wrap.dp01.ecse} \title{Wrapper function: Preprocessing and computing NEON eddy-covariance stroage exchange L1 data product descriptive statistics} \usage{ -wrap.dp01.ecse(dp01 = c("co2Stor", "h2oStor", "tempAirLvl", "tempAirTop", - "isoCo2", "isoH2o")[1], lvl, lvlMfcSampStor = NULL, lvlEnvHut = NULL, - lvlValv = NULL, lvlCrdH2oValvVali = NULL, data = list(), - qfInp = list(), TypeMeas = c("samp", "vali")[1], PrdMeas, PrdAgr, - idxTime = list()) +wrap.dp01.ecse( + dp01 = c("co2Stor", "h2oStor", "tempAirLvl", "tempAirTop", "isoCo2", "isoH2o", + "ch4Conc")[1], + lvl, + lvlMfcSampStor = NULL, + lvlEnvHut = NULL, + lvlValv = NULL, + lvlCrdH2oValvVali = NULL, + lvlCrdCo2Valv = NULL, + data = list(), + qfInp = list(), + TypeMeas = c("samp", "vali")[1], + PrdMeas, + PrdAgr, + idxTime = list() +) } \arguments{ \item{dp01}{A vector of class "character" containing the name of NEON ECSE dp01 which descriptive statistics are being calculated, \cr @@ -24,6 +35,8 @@ c("co2Stor", "h2oStor", "tempAirLvl", "tempAirTop", "isoCo2", "isoH2o"). Default \item{lvlCrdH2oValvVali}{Measurement level of crdH2oValvVali which apply to only dp01 equal to "isoH2o". Defaults to NULL. Of type character. [-]} +\item{lvlCrdCo2Valv}{Horizontal and vertical location for crdCo2 valve. Defaults to NULL. Of type character. [-]} + \item{data}{A list of data frame containing the input dp0p data that related to dp01 which descriptive statistics are being calculated. Of class integer". [User defined]} \item{qfInp}{A list of data frame containing the input quality flag data that related to dp01 are being grouped. Of class integer". [-]} diff --git a/vign/turb.neon.vignette.Rmd b/vign/turb.neon.vignette.Rmd new file mode 100755 index 00000000..3681945b --- /dev/null +++ b/vign/turb.neon.vignette.Rmd @@ -0,0 +1,909 @@ +--- +title: "Example workflow for processing NEON eddy-covariance turbulence data with eddy4R-Docker 1.0.0" +author: "David Durden, Stefan Metzger, Natchaya Pingintha-Durden, Claire Lunch, Megan Jones" +date: "`r Sys.Date()`" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Example workflow for processing NEON eddy-covariance turbulence data with eddy4R-Docker 1.0.0} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r setup, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +## 0. Install and set up + +This vignette assumes you are working with eddy4R within the Docker +environment established by NEON. To get set up in that environment, +follow the instructions here. + +At the end of those instructions, you should have an RStudio +environment running within Docker. Use that workspace for this +vignette. + +## 1. Install packages and set up environment + +First, check for required packages and install any that aren't +already installed. Once all required packages are installed, +load them into the environment. + +Throughout this vignette, we use the `package::function()` +syntax to explicitly refer to functions. This is done to avoid any +possibility of ambiguity, in case there are functions in +different packages with the same name. + + +```{r pack-install} + +packReq <- c("DataCombine", "eddy4R.base", "eddy4R.qaqc", + "ff", "ffbase", "methods", "rhdf5","splus2R") +base::lapply(packReq, function(x) { + print(x) + if(base::require(x, character.only = TRUE) == FALSE) { + utils::install.packages(x) + base::library(x, character.only = TRUE) + }}) +base::rm(packReq) + +``` + + +## 2. Set environment variables + +Here we set file paths and basic input parameters as environment +variables for convenience. + +```{r env-vars} + +# First, we assign a working directory where the input data, output reference data, and workflow will be downloaded to for running the tutorial. This defaults to a temporary directory, but you can change this to any location you like (be aware the downloaded .zip archive is ~ 360 MB) +DirWrk <- tempdir() + +#This is the data download url, we use a permanent URL (PURL) link to ensure the data is always accessible. +UrlDld <- "https://w3id.org/smetzger/Metzger-et-al_2019_NEON-BAMS/tutorial_data/" + +# Download input data and output reference data +eddy4R.base::def.dld.zip(Inp = list(Url = UrlDld, Dir = DirWrk)) + +# File path for the directory containing input L0' data +base::Sys.setenv("DIRINP" = paste0(DirWrk,"/inpExmp")) + +# File path for outputs +base::Sys.setenv("DIROUT" = paste0(DirWrk,"/out")) + +# Date(s) of the input data, specified here for the output file name +base::Sys.setenv("DATEOUT" = "2017-09-01") + +# NEON domain, site, and data product ID of the input data +# for the output file name: +# Domain 10 = D10 +# Central Plains Experimental Range = CPER +# Bundled eddy covariance data = 00200.001 +base::Sys.setenv("FILEOUTBASE" = "NEON.D10.CPER.00200.001") + +# Flag to indicate to the eddy4R.base::def.para.flow() +# function that the variables above can be found as +# environment variables, instead of provided as function +# inputs +base::Sys.setenv("METHPARAFLOW" = "EnvVar") + +``` + +## 3. Read in metadata + +Pass along the environment variables defined in section 2, +then read in additional parameters from the HDF5 file. + +```{r metadata} + +# In this section, define and/or read in a number of terms and +# parameters. + +# Create an empty list for parameters. +Para <- base::list() + +# Use the def.para.flow() function to specify the metadata are stored +# in environment variables, and give file paths to input and output. +# You will see a number of warning messages in response to this line, +# but if you can proceed with the next several commands, don't worry +# about the warnings here. +Para$Flow <- eddy4R.base::def.para.flow(MethParaFlow = "EnvVar") + +# The args() function can be used to see the full list of function +# arguments available in def.para.flow(). +# Additional metadata can be passed to the function as well, by +# assigning a variable name in the functional call. +base::args(eddy4R.base::def.para.flow) + +# Get the NEON-specific 4-letter code for the site location (Loc) +# from the dp0p input file +Para$Flow$Loc <- eddy4R.base::def.para.site(FileInp = Para$Flow$DirFilePara)$Loc + +# Get the level of the tower top (LvlTowr) from the dp0p input file +Para$Flow$LvlTowr <- eddy4R.base::def.para.site(FileInp = Para$Flow$DirFilePara)$LvlTowr + +# Use the def.hdf5.read.para() function to extract parameters +# from the metadata in the input file and assign them to the +# parameter list. + +# For each variable to be written out, set the aggregation +# parameters. +# Variables: +# amrs: Attitude Motion Reference System +# co2turb: Turbulent CO2 flux +# h2oTurb: Turbulent H2O flux +# soni: Windspeed and direction from sonic anemometer +# Parameters: +# PrdIncrAgrDflt: Period Increment Aggregation Default +# PrdWndwAgrDflt: Period Window Aggregation Default +for(idx in c("amrs", "co2Turb", "h2oTurb", "soni")) { + + Para$Flow$dp01[[idx]] <- eddy4R.base::def.hdf5.read.para( + DirFileParaLoca = Para$Flow$DirFilePara, + GrpName = paste0("/", Para$Flow$Loc, "/dp01/data/", idx), + SetPara = c("PrdIncrAgrDflt", "PrdWndwAgrDflt") + ) + +} + +# AngEnuXaxs: Planar fit coefficient of the x axis +# AngEnuYaxs: Planar fit coefficient of the y axis +# Ofst: Planar fit offset +# ZoneTime: Time zone of data collection +# Note that a future release of the eddy4R package will +# include a planar fit function, which will allow you to +# set the planar fit coefficients to NA here, and then +# use the function to estimate them. This workflow will +# require a longer window of input data, on the order of +# days to weeks, be provided to determine a robust +# planar regression for the data. +Para$Sci <- eddy4R.base::def.hdf5.read.para( + DirFileParaLoca = Para$Flow$DirFilePara, + GrpName = Para$Flow$Loc, + SetPara = c("Pf$AngEnuXaxs", "Pf$AngEnuYaxs", "Pf$Ofst", "ZoneTime") +) + +# In the next 5 commands, get the sampling frequency (FreqSamp) +# of each instrument from the parameters in the input file: +# irgaTurb: Turbulent flux gas analyzer +# mfcSampTurb: Mass flow controller +# valvValiNemaTurb: Valve Validation Nema enclosure Turbulence +# soni, amrs: See above +Para$Sci$dp0p$irgaTurb <- def.hdf5.read.para( + DirFileParaLoca = Para$Flow$DirFilePara, + GrpName = paste0("/", Para$Flow$Loc, "/dp0p/data/irgaTurb"), + SetPara = c("FreqSamp") +) + +Para$Sci$dp0p$mfcSampTurb <- def.hdf5.read.para( + DirFileParaLoca = Para$Flow$DirFilePara, + GrpName = paste0("/", Para$Flow$Loc, "/dp0p/data/mfcSampTurb"), + SetPara = c("FreqSamp") +) + +Para$Sci$dp0p$valvValiNemaTurb <- def.hdf5.read.para( + DirFileParaLoca = Para$Flow$DirFilePara, + GrpName = paste0("/", Para$Flow$Loc, "/dp0p/data/valvValiNemaTurb"), + SetPara = c("FreqSamp") +) + +# For the sonic anemometer, also get the angle of the z axis +# relative to the North-East-Down coordinate system used +# for sonic anemometer installation (AngNedZaxs) +Para$Sci$dp0p$soni <- def.hdf5.read.para( + DirFileParaLoca = Para$Flow$DirFilePara, + GrpName = paste0("/", Para$Flow$Loc, "/dp0p/data/soni"), + SetPara = c("AngNedZaxs","FreqSamp") +) + +Para$Sci$dp0p$amrs <- def.hdf5.read.para( + DirFileParaLoca = Para$Flow$DirFilePara, + GrpName = paste0("/", Para$Flow$Loc, "/dp0p/data/amrs"), + SetPara = c("FreqSamp") +) + +# Lag$TimeDiff: The time lag to align gas concentration data +# with wind measurements, allowing for travel time from the inlet +# to the gas analyzer. +# The next two commands create the TimeDiff parameter for CO2 +# and H2O, but leave them set to NA. Lag time is calculated by +# eddy4R.base::def.lag(). If you prefer to set it manually, +# the def.lag() function can also set it. +Para$Sci$dp01$co2Turb$RtioMoleDryCo2 <- def.hdf5.read.para( + DirFileParaLoca = Para$Flow$DirFilePara, + GrpName = paste0("/", Para$Flow$Loc, "/dp01/data/co2Turb/", + Para$Flow$LvlTowr, "_30m/rtioMoleDryCo2"), + SetPara = c("Lag$TimeDiff") +) + +Para$Sci$dp01$h2oTurb$RtioMoleDryH2o <- def.hdf5.read.para( + DirFileParaLoca = Para$Flow$DirFilePara, + GrpName = paste0("/", Para$Flow$Loc, "/dp01/data/h2oTurb/", + Para$Flow$LvlTowr, "_30m/rtioMoleDryH2o"), + SetPara = c("Lag$TimeDiff") +) + +# Create a version label for your outputs, using the +# current system time +Para$Flow$VersDp <- paste0(Para$Flow$VersDp, "_", + format(Sys.time(), "%Y%m%d_%H%M%S_%Z")) + +# Create a list of the sampling frequencies of the essential +# instruments, extracted from the input data above. +FreqSamp <- list( + "irgaTurb" = Para$Sci$dp0p$irgaTurb$FreqSamp, + "mfcSampTurb" = Para$Sci$dp0p$mfcSampTurb$FreqSamp, + "valvValiNemaTurb" = ifelse(length(as.numeric(Para$Sci$dp0p$valvValiNemaTurb$FreqSamp))==0, + 0.2, as.numeric(Para$Sci$dp0p$valvValiNemaTurb$FreqSamp)), + "soni" = Para$Sci$dp0p$soni$FreqSamp, + "amrs" = Para$Sci$dp0p$amrs$FreqSamp +) + +``` + +## 4. Variable ranges + +Set plausible ranges for the input variables, to be tested on ingest. +Then set paths for the working directory. + +```{r range} + +# Create an empty list for range values. +Rng <- list() + +# Set ranges for gas analyzer variables: +# densMoleCo2: CO2 molar density, mol m-3 +# densMoleH2o: H2O molar density, mol m-3 +# presAtm: Atmospheric pressure, Pa +# presDiff: Difference in pressure, Pa +# rtioMoleDryCo2: Mole ratio of CO2 in air, mol mol-1 +# rtioMoleDryH2o: Mole ratio of H2O in air, mol mol-1 +# tempIn: Temperature of air at inlet, K +# tempOut: Temperature of air at outlet, K +Rng$irgaTurb <- data.frame( + "densMoleCo2" = c(0,30) * 1e-3, + "densMoleH2o" = c(0,1500) * 1e-3, + "presAtm" = c(50,120) * 1e3, + "presDiff" = c(-10,1) * 1e3, + "rtioMoleDryCo2" = c(300,450) * 1e-6, + "rtioMoleDryH2o" = c(0,30) * 1e-3, + "tempIn" = c(220,330), + "tempOut" = c(220,330) +) + +# Set ranges for sonic anemometer variables: +# veloXaxs: Velocity on the x axis, m s-1 +# veloYaxs: Velocity on the y axis, m s-1 +# veloZaxs: Velocity on the z axis, m s-1 +# veloSoni: Speed of sound, m s-1 +Rng$soni <- data.frame( + "veloXaxs"=c(-50,50), + "veloYaxs"=c(-50,50), + "veloZaxs"=c(-10,10), + "veloSoni"=c(300,400) +) + +# Set ranges for attitude and motion reference variables: +# angXaxs: Deviation from the x axis, degree +# angYaxs: Deviation from the y axis, degree +# angZaxs: Deviation from the z axis, degree +Rng$amrs <- data.frame( + "angXaxs"=c(-360,360), + "angYaxs"=c(-360,360), + "angZaxs"=c(-360,360) +) + +# Set directories and file paths +# As written, this vignette uses the default paths. To use custom +# paths, assign the file paths you want to use to the parameters +# Para$Flow$DirWrk, Para$Flow$DirInp, and Para$Flow$DirOut before +# proceeding to the code below. + +# Working directory +# Default: use a temporary working directory on Docker filesystem +if(is.na(Para$Flow$DirWrk)) { + Para$Flow$DirWrk <- tempdir() + + # Optional: create a user-specified working directory, e.g. on host filesystem + } else { + dir.create(Para$Flow$DirWrk, recursive = TRUE, showWarnings = FALSE) + } + +# Directory for input data +# Default: use temporary working directory on Docker filesystem +if(is.na(Para$Flow$DirInp)) { + Para$Flow$DirInp <- paste0(Para$Flow$DirWrk, "/inpRefe") + + # Optional: create user-specified input directory + } else { + dir.create(Para$Flow$DirInp, recursive = TRUE, showWarnings = FALSE) + } + +# Directory for output data +# Default: use temporary working directory on Docker filesystem +if(is.na(Para$Flow$DirOut)) { + Para$Flow$DirOut <- paste0(Para$Flow$DirWrk, "/out") + + # Optional: create user-specified input directory + } else { + dir.create(Para$Flow$DirOut, recursive = TRUE, showWarnings = FALSE) + } + + +``` + +## 5. Read in data! + +We have found in testing that this section may run into trouble if +Docker is not assigned sufficient memory in your system. If you +find that this section hangs or errors out, try opening +Docker -> Preferences -> Advanced and increasing the memory +allotment. + +```{r ingest} + +# Create empty lists for data and quality flags +data <- list() +qfqmFlag <- list() + +# Index for the number of days iterated over, and empty list for outputs +numDate <- 0 +out <- list() + +# Loop over days in input data (in the example data here, there is +# only one day) +for(date in Para$Flow$DateOut) { + numDate <- numDate + 1 + + # Begin: read raw data from HDF5 file + if(Para$Flow$Read == "hdf5") { + + # Create directory structure + dir.create(paste(Para$Flow$DirOut, "/", Para$Flow$Loc, "/", + Para$Flow$VersDp, sep=""), + recursive = TRUE, showWarnings = FALSE) + + # Create empty list for inputs + inp <- list() + + # Loop over instruments + for(Var in c("irgaTurb", "mfcSampTurb", "valvValiNemaTurb", "soni", "amrs")){ + + # Use data read function, assigning results to a temporary variable + tmp <- eddy4R.base::wrap.hdf5.read( + DirInpLoca = Para$Flow$DirInp, + SiteLoca = Para$Flow$Loc, + DateLoca = date, + VarLoca = Var, + FreqLoca = FreqSamp[[Var]], + LvlTowr = Para$Flow$LvlTowr, + RngLoca = Rng, + + # Assign de-spiking parameters: + # widt: WndwDspkBr86 de-spiking median filter window width [s] + # nbin: NumDspkBr86Bin de-spiking histogram bins initial number/step size + # rest: ThshDspkBr86Reso de-spiking resolution threshold + DespLoca = list(widt = 9, + nbin = 2, + rest = 10 + ) + ) + + # Read in quality flags from HDF5 file + if(!Var %in% "valvValiNemaTurb"){ + + tmpQfqm <- eddy4R.base::def.hdf5.read.qfqm( + DirInpLoca = Para$Flow$DirInp, + SiteLoca = Para$Flow$Loc, + DateLoca = date, + VarLoca = Var, + FreqLoca = FreqSamp[[Var]], + LvlTowr = Para$Flow$LvlTowr + ) + + # Remove timestamp from flags, since timestamps will be + # saved elsewhere + tmpQfqm <- tmpQfqm[,grep("time",names(tmpQfqm),invert = T)] + } + + # Copy results from temporary variables to inp + + # Time domain, including unit assignment + if(Var == "irgaTurb") { + inp$time <- ff::as.ffdf.data.frame(data.frame(UTC = tmp$time)) + base::attr(x = inp$time$UTC, which = "unit") <- "YYYY-MM-DD hh:mm:ss.sss" + } + + # Sensor data, including unit assignment + inp$data[[Var]] <- ff::as.ffdf.data.frame(tmp$data) + for(idx in base::names(tmp$data)) + base::attr(x = inp$data[[Var]][[idx]], which = "unit") <- + base::attr(x = tmp$data, which = "unit")[[idx]] + + if(exists("tmpQfqm")) inp$qfqm[[Var]] <- ff::as.ffdf.data.frame(tmpQfqm) + + # Remove temporary variables + rm(tmp) + if(exists("tmpQfqm")) + rm(tmpQfqm) + invisible(gc()) + + # End loop over instruments + }; rm(Var) + + # Calculate derived quantities: daily extent, native resolution + inp <- eddy4R.base::wrap.derv.prd.day( + inpList = inp, + ZoneTime = Para$Sci$ZoneTime, + AngZaxsSoniInst = Para$Sci$dp0p$soni$AngNedZaxs + ) + + # Print status message + print(paste0(format(Sys.time(), "%F %T"), ": dataset ", date, + ": derived quantities calculated (daily extent, native resolution)")) + + # End of raw data read + } + + # Assign daily data and attributes to file-backed objects + # to keep RAM footprint small + # If loop is on the first day, create the attributes: + if(numDate == 1) { + + # IRGA + data$irgaTurb <- inp$data$irgaTurb + qfqmFlag$irgaTurb <- inp$qfqm$irgaTurb + + # MFC + data$mfcSampTurb <- inp$data$mfcSampTurb + qfqmFlag$mfcSampTurb <- inp$qfqm$mfcSampTurb + + # Sonic anemometer + data$soni <- inp$data$soni + qfqmFlag$soni <- inp$qfqm$soni + + # AMRS + data$amrs <- inp$data$amrs + qfqmFlag$amrs <- inp$qfqm$amrs + + # Time objects + data$time <- inp$time + qfqmFlag$time <- inp$time + + # If loop is on any day after the first, append the attributes: + } else { + + # IRGA + data$irgaTurb <- ffbase::ffdfappend(x = data$irgaTurb, + dat = inp$data$irgaTurb) + qfqmFlag$irgaTurb <- ffbase::ffdfappend(x =qfqmFlag$irgaTurb, + dat = inp$qfqm$irgaTurb) + + # MFC + data$mfcSampTurb <- ffbase::ffdfappend(x = data$mfcSampTurb, + dat = inp$data$mfcSampTurb) + qfqmFlag$mfcSampTurb <- ffbase::ffdfappend(x = qfqmFlag$mfcSampTurb, + dat = inp$qfqm$mfcSampTurb) + + # valvValiNemaTurb + data$valvValiNemaTurb <- ffbase::ffdfappend(x = data$valvValiNemaTurb, + dat = inp$data$valvValiNemaTurb) + qfqmFlag$valvValiNemaTurb <- ffbase::ffdfappend(x = qfqmFlag$valvValiNemaTurb, + dat = inp$qfqm$valvValiNemaTurb) + + # Sonic anemometer + data$soni <- ffbase::ffdfappend(x = data$soni, dat = inp$data$soni) + qfqmFlag$soni <- ffbase::ffdfappend(x =qfqmFlag$soni, dat = inp$qfqm$soni) + + # AMRS + data$amrs <- ffbase::ffdfappend(x = data$amrs, dat = inp$data$amrs) + qfqmFlag$amrs <- ffbase::ffdfappend(x =qfqmFlag$amrs, dat = inp$qfqm$amrs) + + # Time objects + data$time <- ffbase::ffdfappend(x = data$time, dat = inp$time) + qfqmFlag$time <- ffbase::ffdfappend(x =qfqmFlag$time, dat = inp$time) + + # End attributes assignment + } + + # Remove temporary list of input variables + inp <- NULL + inpQfqm <- NULL + invisible(gc()) + +# End loop +} + +``` + + +## 6. Begin data analysis. + +The first step is to get the half-hourly indices used to extract +data from the file-backed objects into internal memory as a list +of data.frames. Then, the lag time correction is applied to the gas +analyzer (IRGA) data, based on the sonic anemometer vertical velocity, +using the eddy4R.base::def.lag() function. After the lag correction +has been applied, the half-hourly derived data sub-products for relative +humidity and dewpoint temperature are calculated. + +Then, the input data that are provided as NEON Level 0 prime data products (high frequency raw data that has been time regularized and units converted) are +subsetted, and naming syntax is changed to match +NEON Level 1 data product output conventions. +eddy4R.base::wrap.dp01.agr.prd() is used to calculate the 1-minute averaged +data and quality metrics. Finally, we calculate 30-minute data using +eddy4R.base::wrap.dp01() and determine the quality metrics and final quality +flag using eddy4R.base::wrap.dp01.qfqm.ecte(). + +```{r process} + +# Kick off with a time stamp when processing starts +print(paste0(format(Sys.time(), "%F %T"), ": dataset ", date, + " derived quantities (vignette) begin")) + +# Create empty list for working parameters and variables +wrk <- list() + +# Create empty list for quality flag/quality metric data +qfqm <- list() + +# Set begin and end time for each 30-minute averaging interval, +# for each instrument system +invisible(lapply(names(data), function(x) { + if(x == "amrs") { + wrk$idx[[x]] <<- eddy4R.base::def.idx.agr(time = data$amrs$time, + PrdAgr = 1800, + FreqLoca = 40) + } else { + wrk$idx[[x]] <<- eddy4R.base::def.idx.agr(time = data$time$UTC, + PrdAgr = 1800, + FreqLoca = 20) + }})) + +# If Deve flag, meaning Development, is set to TRUE, only run a +# small number of iterations for testing purposes. Otherwise +# iterate over all variables and time periods. This vignette +# sets Deve=TRUE. +if (Para$Flow$Deve == TRUE) { + iter <- 10 + } else { + iter <- max(sapply(names(wrk$idx), + function(x) length(wrk$idx[[x]]$idxBgn))) + } + +# Another time stamp: Loop start +print(paste0(format(Sys.time(), "%F %T"), ": dataset ", + date, " DP01 calculation begin")) +numAgr <- 0 + +for(idxAgr in 1:iter) { + + numAgr <- numAgr + 1 + + # Create an identifier for the loop iterations + lvlAgr <- paste0("numAgr", ifelse(numAgr < 10, + paste0("0", numAgr), + numAgr)) + + # Create empty list for data + wrk$data <- list() + + # Loop over sensor systems to grab the data indices for the current aggregation period (i.e. 30 minute period of data). + for(idxSens in names(data)) { + wrk$data[[idxSens]] <- data[[idxSens]][wrk$idx[[idxSens]]$idxBgn[idxAgr]: + wrk$idx[[idxSens]]$idxEnd[idxAgr],] + wrk$qfqm[[idxSens]] <- qfqmFlag[[idxSens]][wrk$idx[[idxSens]]$idxBgn[idxAgr]: + wrk$idx[[idxSens]]$idxEnd[idxAgr],] + + # Assign units for each variable, from metadata + for(idxVar in base::names(wrk$data[[idxSens]])) { + + base::attr(x = wrk$data[[idxSens]][[idxVar]], which = "unit") <- + base::attr(x = data[[idxSens]][[idxVar]], which = "unit") + + }; rm(idxVar) + + }; rm(idxSens) + + # Lag time correction + # Select variables to be lagged + var <- c("rtioMoleDryCo2", "rtioMoleDryH2o", "rtioMassH2o", "presH2o") + + # Loop over lag correction variables + tmpRun <- 0 + for(idxVar in var) { + + tmpRun <- tmpRun + 1 + + # Is lag pre-determined or calculated? + # If lag is calculated: + if((base::length(base::grep(pattern = "CO2", + x = idxVar, + ignore.case = TRUE)) > 0 && + is.na(Para$Sci$dp01$co2Turb$RtioMoleDryCo2$`Lag$TimeDiff`)) | + (base::length(base::grep(pattern = "H2O", + x = idxVar, ignore.case = TRUE)) > 0 && + is.na(Para$Sci$dp01$h2oTurb$RtioMoleDryH2o$`Lag$TimeDiff`))) { + + # Calculate lag + lag <- def.lag(refe = wrk$data$soni$veloZaxs, + meas = wrk$data$irgaTurb[[idxVar]], + lagMax = 2 * FreqSamp$irgaTurb, + lagCnst = TRUE, + # Only negative lags permitted + lagNgtvPstv = c("n", "p", "np")[1], + lagAll = TRUE, + freq = FreqSamp$irgaTurb, + hpf = TRUE) + + } else { + + # If lag is pre-determined: + lag <- list() + lag$lag <- ifelse(base::length(base::grep(pattern = "CO2", + x = idxVar, + ignore.case = TRUE)) > 0, + Para$Sci$dp01$co2Turb$RtioMoleDryCo2$`Lag$TimeDiff`, + Para$Sci$dp01$h2oTurb$RtioMoleDryH2o$`Lag$TimeDiff`) + lag$corrCros <- NaN + + } + + # Shift data according to lag time + tmpAttr <- attributes(wrk$data$irgaTurb[[idxVar]])$unit + if(!is.na(lag$lag)) { + wrk$data$irgaTurb[[idxVar]] <- + DataCombine:::shift(VarVect = wrk$data$irgaTurb[[idxVar]], + shiftBy = - lag$lag, reminder = FALSE) + } + attributes(wrk$data$irgaTurb[[idxVar]])$unit <- tmpAttr; rm(tmpAttr) + + # Store lag times + if(tmpRun == 1) { + tmpLag <- lag$lag / FreqSamp$irgaTurb + tmpCorrCros <- lag$corrCros + } else { + tmpLag <- c(tmpLag, lag$lag / FreqSamp$irgaTurb) + tmpCorrCros <- c(tmpCorrCros, lag$corrCros) + } + + # End of lag correction loop + } + + # Remove temporary variables + names(tmpLag) <- var + tmpLag <- data.frame(t(tmpLag)) + names(tmpCorrCros) <- var + tmpCorrCros <- data.frame(t(tmpCorrCros)) + rm(tmpRun, lag, idxVar, var) + + # Derived variables + # Fast air temperature + wrk$data$soni$T_air_SONIC <- unlist(wrk$data$soni$tempSoni / + (1 + 0.51 * wrk$data$irgaTurb$rtioMassH2o)) + base::attr(x = wrk$data$soni$T_air_SONIC, which = "unit") <- "K" + + # Ambient water vapor saturation pressure + if(!is.na(mean(wrk$data$soni$T_air_SONIC, na.rm=TRUE))) { + wrk$data$irgaTurb$presH2oSatAtm <- + unlist(def.pres.h2o.sat.temp.mag(temp=wrk$data$soni$T_air_SONIC)) + } else { + wrk$data$irgaTurb$presH2oSatAtm <- + rep(NaN, length(wrk$data$soni$T_air_SONIC)) + } + base::attr(x = wrk$data$irgaTurb$presH2oSatAtm, which = "unit") <- "Pa" + + # Ambient relative humidity, including adjustment of partial + # pressure from cell to ambient via Dalton's law + wrk$data$irgaTurb$rhAtm <- + def.rh.pres.h2o.pres.sat.h2o(presH2o = wrk$data$irgaTurb$presH2o, + presH2oSat = wrk$data$irgaTurb$presH2oSatAtm) * + mean(wrk$data$irgaTurb$presAtm, na.rm=TRUE) / + mean(wrk$data$irgaTurb$presSum, na.rm=TRUE) + + # Ambient dew point + wrk$data$irgaTurb$tempDew <- + def.temp.dew.pres.h2o.temp.mag(presH2o = wrk$data$irgaTurb$presH2o, + temp = wrk$data$soni$T_air_SONIC) + + # Assemble data for Level 1 data product generation + + # Create empty temporary lists + # For data + wrk$tmp$data <- list() + + # For quality flags and quality metrics + wrk$tmp$qfqm <- list() + + # Assemble data + # For sonic anemometer + wrk$tmp$data$soni <- data.frame(stringsAsFactors = FALSE, + "veloXaxsErth" = wrk$data$soni$veloXaxs, + "veloYaxsErth" = wrk$data$soni$veloYaxs, + "veloZaxsErth" = wrk$data$soni$veloZaxs, + "veloXaxsYaxsErth" = wrk$data$soni$veloXaxsYaxsErth, + "angZaxsErth" = wrk$data$soni$angZaxsErth, + "tempSoni" = wrk$data$soni$tempSoni, + "tempAir" = wrk$data$soni$T_air_SONIC) + + # For AMRS + wrk$tmp$data$amrs <- data.frame(stringsAsFactors = FALSE, + "angNedXaxs" = wrk$data$amrs$angXaxs, + "angNedYaxs" = wrk$data$amrs$angYaxs, + "angNedZaxs" = wrk$data$amrs$angZaxs) + + # For CO2 + wrk$tmp$data$co2Turb <- data.frame(stringsAsFactors = FALSE, + "rtioMoleDryCo2" = wrk$data$irgaTurb$rtioMoleDryCo2, + "densMoleCo2" = wrk$data$irgaTurb$densMoleCo2, + "presAtm" = wrk$data$irgaTurb$presAtm, + "presSum" = wrk$data$irgaTurb$presSum, + "frt00Samp" = wrk$data$mfcSampTurb$frt00, + "tempAve" = wrk$data$irgaTurb$tempMean) + + # for H2O + wrk$tmp$data$h2oTurb <- data.frame(stringsAsFactors = FALSE, + "rtioMoleDryH2o" = wrk$data$irgaTurb$rtioMoleDryH2o, + "densMoleH2o" = wrk$data$irgaTurb$densMoleH2o, + "tempDew" = wrk$data$irgaTurb$tempDew, + "presAtm" = wrk$data$irgaTurb$presAtm, + "presSum" = wrk$data$irgaTurb$presSum, + "frt00Samp" = wrk$data$mfcSampTurb$frt00, + "tempAve" = wrk$data$irgaTurb$tempMean) + + # Calculate data products + # Calculate 1 and 2 minute data products + wrk$dp01AgrSub[[lvlAgr]] <- eddy4R.base::wrap.dp01.agr.prd(inpList = wrk) + + # 30-minute data products + wrk$dp01[[lvlAgr]] <- eddy4R.base::wrap.dp01( + data = wrk$tmp$data, + idx = c("soni", "amrs", "co2Turb", "h2oTurb") + ) + + # Standard error removed from 1- and 30- minute, it is now part of the ucrt calculation function + lapply(names(wrk$dp01AgrSub[[lvlAgr]]$data), function(idxData){ + wrk$dp01AgrSub[[lvlAgr]]$data[[idxData]]$se <<- NULL + wrk$dp01[[lvlAgr]][[idxData]]$se <<- NULL + }) + + + # Another time stamp: Starting quality processing + print(paste0(format(Sys.time(), "%F %T"), + ": Beginning the qfqm data processing for data in loop...", + idxAgr)) + + # Calculate the quality metrics and determine the final quality flag + wrk$qfqmOut[[lvlAgr]] <- eddy4R.base::wrap.dp01.qfqm.ecte( + qfqm = wrk$qfqm, + idx = c("soni", "amrs", "co2Turb", "h2oTurb"), + MethMeas = "ecte", + RptExpd = TRUE ) + + # Remove temporary files + wrk$data <- NULL + wrk$tmp <- NULL + invisible(gc()) + +# End loop +} +print(paste0(format(Sys.time(), "%F %T"), + ": dataset ", date, + " DP01 calculation complete")) + + +``` + +## 7. Aggregate data and quality metrics + +Now, we aggregate the results from section 6. The first step is to +aggregate the half-hourly sets of data and qfqm. Then, we plot the +final quality flag and its relationship to the alpha and beta quality +metrics. (alpha = fraction of data flagged by sensor and statistical +plausibility tests; beta = fraction of quality flags not available +due to missing data) + +```{r aggregation} + +# Concatenate results using def.dp01.agr.ecte() +out <- eddy4R.base::def.dp01.agr.ecte(inpList = wrk, + MethSubAgr = TRUE, + MethUcrt = FALSE) + + +# Generate qfqm table to be plotted +inpPlot <- data.frame(time = + format(as.POSIXct(out$time$co2Turb$timeBgn, + format="%Y-%m-%d %H:%M:%S"), + format="%H:%M"), + qfFinl = out$qfqm$soni$qfFinl$veloXaxsYaxsErth, + qmAlph = out$qfqm$soni$qmAlph$veloXaxsYaxsErth, + qmBeta = out$qfqm$soni$qmBeta$veloXaxsYaxsErth) + +# Plot final quality flag, alpha metric, and beta metric +library(ggplot2) + +plot = ggplot(inpPlot, aes(time)) + + geom_bar(aes(y = qfFinl, fill= "qfFinl"), stat="identity") + + geom_line(aes(y = qmAlph, group = 1, color = "qmAlph"), + size = 1, linetype = "dashed") + + geom_line(aes(y = qmBeta, group = 2, color = "qmBeta"), + size = 1, linetype = "dashed") + + ggtitle("veloXaxsYaxsErth") + + scale_colour_manual(" ", + values=c("qfFinl" = "black", + "qmAlph" = "blue", + "qmBeta" = "red")) + + scale_fill_manual("",values="black") + + theme(legend.key = element_blank(), + legend.title = element_blank(), + legend.box = "vertical", + plot.title = element_text(size=14, + face="bold.italic", + hjust = 0.5), + axis.title.x = element_text(size=14, face="bold"), + axis.title.y = element_text(size=14, face="bold")) + + ylim(-0.2,1.2) + + labs(y = "qfqm") + +print(plot) + +# Remove temporary files +wrk$dp01 <- NULL +wrk$dp01AgrSub <- NULL +wrk$qfqmOut <- NULL + +``` + +## 8. Format for output + +Lastly, we prepare the data for output and write it to the dp01 +output HDF5 file. We first create the skeleton structure of the +NEON HDF5 file using eddy4R.base::def.hdf5.crte(). We then perform +some packaging of the results to match the NEON HDF5 structure and +write the output to the NEON dp01 HDF5 file. + +```{r format} + +# Drop the quality column (qfqm) after the example; +# these data will be added to the HDF5 structure in +# a future release +lapply(names(out$qfqm), function(x){ + out$qfqm[[x]]$qm <<- NULL +}) + +# Use the def.hdf5.crte() function to generate the +# H5 file in NEON EC format +eddy4R.base::def.hdf5.crte(Date = date, + Site = Para$Flow$Loc, + LvlTowr = Para$Flow$LvlTowr, + DirOut = base::paste0(Para$Flow$DirOut, + "/", Para$Flow$Loc, + "/", Para$Flow$VersDp), + FileOutBase = Para$Flow$FileOutBase, + MethExpd = TRUE) + +# Determine the output filename of the file we just created +FileOut <- base::list.files(path = base::paste0(Para$Flow$DirOut, + "/", Para$Flow$Loc, + "/", Para$Flow$VersDp), + pattern = ".h5", full.names = TRUE) + +# Call the wrapper function wrap.hdf5.wrte.dp01() to package +# and write data to output HDF5 files, just expanded 30 minute data +eddy4R.base::wrap.hdf5.wrte.dp01(inpList = out, + FileIn = base::paste0(Para$Flow$DirInp, + "/","ECTE_dp0p_", + Para$Flow$Loc, "_", + date, ".h5"), + FileOut = FileOut, + SiteLoca = Para$Flow$Loc, + LvlTowr = Para$Flow$LvlTowr, + MethUcrt = FALSE, + MethSubAgr = FALSE) + +```