V0.1 - Preliminary analysis at the submission date of DFG grant proposal

Author: derele

R/Hyena_first_rough.R

@@ -40,6 +40,8 @@ OTUs <- do.call(rbind, OTU.all)
 rownames(OTUs) <- OTUs$OTUID
 OTUs$OTUID <- NULL
 
+sum(OTUs)
+
 devSVG("figures/Hyena_OTU_heat.svg", width=14, height=14)
 pheatmap(log10(OTUs+1),
          show_rownames=FALSE,
@@ -67,7 +69,39 @@ pheatmap(log10(OTUs+1),
 dev.off()
 
 
+## a crude background reduction by setting  all counts below an outlier detection to zero
+## http://stats.stackexchange.com/questions/56402/detecting-outliers-in-count-data
+out.z <- function(x){
+    trans <- log10(as.numeric(x))
+    ## a trick to not assess the distribution of zeros, ones and twos
+    ## assumed here to be true negatives
+    NN <- which(trans>0)
+    rob.z <- (trans-median(trans[NN]))/mad(trans[NN])
+    z.outl <- which(!rob.z>quantile(rob.z[NN], 0.05, na.rm=TRUE))
+}
+
+for(i in 1:nrow(OTUs)){
+    OTUs[i, out.z(OTUs[i,])] <- 0
+}
+
+sum(OTUs)
+
+## 90,427 removed 06/09/2016
+
+png("figures/Hyena_OTU_heat_BCcor.png", res=300, width = 1480, height = 1480)
+pheatmap(log10(OTUs+1),
+         show_rownames=FALSE,
+         show_colnames=TRUE,
+         treeheight_row=0,
+         treeheight_col=0,
+         annotation_col=data.frame(row.names=colnames(OTUs),
+                                   is.control=as.numeric(
+                                       grepl("H2O|Argave|Wolf|Paramix",
+                                             colnames(OTUs)))),
+         annotation_legend = FALSE)
+dev.off()
 
+sum(OTUs)
 
 amplicon <- gsub("OTU\\d+\\|", "", rownames(OTUs))
 
@@ -91,8 +125,9 @@ pheatmap(log10(t(SUM.amp)+1),
          annotation_legend = FALSE)
 dev.off()
 
-TAX.raw <- read.csv("/SAN/Metabarcoding/Hyena/second/sorted_amps/usearch/ALL_outs.taxtable",
-                sep = ",")
+TAX.raw <-
+    read.csv("/SAN/Metabarcoding/Hyena/second/sorted_amps/usearch/ALL_outs_nt.taxtable",
+             sep = ",")
 
 TAX.raw$query <- gsub(".fastq.otus.fa", "", TAX.raw$query)
 
@@ -104,31 +139,39 @@ T.l <- by(TAX.raw, TAX.raw$query,  function (x) {
      all.best <- x[x$bitscore==b.bit, ]
      ### A little last common ancestor play here... BUT wait ... 
      ## a shortcut throwing out OTUs that don't agree at least on the
-     ## family level and allowing only the best hit afterwards
-     u.family <- unique(all.best$family)
+     ## class level and allowing only the best hit afterwards
+     u.family <- unique(all.best$class)
      if(length(u.family)==1){
          return(all.best)
      }
 })
 
 
 TAX <- do.call(rbind, T.l)
-head(TAX[order(TAX$amplicon), ])
+tail(TAX[order(TAX$amplicon), ])
 rownames(TAX) <- NULL
 
+
+
 ## Only consider Euks now
 TAX <- TAX[TAX$superkingdom%in%"Eukaryota", ]
 
 ## remove some really weird stuff FIND later out where the errors
 ## are!! Database errors... 
 table(TAX$phylum)
 
+## should be fixed in database at some point... but now as a shortkut
+## here
 TAX <- TAX[!TAX$phylum%in%c("Cnidaria", "Porifera",
                             "Bacillariophyta",  ## maybe okay?
                             "Eustigmatophyceae" ## maybe okay?
                             ) ,]
 
-table(TAX$phylum)
+## ## now use only best hit
+## TAX <- TAX[!duplicated(TAX$query), ]
+## tail(TAX[order(TAX$amplicon), ])
+
+## table(TAX$phylum)
 
 ### Summarizing by class
 foo <- merge(TAX, OTUs, by.x="query", by.y=0)
@@ -137,18 +180,22 @@ foobar <- foobar[order(rowSums(foobar), decreasing=TRUE), ]
 foobar <- foobar[!rownames(foobar)%in%c("", "undef"), ]
 foobar <- foobar[, !grepl("H2O|Argave|Wolf", colnames(foobar))]
 
-foobar <- foobar[rowSums(foobar)>116, ]
-
 mean.columns <- function(x){
   reps <- as.factor(gsub("_S\\d+$", "", colnames(x)))
   y <- do.call(rbind, by(t(x), reps, colMeans))
   t(y)
 }
 
+## mean between replicates
 baz <- mean.columns(foobar)
+## removing stuff with very low support from only one replicate
 baz[baz<1] <- 0
 
-devSVG("figures/Hyena_class_sum_heat.svg", width=7, height=7)
+## remove lowly represented classes
+baz <- baz[rowSums(baz)>20, ]
+
+
+devSVG("figures/Hyena_class_sumNT_heat.svg", width=7, height=7)
 pheatmap(log10(baz+1),
          show_rownames=TRUE,
          show_colnames=FALSE,

R/Wolf_first_rough.R

@@ -52,7 +52,6 @@ for(i in 1:nrow(OTUs)){
     OTUs[i, out.z(OTUs[i,])] <- 0
 }
 
-## Maybe much harder just kill when away from the mean/median?
 
 ######## Analyse just the READ counts per amplicon and sample #####
 SUM.amp <- do.call(cbind, by(OTUs, amplicon, colSums))

README.md

@@ -1,5 +1,11 @@
 # AA_Metabarcoding
 
+An overview of a metabarcoding pipeline for multiple marker data from
+the Fluidigm Access Array. The present version is intended for a
+overview of the preliminary processing of data. Code can be reviewed
+in the /scripts and /R folders. The processing is not reproducible at
+the present version as raw data files cannot be accessed (yet). 
+
 ## Preprocessing
 
 ### stratify the data into amplicons and samples