diff --git a/Makefile b/Makefile
index 7b0ec06..a29ea50 100644
--- a/Makefile
+++ b/Makefile
@@ -203,7 +203,7 @@ DEP := $(OBJ:.o=.d)
FLAGS := $(CPPFLAGS) $(CXXFLAGS)
-VERSION = v0.3.1
+VERSION = v0.4
ifneq ($(wildcard .git),)
VERSION := $(VERSION)-$(shell git describe --always)
@@ -260,7 +260,7 @@ $(PROGRAM): $(OBJ)
.PHONY: clean
clean:
- $(RM) $(OBJ) $(DEP) $(PREP) $(ASM) $(VERSIONH) $(BUILDH) $(PROGRAM)
+ $(RM) $(OBJ) $(DEP) $(PREP) $(ASM) $(VERSIONH) $(BUILDH) $(PROGRAM) vgcore.*
####################################################################################################
## [test]
diff --git a/amova.cpp b/amova.cpp
index e906d98..75d188e 100644
--- a/amova.cpp
+++ b/amova.cpp
@@ -1,8 +1,10 @@
#include "amova.h"
+#include "dmat.h"
+
/// @brief calculate part of AMOVA statistics where the calculations are independent of the distance matrix but depend on the metadata
-/// @param amv amovaStruct
+/// @param amova amovaStruct
/// @param mtd metadataStruct
/// @return void
/// @details
@@ -11,16 +13,16 @@
/// i.e. one call is enough for all amova replicates
///
///
-inline void amova_run_shared(amovaStruct* amv) {
+void amova_run_shared(amovaStruct* amova) {
- metadataStruct* mtd = amv->metadata;
+ metadataStruct* mtd = amova->metadata;
const int nInd = mtd->nInd;
const size_t nLevels = (size_t)mtd->nLevels; // L
size_t lvlidx; // 0-based lvl
/// ------------------------------------------------------------------- ///
- /// -> DEGREES OF FREEDOM (amv->df)
+ /// -> DEGREES OF FREEDOM (amova->df)
/// @note
///
/// df_i = k_i - k_{i-1} , 0 < i < L
@@ -46,21 +48,21 @@ inline void amova_run_shared(amovaStruct* amv) {
lvlidx = 0;
// $ df_1 = k_1 - k_0 $
- amv->df[lvlidx] = mtd->level2groupIndices[lvlidx]->len - 1;
+ amova->df[lvlidx] = mtd->level2groupIndices[lvlidx]->len - 1;
++lvlidx;
while (lvlidx < nLevels - 1) {
// $ df_{lvlidx+1} = k_{lvlidx+1} - k_{lvlidx} $
- amv->df[lvlidx] = mtd->level2groupIndices[lvlidx]->len - mtd->level2groupIndices[lvlidx - 1]->len;
+ amova->df[lvlidx] = mtd->level2groupIndices[lvlidx]->len - mtd->level2groupIndices[lvlidx - 1]->len;
++lvlidx;
}
// $ df_L = N - k_L $
- amv->df[lvlidx] = nInd - mtd->level2groupIndices[nLevels - 2]->len;
+ amova->df[lvlidx] = nInd - mtd->level2groupIndices[nLevels - 2]->len;
++lvlidx;
// $ df_{total} = N - 1 $
- amv->df_total = nInd - 1;
+ amova->df_total = nInd - 1;
double sum;
@@ -119,7 +121,7 @@ inline void amova_run_shared(amovaStruct* amv) {
}
} while (0);
- amv->vmat[idx] = val;
+ amova->vmat[idx] = val;
++idx;
}
@@ -149,8 +151,8 @@ inline void amova_run_shared(amovaStruct* amv) {
double left;
double right;
- // left = amv->vmat[MATRIX_GET_INDEX_UTID_IJ(iti, itj, nLevels)];
- left = amv->vmat[idx];
+ // left = amova->vmat[MATRIX_GET_INDEX_UTID_IJ(iti, itj, nLevels)];
+ left = amova->vmat[idx];
if (iti == 0) {
@@ -171,16 +173,16 @@ inline void amova_run_shared(amovaStruct* amv) {
} else {
// 1 < i <= j (0 < iti <= itj)
- right = amv->vmat[MATRIX_GET_INDEX_UTID_IJ(iti - 1, itj, nLevels)];
+ right = amova->vmat[MATRIX_GET_INDEX_UTID_IJ(iti - 1, itj, nLevels)];
}
- val = (1.0 / (double)amv->df[iti]) * (left - right);
+ val = (1.0 / (double)amova->df[iti]) * (left - right);
} while (0);
- // amv->cmat[MATRIX_GET_INDEX_UTID_IJ(iti, itj, nLevels)] = val;
- amv->cmat[idx] = val;
+ // amova->cmat[MATRIX_GET_INDEX_UTID_IJ(iti, itj, nLevels)] = val;
+ amova->cmat[idx] = val;
++idx;
}
@@ -190,7 +192,7 @@ inline void amova_run_shared(amovaStruct* amv) {
for (iti = 0;iti < nLevels;++iti) {
// itj==iti
const size_t idx = MATRIX_GET_INDEX_UTID_IJ(iti, iti, nLevels);
- amv->lmat[idx] = 1.0 / amv->cmat[idx];
+ amova->lmat[idx] = 1.0 / amova->cmat[idx];
}
@@ -206,9 +208,9 @@ inline void amova_run_shared(amovaStruct* amv) {
for (size_t itj = iti + 1;itj < nLevels;++itj) {
sum = 0.0;
for (size_t p = iti + 1;p <= itj;++p) {
- sum += amv->cmat[MATRIX_GET_INDEX_UTID_IJ(iti, p, nLevels)] * amv->lmat[MATRIX_GET_INDEX_UTID_IJ(p, itj, nLevels)];
+ sum += amova->cmat[MATRIX_GET_INDEX_UTID_IJ(iti, p, nLevels)] * amova->lmat[MATRIX_GET_INDEX_UTID_IJ(p, itj, nLevels)];
}
- amv->lmat[MATRIX_GET_INDEX_UTID_IJ(iti, itj, nLevels)] = -1.0 * (sum / amv->cmat[MATRIX_GET_INDEX_UTID_IJ(iti, iti, nLevels)]);
+ amova->lmat[MATRIX_GET_INDEX_UTID_IJ(iti, itj, nLevels)] = -1.0 * (sum / amova->cmat[MATRIX_GET_INDEX_UTID_IJ(iti, iti, nLevels)]);
}
}
@@ -223,27 +225,38 @@ struct simple_pthread_data_t {
void* amova_run_private(void* data) {
+ DEVASSERT(data != NULL);
const size_t runidx = (size_t)((simple_pthread_data_t*)data)->job_id;
- amovaStruct* amv = (amovaStruct*)((simple_pthread_data_t*)data)->shared_data;
+ amovaStruct* amova = (amovaStruct*)((simple_pthread_data_t*)data)->shared_data;
dmat_t* dm = (dmat_t*)((simple_pthread_data_t*)data)->private_data;
double* matrix = dm->matrix[runidx];
- metadataStruct* mtd = amv->metadata;
- double* ss_total = amv->ss_total + runidx;
- double* ssd_total = amv->ssd_total + runidx;
- double* msd_total = amv->msd_total + runidx;
+ //TODO investigate the squared transform
+ if (DMAT_TRANSFORM_SQUARE == dm->transform) {
+ // ok
+ } else if (DMAT_TRANSFORM_NONE == dm->transform) {
+ for (size_t i = 0;i < dm->size;++i) {
+ matrix[i] = SQUARE(matrix[i]);
+ }
+ }
+
+
+ metadataStruct* mtd = amova->metadata;
+ double* ss_total = amova->ss_total + runidx;
+ double* ssd_total = amova->ssd_total + runidx;
+ double* msd_total = amova->msd_total + runidx;
size_t iti;
size_t itj;
- double* ss = amv->ss[runidx];
- double* ssd = amv->ssd[runidx];
- double* msd = amv->msd[runidx];
- double* sigmasq = amv->sigmasq[runidx];
- double* phi_xt = amv->phi_xt[runidx];
- double* phi_xy = (amv->phi_xy == NULL ? NULL : amv->phi_xy[runidx]);
+ double* ss = amova->ss[runidx];
+ double* ssd = amova->ssd[runidx];
+ double* msd = amova->msd[runidx];
+ double* sigmasq = amova->sigmasq[runidx];
+ double* phi_xt = amova->phi_xt[runidx];
+ double* phi_xy = (amova->phi_xy == NULL ? NULL : amova->phi_xy[runidx]);
const int nInd = mtd->nInd;
const size_t nLevels = (size_t)mtd->nLevels;
@@ -275,6 +288,7 @@ void* amova_run_private(void* data) {
nPairsInGroup = mtd->group2pairIndices[groupIndex]->len;
for (p = 0; p < nPairsInGroup; ++p) {
+ DEVASSERT(pairsInGroup[p] < dm->size);
sum += matrix[pairsInGroup[p]];
}
@@ -327,11 +341,11 @@ void* amova_run_private(void* data) {
lvlidx = 0;
while (lvlidx < nLevels) {
- msd[lvlidx] = ssd[lvlidx] / amv->df[lvlidx];
+ msd[lvlidx] = ssd[lvlidx] / amova->df[lvlidx];
++lvlidx;
}
- *msd_total = *ssd_total / amv->df_total;
+ *msd_total = *ssd_total / amova->df_total;
/// -----------------------------------------------------------------------
/// -> SIGMA SQUARED (VARIANCE COMPONENTS)
@@ -341,14 +355,14 @@ void* amova_run_private(void* data) {
for (iti = 0;iti < nLevels;++iti) {
sigmasq[iti] = 0.0;
for (itj = iti; itj < nLevels; ++itj) {
- // sigmasq[iti] += msd[itj] * amv->lmat[MATRIX_GET_INDEX_UTID_IJ(iti, itj, nLevels)];
- sigmasq[iti] += msd[itj] * amv->lmat[idx];
+ // sigmasq[iti] += msd[itj] * amova->lmat[MATRIX_GET_INDEX_UTID_IJ(iti, itj, nLevels)];
+ sigmasq[iti] += msd[itj] * amova->lmat[idx];
++idx;
}
}
- amv->sigmasq_total[0] = 0.0;
+ amova->sigmasq_total[0] = 0.0;
for (size_t i = 0; i < nLevels; ++i) {
- amv->sigmasq_total[0] += sigmasq[i];
+ amova->sigmasq_total[0] += sigmasq[i];
}
/// -----------------------------------------------------------------------
@@ -371,7 +385,7 @@ void* amova_run_private(void* data) {
for (itj = 0; itj <= iti;++itj) {
sum += sigmasq[itj];
}
- phi_xt[iti] = sum / amv->sigmasq_total[0];
+ phi_xt[iti] = sum / amova->sigmasq_total[0];
}
return(NULL);
@@ -379,7 +393,7 @@ void* amova_run_private(void* data) {
}
-void amovaStruct_print_as_csv(amovaStruct* amv, metadataStruct* mtd) {
+void amovaStruct_print_as_csv(amovaStruct* amova, metadataStruct* mtd, const char* bootstrap_results) {
// header
// type,label,value
@@ -392,9 +406,9 @@ void amovaStruct_print_as_csv(amovaStruct* amv, metadataStruct* mtd) {
outFiles->out_amova_fs->kbuf = kbuf_init();
kstring_t* kbuf = outFiles->out_amova_fs->kbuf;
- ksprintf(kbuf, "df,Total,%d\n", amv->df_total);
- ksprintf(kbuf, "SSD,Total,%f\n", amv->ssd_total[0]);
- ksprintf(kbuf, "MSD,Total,%f\n", amv->msd_total[0]);
+ ksprintf(kbuf, "df,Total,%d\n", amova->df_total);
+ ksprintf(kbuf, "SSD,Total,%f\n", amova->ssd_total[0]);
+ ksprintf(kbuf, "MSD,Total,%f\n", amova->msd_total[0]);
// among idx | within idx
@@ -408,233 +422,271 @@ void amovaStruct_print_as_csv(amovaStruct* amv, metadataStruct* mtd) {
size_t amonglvlidx;
amonglvlidx = 0;
- ksprintf(kbuf, "df,Among_%s_within_Total,%d\n", mtd->levelNames->d[amonglvlidx], amv->df[amonglvlidx]);
- ksprintf(kbuf, "SSD,Among_%s_within_Total,%f\n", mtd->levelNames->d[amonglvlidx], amv->ssd[0][amonglvlidx]);
- ksprintf(kbuf, "MSD,Among_%s_within_Total,%f\n", mtd->levelNames->d[amonglvlidx], amv->msd[0][amonglvlidx]);
+ ksprintf(kbuf, "df,Among_%s_within_Total,%d\n", mtd->levelNames->d[amonglvlidx], amova->df[amonglvlidx]);
+ ksprintf(kbuf, "SSD,Among_%s_within_Total,%f\n", mtd->levelNames->d[amonglvlidx], amova->ssd[0][amonglvlidx]);
+ ksprintf(kbuf, "MSD,Among_%s_within_Total,%f\n", mtd->levelNames->d[amonglvlidx], amova->msd[0][amonglvlidx]);
++amonglvlidx;
while (amonglvlidx < nLevels) {
- ksprintf(kbuf, "df,Among_%s_within_%s,%d\n", mtd->levelNames->d[amonglvlidx], mtd->levelNames->d[amonglvlidx - 1], amv->df[amonglvlidx]);
- ksprintf(kbuf, "SSD,Among_%s_within_%s,%f\n", mtd->levelNames->d[amonglvlidx], mtd->levelNames->d[amonglvlidx - 1], amv->ssd[0][amonglvlidx]);
- ksprintf(kbuf, "MSD,Among_%s_within_%s,%f\n", mtd->levelNames->d[amonglvlidx], mtd->levelNames->d[amonglvlidx - 1], amv->msd[0][amonglvlidx]);
+ ksprintf(kbuf, "df,Among_%s_within_%s,%d\n", mtd->levelNames->d[amonglvlidx], mtd->levelNames->d[amonglvlidx - 1], amova->df[amonglvlidx]);
+ ksprintf(kbuf, "SSD,Among_%s_within_%s,%f\n", mtd->levelNames->d[amonglvlidx], mtd->levelNames->d[amonglvlidx - 1], amova->ssd[0][amonglvlidx]);
+ ksprintf(kbuf, "MSD,Among_%s_within_%s,%f\n", mtd->levelNames->d[amonglvlidx], mtd->levelNames->d[amonglvlidx - 1], amova->msd[0][amonglvlidx]);
++amonglvlidx;
}
// -> phi_xy
// only run if nLevels > 2
- if (amv->phi_xy != NULL) {
+ if (amova->phi_xy != NULL) {
for (size_t iti = 1; iti < nLevels - 1;++iti) {
- ksprintf(kbuf, "Phi,%s_in_%s,%f\n", mtd->levelNames->d[iti], mtd->levelNames->d[iti - 1], amv->phi_xy[0][iti - 1]);
+ ksprintf(kbuf, "Phi,%s_in_%s,%f\n", mtd->levelNames->d[iti], mtd->levelNames->d[iti - 1], amova->phi_xy[0][iti - 1]);
}
}
// phi_xt
for (size_t iti = 0;iti < (size_t)(mtd->nLevels - 1);++iti) {
- ksprintf(kbuf, "Phi,%s_in_Total,%f\n", mtd->levelNames->d[iti], amv->phi_xt[0][iti]);
+ ksprintf(kbuf, "Phi,%s_in_Total,%f\n", mtd->levelNames->d[iti], amova->phi_xt[0][iti]);
}
for (size_t iti = 0;iti < nLevels;++iti) {
for (size_t itj = iti;itj < nLevels;++itj) {
- ksprintf(kbuf, "Variance_coefficient,c_%ld_%ld,%f\n", iti, itj, amv->cmat[MATRIX_GET_INDEX_UTID_IJ(iti, itj, nLevels)]);
+ ksprintf(kbuf, "Variance_coefficient,c_%ld_%ld,%f\n", iti, itj, amova->cmat[MATRIX_GET_INDEX_UTID_IJ(iti, itj, nLevels)]);
}
}
for (size_t i = 0;i < nLevels;++i) {
- ksprintf(kbuf, "Variance_component,%s,%f\n", mtd->levelNames->d[i], amv->sigmasq[0][i]);
- ksprintf(kbuf, "Percentage_variance,%s,%f\n", mtd->levelNames->d[i], (amv->sigmasq[0][i] / amv->sigmasq_total[0]) * 100.0);
+ ksprintf(kbuf, "Variance_component,%s,%f\n", mtd->levelNames->d[i], amova->sigmasq[0][i]);
+ ksprintf(kbuf, "Percentage_variance,%s,%f\n", mtd->levelNames->d[i], (amova->sigmasq[0][i] / amova->sigmasq_total[0]) * 100.0);
}
+ if (NULL != bootstrap_results) {
+ ksprintf(kbuf, "%s", bootstrap_results);
+ }
outFiles->out_amova_fs->kbuf_write();
}
-void amovaStruct_print_as_table(FILE* fp, amovaStruct* amv, metadataStruct* mtd) {
- //TODO UPDATEME
-
-// const size_t nLevels = (size_t)mtd->nLevels;
-
-// fprintf(fp, "\n");
-// fprintf(fp, "\n\n");
-// fprintf(fp, "========================================== AMOVA ==========================================");
-
-// fprintf(fp, "\n\n");
-// fprintf(fp, "Formula: %s ", args->formula);
-// fprintf(fp, "\n");
-// fprintf(fp, "Source of variation\t\t\t\t\td.f.\tSSD\t\tMSD");
-// fprintf(fp, "\n");
-// fprintf(fp, "---------------------------------------------------------------------------------------------");
-// fprintf(fp, "\n\n");
-// fprintf(fp, "\n");
-
-
-// size_t x = 1;
-// fprintf(fp, "Among %-15s\t\t\t\t\t%d\t%f\t%f", mtd->levelNames->get(x), amv->df[x], amv->ssd[x], amv->msd[x]);
-
-// while (x < mtd->nLevels) {
-// fprintf(fp, "\n");
-// fprintf(fp, "Among %s within %-25s\t%d\t%f\t%f", mtd->levelNames->get(x + 1), mtd->levelNames->get(x), amv->df[x], amv->ssd[x], amv->msd[x]);
-// fprintf(fp, "\n");
-// ++x;
-// }
-
-// fprintf(fp, "\n");
-// fprintf(fp, "Among %s within %-25s\t%d\t%f\t%f", mtd->levelNames->get(0), mtd->levelNames->get(x), amv->df[x], amv->ssd[x], amv->msd[x]);
-// fprintf(fp, "\n");
-
-
-// ++x;
-// fprintf(fp, "\n");
-// fprintf(fp, "Total\t\t\t\t\t\t\t%d\t%f\t%f", amv->df[x], amv->ssd[x], amv->msd[x]);
-
-// fprintf(fp, "\n\n\n");
-// fprintf(fp, "Variance components:\n\n\tVariance Component\tSigma^2\t\t%% of total");
-
-// for (size_t i = 0;i < nLevels;++i) {
-// fprintf(fp, "\n\t%-20s", mtd->levelNames->d[i + 1]);
-// fprintf(fp, "\t%f", amv->sigmasq[0][i]);
-// // fprintf(fp, "\t%f", pct_sigmasq[i]);
-// }
-
-// // Lowest level (i.e. Individual)
-// fprintf(fp, "\n\t%-20s", mtd->levelNames->d[0]);
-// fprintf(fp, "\t%f", amv->sigmasq[0][nLevels - 1]);
-// // fprintf(fp, "\t%f", pct_sigmasq[_ncoef - 1]);
-
-// fprintf(fp, "\n\n\n");
-// fprintf(fp, "\nVariance coefficients:\n\n\t");
-// if (mtd->nLevels == 2) {
-// fprintf(fp, "%f", amv->ncoef[0]);
-// } else {
-// for (size_t i = 0;i < nLevels;++i) {
-// fprintf(fp, "%f\t", amv->ncoef[i]);
-// }
-// }
-
-// fprintf(fp, "\n\n\n");
-// fprintf(fp, "Phi-statistic:\n\n");
-// for (size_t i = 0; i < ((2 * nLevels) - 3); ++i) {
-// fprintf(fp, "\t%f", amv->phi[i]);
-// }
-// fprintf(fp, "\n\n");
-// fprintf(fp, "=============================================================================================");
-// fprintf(fp, "\n\n");
+void amovaStruct_print_as_table(amovaStruct* amova, metadataStruct* mtd) {
+
+ kstring_t kbuf = KS_INITIALIZE;
+ ksprintf(&kbuf, "=== AMOVA ======================================================================\n");
+ ksprintf(&kbuf, "Formula: %s\n\n", args->formula);
+ ksprintf(&kbuf, "Source of variation%-30sd.f.%-6sSSD%-10sMSD\n", " ", " ", " ");
+ ksprintf(&kbuf, "--------------------------------------------------------------------------------\n");
+ size_t amonglvlidx = 0;
+ kstring_t tmp = KS_INITIALIZE;
+ ksprintf(&tmp, "Among %s within %s", mtd->levelNames->d[amonglvlidx], "Total");
+ ksprintf(&kbuf, "%-49s%-10d%-13f%-13f\n", tmp.s, amova->df[amonglvlidx], amova->ssd[0][amonglvlidx], amova->msd[0][amonglvlidx]);
+ ++amonglvlidx;
+ while (amonglvlidx < mtd->nLevels) {
+ ks_clear(&tmp);
+ ksprintf(&tmp, "Among %s within %s", mtd->levelNames->d[amonglvlidx], mtd->levelNames->d[amonglvlidx - 1]);
+ ksprintf(&kbuf, "%-49s%-10d%-13f%-13f\n", tmp.s, amova->df[amonglvlidx], amova->ssd[0][amonglvlidx], amova->msd[0][amonglvlidx]);
+ ++amonglvlidx;
+ }
+ ksprintf(&kbuf, "\n\n");
+
+
+ ksprintf(&kbuf, "\nVariance coefficients:\n");
+ for (size_t iti = 0;iti < mtd->nLevels;++iti) {
+ for (size_t itj = iti;itj < mtd->nLevels;++itj) {
+ ksprintf(&kbuf, "c_%ld_%ld\t%f\n", iti, itj, amova->cmat[MATRIX_GET_INDEX_UTID_IJ(iti, itj, mtd->nLevels)]);
+ }
+ }
+
+ ksprintf(&kbuf, "\n\n");
+
+ // print variance components
+ ksprintf(&kbuf, "Variance components:\n");
+ for (size_t i = 0;i < mtd->nLevels;++i) {
+ ksprintf(&kbuf, "%s\t%f\t%f%%\n", mtd->levelNames->d[i], amova->sigmasq[0][i], (amova->sigmasq[0][i] / amova->sigmasq_total[0]) * 100.0);
+ }
+
+ ksprintf(&kbuf, "================================================================================\n");
+
+ ksprintf(&kbuf, "\n\n");
+ fprintf(stdout, "%s\n", kbuf.s);
+
+ ks_free(&kbuf);
+ ks_free(&tmp);
+
+ return;
}
+void amovaStruct_destroy(amovaStruct* amova) {
+
+ amova->metadata = NULL;
+
+ for (size_t i = 0;i < amova->nRuns;++i) {
+ FREE(amova->ss[i]);
+ FREE(amova->ssd[i]);
+ FREE(amova->msd[i]);
+ FREE(amova->sigmasq[i]);
+ FREE(amova->phi_xt[i]);
+ if (amova->phi_xy != NULL) {
+ FREE(amova->phi_xy[i]);
+ }
+ }
+
+ FREE(amova->df);
+ FREE(amova->ss);
+ FREE(amova->ss_total);
+ FREE(amova->ssd);
+ FREE(amova->ssd_total);
+ FREE(amova->msd);
+ FREE(amova->msd_total);
+ FREE(amova->vmat);
+ FREE(amova->cmat);
+ FREE(amova->lmat);
+ FREE(amova->sigmasq);
+ FREE(amova->sigmasq_total);
+ FREE(amova->phi_xt);
+ if (amova->phi_xy != NULL) {
+ FREE(amova->phi_xy);
+ }
-// amovaStruct* amovaStruct_get(distanceMatrixStruct* dm, metadataStruct* mtd, blobStruct* blobs) {
+ FREE(amova);
-// // nRuns = n bootstrap runs + 1 (the original run)
-// const size_t nRuns = (size_t)args->nBootstraps + 1;
+}
-// amovaStruct* amova = amovaStruct_init(mtd, nRuns);
+amovaStruct* amovaStruct_init(metadataStruct* mtd, const int nAmovaRuns) {
-// const size_t maxnThreads = (args->nThreads == 0) ? 1 : args->nThreads;
+ amovaStruct* ret = (amovaStruct*)malloc(sizeof(amovaStruct));
+ ret->metadata = mtd;
-// amova_run_shared(amova);
+ const size_t nLevels = (size_t)mtd->nLevels;
-// pthread_t threads[nRuns];
-// simple_pthread_data_t data[nRuns];
+ const size_t nRuns = (size_t)nAmovaRuns;
+ ret->nRuns = nRuns;
+
+
+ const size_t nCmat = (nLevels * (nLevels + 1)) / 2;
+ ret->vmat = NULL;
+ ret->vmat = (double*)malloc((nCmat) * sizeof(double));
+ ASSERT(ret->vmat != NULL);
+ ret->cmat = NULL;
+ ret->cmat = (double*)malloc((nCmat) * sizeof(double));
+ ASSERT(ret->cmat != NULL);
+ ret->lmat = NULL;
+ ret->lmat = (double*)malloc((nCmat) * sizeof(double));
+ ASSERT(ret->cmat != NULL);
+ for (size_t i = 0; i < nCmat; ++i) {
+ ret->cmat[i] = 0.0;
+ ret->lmat[i] = 0.0;
+ ret->vmat[i] = 0.0;
+ }
-// int nJobsAlive = 0;
-// size_t run_to_wait = 0;
-// size_t runidx = 0;
+ ret->df = NULL;
+ ret->df = (int*)malloc((nLevels) * sizeof(int));
+ ASSERT(ret->df != NULL);
-// while (runidx < nRuns) {
+ ret->df_total = 0;
-// while (1) {
-// if (nJobsAlive < maxnThreads) {
-// break;
-// }
-// while (nJobsAlive >= maxnThreads) {
-// // wait for the run that was sent first
-// if (0 != pthread_join(threads[run_to_wait], NULL)) {
-// ERROR("Problem with joining the thread.");
-// }
-// ++run_to_wait;
-// nJobsAlive--;
-// }
-// }
+ ret->ss = NULL;
+ ret->ss = (double**)malloc((nRuns) * sizeof(double*));
+ ASSERT(ret->ss != NULL);
-// data[runidx].job_id = runidx;
-// data[runidx].shared_data = (void*)amova;
-// data[runidx].private_data = (void*)dm;
+ ret->ss_total = NULL;
+ ret->ss_total = (double*)malloc((nRuns) * sizeof(double));
+ ASSERT(ret->ss_total != NULL);
-// if (0 != pthread_create(&threads[runidx], NULL, amova_run_private, (void*)&data[runidx])) {
-// ERROR("Problem with the spawning thread.");
-// }
-// nJobsAlive++;
+ ret->ssd = NULL;
+ ret->ssd = (double**)malloc((nRuns) * sizeof(double*));
+ ASSERT(ret->ssd != NULL);
-// ++runidx;
-// }
+ ret->ssd_total = NULL;
+ ret->ssd_total = (double*)malloc((nRuns) * sizeof(double));
+ ASSERT(ret->ssd_total != NULL);
-// while (nJobsAlive > 0) {
-// if (0 != pthread_join(threads[run_to_wait], NULL)) {
-// ERROR("Problem with joining the thread.");
-// }
-// ++run_to_wait;
-// nJobsAlive--;
-// }
+ ret->msd = NULL;
+ ret->msd = (double**)malloc((nRuns) * sizeof(double*));
+ ASSERT(ret->msd != NULL);
-// // //TODO HERE BURADAKALDIN!
-// // const int nReplicates = blob->bootstraps->nReplicates;
-// // THREADS[i] = new amovaBootstrapThreads(blob->bootstraps->replicates[i]->amova, blob->bootstraps->replicates[i]->distanceMatrix, mtd);
+ ret->msd_total = NULL;
+ ret->msd_total = (double*)malloc((nRuns) * sizeof(double));
+ ASSERT(ret->msd_total != NULL);
-// // //TODO change these with new vals
-// // blob->bootstraps->nPhiValues = (mtd->nLevels * 2) - 3;
-// // blob->bootstraps->phiValues = (double**)malloc(blob->bootstraps->nPhiValues * sizeof(double*));
+ ret->sigmasq = NULL;
+ ret->sigmasq = (double**)malloc((nRuns) * sizeof(double*));
+ ASSERT(ret->sigmasq != NULL);
-// // for (int i = 0; i < blob->bootstraps->nPhiValues; i++) {
-// // blob->bootstraps->phiValues[i] = (double*)malloc(nReplicates * sizeof(double));
+ ret->sigmasq_total = NULL;
+ ret->sigmasq_total = (double*)malloc((nRuns) * sizeof(double));
+ ASSERT(ret->sigmasq_total != NULL);
-// // for (int r = 0; r < nReplicates; r++) {
-// // // blob->bootstraps->phiValues[i][r] = THREADS[r]->amova->phi[i];
-// // }
-// // }
+ ret->phi_xt = NULL;
+ ret->phi_xt = (double**)malloc((nRuns) * sizeof(double*));
+ ASSERT(ret->phi_xt != NULL);
-// amovaStruct_print_as_csv(amova, mtd);
-// // if (0 < args->nBootstraps) {
-// // spawnThreads_amovaBootstrap(metadata, blobs);
-// // blobs->bootstraps->print_confidenceInterval(stderr);
-// // }
+ ret->phi_xy = NULL;
+ if (nLevels > 2) {
+ ret->phi_xy = (double**)malloc((nRuns) * sizeof(double*));
+ ASSERT(ret->phi_xy != NULL);
+ }
-// // if (0 < args->nBootstraps) {
-// // ASSERT(blobs != NULL);
+ for (size_t i = 0;i < nRuns;++i) {
+ ret->ss[i] = (double*)malloc((nLevels) * sizeof(double));
+ ASSERT(ret->ss[i] != NULL);
+ ret->ss_total[i] = 0.0;
-// // spawnThreads_amovaBootstrap(mtd, blobs);
-// // blobs->bootstraps->print_confidenceInterval(stderr);
-// // }
+ ret->ssd[i] = (double*)malloc((nLevels) * sizeof(double));
+ ASSERT(ret->ssd[i] != NULL);
+ ret->ssd_total[i] = 0.0;
-// // if (0 == doAmova(amova, dm, metadata)) {
-// // fprintf(stderr, "\n[INFO]\t-> Finished running AMOVA");
-// // if (0 < args->nBootstraps) {
-// // fprintf(stderr, " for the original dataset.\n");
-// // } else {
-// // fprintf(stderr, ".\n");
-// // }
+ ret->msd[i] = (double*)malloc((nLevels) * sizeof(double));
+ ASSERT(ret->msd[i] != NULL);
-// // if (args->printAmovaTable == 1) {
-// // amova->print_as_table(stdout);
-// //TODO make this to print file instead
-// // }
+ ret->msd_total[i] = 0.0;
-// return (amova);
-// }
+ ret->sigmasq[i] = (double*)malloc((nLevels) * sizeof(double));
+ ASSERT(ret->sigmasq[i] != NULL);
+ ret->sigmasq_total[i] = 0.0;
+
+ ret->phi_xt[i] = (double*)malloc((nLevels - 1) * sizeof(double));
+ ASSERT(ret->phi_xt[i] != NULL);
+
+ if (ret->phi_xy != NULL) {
+ ret->phi_xy[i] = NULL;
+ ret->phi_xy[i] = (double*)malloc((nLevels - 2) * sizeof(double));
+ ASSERT(ret->phi_xy[i] != NULL);
+ for (size_t j = 0;j < nLevels - 2;++j) {
+ ret->phi_xy[i][j] = 0.0;
+ }
+ }
+
+
+ for (size_t j = 0;j < nLevels;++j) {
+ ret->ss[i][j] = 0.0;
+ ret->ssd[i][j] = 0.0;
+ ret->msd[i][j] = 0.0;
+ ret->sigmasq[i][j] = 0.0;
+ if (j < nLevels - 1) {
+ ret->phi_xt[i][j] = 0.0;
+ }
+ }
+
+ }
+
+ return(ret);
+}
+
+
+amovaStruct* amovaStruct_get(paramStruct* pars, metadataStruct* mtd) {
-amovaStruct* amovaStruct_get(paramStruct* pars, metadataStruct* mtd, blobStruct* blobs) {
// nRuns = n bootstrap runs + 1 (the original run)
- const size_t nRuns = (size_t)args->nBootstraps + 1;
+ const int nRuns = (args->nBootstraps > 0) ? (args->nBootstraps + 1) : 1;
+
amovaStruct* amova = amovaStruct_init(mtd, nRuns);
@@ -686,50 +738,93 @@ amovaStruct* amovaStruct_get(paramStruct* pars, metadataStruct* mtd, blobStruct*
nJobsAlive--;
}
- // //TODO HERE BURADAKALDIN!
- // const int nReplicates = blob->bootstraps->nReplicates;
- // THREADS[i] = new amovaBootstrapThreads(blob->bootstraps->replicates[i]->amova, blob->bootstraps->replicates[i]->distanceMatrix, mtd);
+ kstring_t kbuf_csv = KS_INITIALIZE;
+ kstring_t kbuf_table = KS_INITIALIZE;
+
+ if (nRuns > 1) {
+
+ double mean, sd, margin_of_error, ci_lower, ci_upper;
+
+ double ci = args->bootstrap_ci;
+ int nReps = nRuns - 1;
- // //TODO change these with new vals
- // blob->bootstraps->nPhiValues = (mtd->nLevels * 2) - 3;
- // blob->bootstraps->phiValues = (double**)malloc(blob->bootstraps->nPhiValues * sizeof(double*));
+ ksprintf(&kbuf_csv, "Block_Bootstrapping,nReplicates,%d\n", nReps);
+ ksprintf(&kbuf_csv, "Block_Bootstrapping,Confidence_Interval,%f\n", ci);
+ ksprintf(&kbuf_table, "Block Bootstrapping:\n");
+ ksprintf(&kbuf_table, "Number of replicates: %d\n", nReps);
+ ksprintf(&kbuf_table, "Confidence interval: %f\n", ci);
- // for (int i = 0; i < blob->bootstraps->nPhiValues; i++) {
- // blob->bootstraps->phiValues[i] = (double*)malloc(nReplicates * sizeof(double));
+ for (size_t i = 0;i < (mtd->nLevels - 1);++i) {
- // for (int r = 0; r < nReplicates; r++) {
- // // blob->bootstraps->phiValues[i][r] = THREADS[r]->amova->phi[i];
- // }
- // }
+ mean = 0.0;
+ for (size_t r=0;r < nReps;++r) {
+ mean += amova->phi_xt[r][i];
+ }
+ mean = mean / (double)nReps;
+
+ sd = 0.0;
+ for (size_t r=0;r < nReps;++r) {
+ sd += pow(amova->phi_xt[r][i] - mean, 2);
+ }
+ sd = sqrt(sd / (nReps-1)); // sample stdev
+
+ margin_of_error= 1.96 * (sd / sqrt((double)nReps));
+ ci_lower = mean - margin_of_error;
+ ci_upper = mean + margin_of_error;
+
+ ksprintf(&kbuf_csv, "Block_Bootstrapping_Phi_Mean,%s_in_Total,%f\n", mtd->levelNames->d[i], mean);
+ ksprintf(&kbuf_csv, "Block_Bootstrapping_Phi_SD,%s_in_Total,%f\n", mtd->levelNames->d[i], sd);
+ ksprintf(&kbuf_csv, "Block_Bootstrapping_Phi_LowerCI,%s_in_Total,%f\n", mtd->levelNames->d[i], ci_lower);
+ ksprintf(&kbuf_csv, "Block_Bootstrapping_Phi_UpperCI,%s_in_Total,%f\n", mtd->levelNames->d[i], ci_upper);
+
+ ksprintf(&kbuf_table, "Phi(%s in Total): %f\n", mtd->levelNames->d[i], mean);
+ ksprintf(&kbuf_table, "Phi(%s in Total) SD: %f\n", mtd->levelNames->d[i], sd);
+ ksprintf(&kbuf_table, "Phi(%s in Total) Lower CI: %f\n", mtd->levelNames->d[i], ci_lower);
+ ksprintf(&kbuf_table, "Phi(%s in Total) Upper CI: %f\n", mtd->levelNames->d[i], ci_upper);
+
+ }
- amovaStruct_print_as_csv(amova, mtd);
+ if (amova->phi_xy != NULL) {
+ for (size_t i = 0;i < (mtd->nLevels - 2);++i) {
- // if (0 < args->nBootstraps) {
- // spawnThreads_amovaBootstrap(metadata, blobs);
- // blobs->bootstraps->print_confidenceInterval(stderr);
- // }
+ mean = 0.0;
+ for (size_t r=0;r < nReps;++r) {
+ mean += amova->phi_xy[r][i];
+ }
+ mean = mean / (double)nReps;
+
+ sd = 0.0;
+ for (size_t r=0;r < nReps;++r) {
+ sd += pow(amova->phi_xy[r][i] - mean, 2);
+ }
+ sd = sqrt(sd / (nReps-1)); // sample stdev
-// if (0 < args->nBootstraps) {
-// ASSERT(blobs != NULL);
+ margin_of_error= 1.96 * (sd / sqrt((double)nReps));
+ ci_lower = mean - margin_of_error;
+ ci_upper = mean + margin_of_error;
+ ksprintf(&kbuf_csv, "Block_Bootstrapping_Phi_Mean,%s_in_%s,%f\n", mtd->levelNames->d[i + 1], mtd->levelNames->d[i], mean);
+ ksprintf(&kbuf_csv, "Block_Bootstrapping_Phi_SD,%s_in_%s,%f\n", mtd->levelNames->d[i + 1], mtd->levelNames->d[i], sd);
+ ksprintf(&kbuf_csv, "Block_Bootstrapping_Phi_LowerCI,%s_in_%s,%f\n", mtd->levelNames->d[i + 1], mtd->levelNames->d[i], ci_lower);
+ ksprintf(&kbuf_csv, "Block_Bootstrapping_Phi_UpperCI,%s_in_%s,%f\n", mtd->levelNames->d[i + 1], mtd->levelNames->d[i], ci_upper);
-// spawnThreads_amovaBootstrap(mtd, blobs);
-// blobs->bootstraps->print_confidenceInterval(stderr);
-// }
+ ksprintf(&kbuf_table, "Phi(%s in %s): %f\n", mtd->levelNames->d[i + 1], mtd->levelNames->d[i], mean);
+ ksprintf(&kbuf_table, "Phi(%s in %s) SD: %f\n", mtd->levelNames->d[i + 1], mtd->levelNames->d[i], sd);
+ ksprintf(&kbuf_table, "Phi(%s in %s) Lower CI: %f\n", mtd->levelNames->d[i + 1], mtd->levelNames->d[i], ci_lower);
+ ksprintf(&kbuf_table, "Phi(%s in %s) Upper CI: %f\n", mtd->levelNames->d[i + 1], mtd->levelNames->d[i], ci_upper);
+ }
+ }
+ }
-// if (0 == doAmova(amova, dm, metadata)) {
-// fprintf(stderr, "\n[INFO]\t-> Finished running AMOVA");
-// if (0 < args->nBootstraps) {
-// fprintf(stderr, " for the original dataset.\n");
-// } else {
-// fprintf(stderr, ".\n");
-// }
+ amovaStruct_print_as_csv(amova, mtd, kbuf_csv.s);
+ if (kbuf_csv.l > 0) {
+ ks_free(&kbuf_csv);
+ }
-// if (args->printAmovaTable == 1) {
- // amova->print_as_table(stdout);
- //TODO make this to print file instead
-// }
+ if (args->printAmovaTable == 1) {
+ amovaStruct_print_as_table(amova, mtd);
+ }
return (amova);
}
diff --git a/amova.h b/amova.h
index 005cbe7..f980451 100644
--- a/amova.h
+++ b/amova.h
@@ -25,7 +25,6 @@ struct amovaStruct {
metadataStruct* metadata;
size_t nRuns;
- // size_t nPermutations; // TODO
/// @var df - array of degrees of freedom
/// @note size = nLevels
@@ -110,172 +109,11 @@ struct amovaStruct {
void amovaStruct_print_as_csv(amovaStruct* amv);
-inline amovaStruct* amovaStruct_init(metadataStruct* mtd, const int nAmovaRuns) {
+amovaStruct* amovaStruct_init(metadataStruct* mtd, const int nAmovaRuns);
- amovaStruct* ret = (amovaStruct*)malloc(sizeof(amovaStruct));
- ret->metadata = mtd;
+void amovaStruct_destroy(amovaStruct* amv);
- const size_t nLevels = (size_t)mtd->nLevels;
-
- const size_t nRuns = (size_t)nAmovaRuns;
- ret->nRuns = nRuns;
-
-
- const size_t nCmat = (nLevels * (nLevels + 1)) / 2;
- ret->vmat = NULL;
- ret->vmat = (double*)malloc((nCmat) * sizeof(double));
- ASSERT(ret->vmat != NULL);
- ret->cmat = NULL;
- ret->cmat = (double*)malloc((nCmat) * sizeof(double));
- ASSERT(ret->cmat != NULL);
- ret->lmat = NULL;
- ret->lmat = (double*)malloc((nCmat) * sizeof(double));
- ASSERT(ret->cmat != NULL);
- for (size_t i = 0; i < nCmat; ++i) {
- ret->cmat[i] = 0.0;
- ret->lmat[i] = 0.0;
- ret->vmat[i] = 0.0;
- }
-
-
- ret->df = NULL;
- ret->df = (int*)malloc((nLevels) * sizeof(int));
- ASSERT(ret->df != NULL);
-
- ret->df_total = 0;
-
- ret->ss = NULL;
- ret->ss = (double**)malloc((nRuns) * sizeof(double*));
- ASSERT(ret->ss != NULL);
-
- ret->ss_total = NULL;
- ret->ss_total = (double*)malloc((nRuns) * sizeof(double));
- ASSERT(ret->ss_total != NULL);
-
- ret->ssd = NULL;
- ret->ssd = (double**)malloc((nRuns) * sizeof(double*));
- ASSERT(ret->ssd != NULL);
-
- ret->ssd_total = NULL;
- ret->ssd_total = (double*)malloc((nRuns) * sizeof(double));
- ASSERT(ret->ssd_total != NULL);
-
- ret->msd = NULL;
- ret->msd = (double**)malloc((nRuns) * sizeof(double*));
- ASSERT(ret->msd != NULL);
-
- ret->msd_total = NULL;
- ret->msd_total = (double*)malloc((nRuns) * sizeof(double));
- ASSERT(ret->msd_total != NULL);
-
- ret->sigmasq = NULL;
- ret->sigmasq = (double**)malloc((nRuns) * sizeof(double*));
- ASSERT(ret->sigmasq != NULL);
-
- ret->sigmasq_total = NULL;
- ret->sigmasq_total = (double*)malloc((nRuns) * sizeof(double));
- ASSERT(ret->sigmasq_total != NULL);
-
- ret->phi_xt = NULL;
- ret->phi_xt = (double**)malloc((nRuns) * sizeof(double*));
- ASSERT(ret->phi_xt != NULL);
-
-
- ret->phi_xy = NULL;
- if (nLevels > 2) {
- ret->phi_xy = (double**)malloc((nRuns) * sizeof(double*));
- ASSERT(ret->phi_xy != NULL);
- }
-
- for (size_t i = 0;i < nRuns;++i) {
- ret->ss[i] = (double*)malloc((nLevels) * sizeof(double));
- ASSERT(ret->ss[i] != NULL);
-
- ret->ss_total[i] = 0.0;
-
- ret->ssd[i] = (double*)malloc((nLevels) * sizeof(double));
- ASSERT(ret->ssd[i] != NULL);
-
- ret->ssd_total[i] = 0.0;
-
- ret->msd[i] = (double*)malloc((nLevels) * sizeof(double));
- ASSERT(ret->msd[i] != NULL);
-
- ret->msd_total[i] = 0.0;
-
- ret->sigmasq[i] = (double*)malloc((nLevels) * sizeof(double));
- ASSERT(ret->sigmasq[i] != NULL);
-
- ret->sigmasq_total[i] = 0.0;
-
- ret->phi_xt[i] = (double*)malloc((nLevels - 1) * sizeof(double));
- ASSERT(ret->phi_xt[i] != NULL);
-
- if (ret->phi_xy != NULL) {
- ret->phi_xy[i] = NULL;
- ret->phi_xy[i] = (double*)malloc((nLevels - 2) * sizeof(double));
- ASSERT(ret->phi_xy[i] != NULL);
- for (size_t j = 0;j < nLevels - 2;++j) {
- ret->phi_xy[i][j] = 0.0;
- }
- }
-
-
- for (size_t j = 0;j < nLevels;++j) {
- ret->ss[i][j] = 0.0;
- ret->ssd[i][j] = 0.0;
- ret->msd[i][j] = 0.0;
- ret->sigmasq[i][j] = 0.0;
- if (j < nLevels - 1) {
- ret->phi_xt[i][j] = 0.0;
- }
- }
-
- }
-
- return(ret);
-}
-
-
-inline void amovaStruct_destroy(amovaStruct* amv) {
-
- amv->metadata = NULL;
-
- for (size_t i = 0;i < amv->nRuns;++i) {
- FREE(amv->ss[i]);
- FREE(amv->ssd[i]);
- FREE(amv->msd[i]);
- FREE(amv->sigmasq[i]);
- FREE(amv->phi_xt[i]);
- if (amv->phi_xy != NULL) {
- FREE(amv->phi_xy[i]);
- }
- }
-
- FREE(amv->df);
- FREE(amv->ss);
- FREE(amv->ss_total);
- FREE(amv->ssd);
- FREE(amv->ssd_total);
- FREE(amv->msd);
- FREE(amv->msd_total);
- FREE(amv->vmat);
- FREE(amv->cmat);
- FREE(amv->lmat);
- FREE(amv->sigmasq);
- FREE(amv->sigmasq_total);
- FREE(amv->phi_xt);
- if (amv->phi_xy != NULL) {
- FREE(amv->phi_xy);
- }
-
- FREE(amv);
-
-}
-
-
-// amovaStruct* amovaStruct_get(distanceMatrixStruct* dm, metadataStruct* mtd, blobStruct* blob);
-amovaStruct* amovaStruct_get(paramStruct* pars, metadataStruct* mtd, blobStruct* blobs);
+amovaStruct* amovaStruct_get(paramStruct* pars, metadataStruct* mtd);
diff --git a/argStruct.cpp b/argStruct.cpp
index 9b125e0..c8de654 100644
--- a/argStruct.cpp
+++ b/argStruct.cpp
@@ -61,9 +61,6 @@ argStruct* argStruct_get(int argc, char** argv) {
//
// -doMajorMinor <int> : get major and minor alleles for each site
-//TODO
- // e.g. `-doAMOVA` in lut1 -> "args->doAMOVA" in lut2 ->integer in lut3
- // so if `doAMOVA` is used, use multilayer luts to determine the type of required val etc
if (strcasecmp("-doUnitTests", arv) == 0) {
args->doUnitTests = atoi(val);
return(args);
@@ -82,6 +79,10 @@ argStruct* argStruct_get(int argc, char** argv) {
} else if (strcasecmp("-doDist", arv) == 0) {
args->doDist = atoi(val);
+ }else if(strcasecmp("--dm-method", arv) == 0){
+ args->dm_method = atoi(val);
+ }else if(strcasecmp("--dm-transform", arv) == 0){
+ args->dm_transform = atoi(val);
} else if (strcasecmp("-doIbd", arv) == 0) {
args->doIbd = atoi(val);
} else if (strcasecmp("-doMajorMinor", arv) == 0) {
@@ -157,7 +158,6 @@ argStruct* argStruct_get(int argc, char** argv) {
//
// TODO maybe use hypen style here --print-joint-geno-count-dist to be consistent with other commands
- // TODO maybe make <int> optional and choose a default one, most people won't care about this
else if ((strcasecmp("--printJointGenotypeCountMatrix", arv) == 0) || (strcasecmp("--printJGCD", arv) == 0) || (strcasecmp("-pJGCD", arv) == 0)) {
args->printJointGenotypeCountMatrix = atoi(val);
@@ -357,10 +357,6 @@ argStruct* argStruct_get(int argc, char** argv) {
//
// This file can be obtained using ANGSD maf files.
-
- else if (strcasecmp("-dev", arv) == 0)
- args->printDev = atoi(val);
-
else if (strcasecmp("--minInd", arv) == 0)
args->minInd = atoi(val);
else if (strcasecmp("--min-af", arv) == 0) {
@@ -404,6 +400,10 @@ argStruct* argStruct_get(int argc, char** argv) {
args->nBootstraps = (int)atof(val);
}
+ else if ((strcasecmp("--bootstrap-ci", arv) == 0)) {
+ args->bootstrap_ci = atof(val);
+ }
+
else if (strcasecmp("--maxEmIter", arv) == 0) {
args->maxEmIter = atoi(val);
@@ -440,9 +440,108 @@ argStruct* argStruct_get(int argc, char** argv) {
if (0 == args->nThreads) {
args->nThreads = 1;
}
+ if (NULL != args->in_vcf_fn) {
+ fprintf(stderr, "\n[INFO]\tFound input VCF file: %s\n", args->in_vcf_fn);
+ args->in_ft = args->in_ft | ARG_INTPLUS_INPUT_VCF;
+ }
+
+ if (NULL != args->in_dm_fn) {
+ fprintf(stderr, "\n[INFO]\tFound input distance matrix file: %s\n", args->in_dm_fn);
+ args->in_ft = args->in_ft | ARG_INTPLUS_INPUT_DM;
+ }
+
+ if (NULL != args->in_dxy_fn) {
+ //TODO ??
+ fprintf(stderr, "\n[INFO]\tFound input dxy file: %s\n", args->in_dxy_fn);
+ args->in_ft = args->in_ft | ARG_INTPLUS_INPUT_DXY;
+ }
+
+ if (NULL != args->in_mtd_fn) {
+ fprintf(stderr, "\n[INFO]\tFound input metadata file: %s\n", args->in_mtd_fn);
+ args->in_ft = args->in_ft | ARG_INTPLUS_INPUT_METADATA;
+ }
+
+ if (NULL != args->in_majorminor_fn) {
+ fprintf(stderr, "\n[INFO]\tFound input major/minor alleles file: %s\n", args->in_majorminor_fn);
+ args->in_ft = args->in_ft | ARG_INTPLUS_INPUT_MAJORMINOR;
+ }
+
+ if (NULL != args->in_ancder_fn) {
+ fprintf(stderr, "\n[INFO]\tFound input ancestral/derived alleles file: %s\n", args->in_ancder_fn);
+ args->in_ft = args->in_ft | ARG_INTPLUS_INPUT_ANCDER;
+ }
+
+ if (NULL != args->in_blocks_bed_fn) {
+ fprintf(stderr, "\n[INFO]\tFound input blocks BED file: %s\n", args->in_blocks_bed_fn);
+ args->in_ft = args->in_ft | ARG_INTPLUS_INPUT_BLOCKS;
+ }
+
+ if (NULL != args->in_blocks_tab_fn) {
+ fprintf(stderr, "\n[INFO]\tFound input blocks TSV file: %s\n", args->in_blocks_tab_fn);
+ args->in_ft = args->in_ft | ARG_INTPLUS_INPUT_BLOCKS;
+ }
+
+ if (NULL != args->in_regions_bed_fn) {
+ fprintf(stderr, "\n[INFO]\tFound input regions BED file: %s\n", args->in_regions_bed_fn);
+ args->in_ft = args->in_ft | ARG_INTPLUS_INPUT_REGIONS;
+ }
+
+ if (NULL != args->in_regions_tab_fn) {
+ //TODO test
+ fprintf(stderr, "\n[INFO]\tFound input regions TSV file: %s\n", args->in_regions_tab_fn);
+ args->in_ft = args->in_ft | ARG_INTPLUS_INPUT_REGIONS;
+ }
+
+ if ((PROGRAM_HAS_INPUT_DM || PROGRAM_HAS_INPUT_MULTIDM) && PROGRAM_HAS_INPUT_VCF) {
+ ERROR("Both VCF and distance matrix input are provided. Only one type of input is allowed at the same time.");
+ }
+
+ if (PROGRAM_HAS_INPUT_DM) {
+ if (args->blockSize != 0) {
+ ERROR("-blockSize is not supported for distance matrix input.");
+ }
+ if (args->doEM) {
+ ERROR("-doEM is not available for distance matrix input.");
+ }
+ if (args->doJGTM) {
+ ERROR("-doJGTM is not available for distance matrix input.");
+ }
+ }
+
+ if (PROGRAM_HAS_INPUT_METADATA && (!PROGRAM_NEEDS_METADATA)) {
+ WARN("Metadata file is provided but no analysis requires it; will ignore metadata file.");
+ }
+
+ if ((!(PROGRAM_HAS_INPUT_METADATA)) && PROGRAM_NEEDS_METADATA) {
+ ERROR("Metadata file is not provided but required for the analysis.");
+ }
+
+ if (PROGRAM_WILL_PERFORM_BLOCK_BOOTSTRAPPING) {
+
+ if (args->nBootstraps != -1) {
+ CHECK_ARG_INTERVAL_INT(args->nBootstraps, 1, 100000, "-nb/--nBootstraps");
+ }
+
+ if(-1.0==args->bootstrap_ci){
+ args->bootstrap_ci = 0.95;
+ LOG("Bootstrap confidence interval is not set, setting to default value %f.", args->bootstrap_ci);
+
+ }else{
+ CHECK_ARG_INTERVAL_II_DBL(args->bootstrap_ci, 0.0, 1.0, "--bootstrap-ci");
+ }
+
+ if (args->blockSize == 0 && args->in_blocks_tab_fn == NULL && args->in_blocks_bed_fn == NULL) {
+ ERROR("Block bootstrapping is enabled but no block specification is provided. Please provide block size or block file.");
+ }
+ if (args->nBootstraps == -1) {
+ ERROR("Block bootstrapping is enabled but the number of bootstraps is not set. Please set the number of bootstraps using -nb/--nBootstraps.");
+ }
+ }
- CHECK_ARG_INTERVAL_INT(args->nBootstraps, 0, 100000, "-nb/--nBootstraps");
+ if (PROGRAM_HAS_INPUT_METADATA && (!PROGRAM_NEEDS_METADATA)) {
+ ERROR("Metadata file is provided but none of the specified analyses use it. Please remove the metadata file or set the correct analysis.");
+ }
args->check_arg_dependencies();
@@ -463,18 +562,12 @@ void argStruct::check_arg_dependencies() {
ERROR("minInd is set to %d. Minimum value allowed for minInd is 2.", minInd);
}
- if (in_vcf_fn == NULL && in_dm_fn == NULL) {
- fprintf(stderr, "\n[ERROR] Must supply either --in-vcf <VCF_file> or --in-dm <Distance_matrix_file>.\n");
- exit(1);
- } else if (in_vcf_fn != NULL && in_dm_fn != NULL) {
- fprintf(stderr, "\n[ERROR] Cannot use --in-vcf %s and --in-dm %s at the same time.\n", in_vcf_fn, in_dm_fn);
- exit(1);
- }
-
if (printDistanceMatrix != 0) {
fprintf(stderr, "\n[INFO]\t-> -printMatrix %d; will print distance matrix\n", printDistanceMatrix);
}
+
+if(PROGRAM_WILL_USE_RNG){
if (seed == -1) {
seed = time(NULL);
srand48(seed);
@@ -484,6 +577,7 @@ void argStruct::check_arg_dependencies() {
srand48(seed);
LOG("Seed is set to: %d.\n", seed);
}
+}
if (in_dm_fn != NULL) {
if (doEM != 0) {
@@ -503,16 +597,33 @@ void argStruct::check_arg_dependencies() {
// default: 0
//
// 0: do not estimate distance matrix
- //TODO
- // 1: Dij
- // 2:
- // 3:
+ // 1: calculate the pairwise distance matrix using the method defined via --distance-method
+ // 2: read the distance matrix from the file defined via --in-dm
if (0 == doDist) {
//
} else if (1 == doDist) {
LOG("-doDist %d; will estimate pairwise distance matrix using Dij.", doDist);
+ // if dm_method dm_transform not defined, set to default and inform user
+ if(-1==dm_method){
+ dm_method = DMAT_METHOD_DIJ;
+ LOG("Distance matrix method is not set, setting to default value %d (Dij).", dm_method);
+ }else{
+ CHECK_ARG_INTERVAL_INT(dm_method, 0, 9, "--dm-method");
+ }
+ if(-1==dm_transform){
+ dm_transform = DMAT_TRANSFORM_NONE;
+ LOG("Distance matrix transform is not set, setting to default value %d (none).", dm_transform);
+ }else{
+ CHECK_ARG_INTERVAL_INT(dm_transform, 0, 1, "--dm-transform");
+ }
+
+ } else if (2 == doDist) {
+ if (NULL == in_dm_fn) {
+ ERROR("-doDist %d requires a distance matrix file (--in-dm <file>).", doDist);
+ }
+ LOG("-doDist %d; will read the distance matrix from the file %s.", doDist, in_dm_fn);
} else {
ERROR("-doDist %d is not a valid option.", doDist);
}
@@ -531,25 +642,30 @@ void argStruct::check_arg_dependencies() {
// - metadata
// - distance matrix
- if (0 == doAMOVA) {
- //
- if (NULL != in_mtd_fn) {
- WARN("-m/--metadata is provided but no analysis requires it; will ignore -m/--metadata.");
- }
- if (NULL != formula) {
- WARN("-f/--formula is provided but no analysis requires it; will ignore -f/--formula.");
- }
+ if (doAMOVA) {
+ if (ARG_DOAMOVA_SINGLERUN == doAMOVA) {
+ IO::requireArgStr(formula, "--formula/-f", "-doAMOVA 1");
+ IO::requireArgFile(in_mtd_fn, "--metadata/-m", "-doAMOVA 1");
- } else if (1 == doAMOVA) {
- IO::requireArgStr(formula, "--formula/-f", "-doAMOVA 1");
- IO::requireArgFile(in_mtd_fn, "--metadata/-m", "-doAMOVA 1");
+ if (0 == doDist && NULL == in_dm_fn) {
+ ERROR("-doAMOVA %d requires either (1) a method to calculate the pairwise distance matrix (-doDist <int>) or (2) a distance matrix file (--in-dm <file>).", doAMOVA);
+ }
- if (0 == doDist && NULL == in_dm_fn) {
- ERROR("-doAMOVA %d requires either (1) a method to calculate the pairwise distance matrix (-doDist <int>) or (2) a distance matrix file (--in-dm <file>).", doAMOVA);
- }
+ } else if (ARG_DOAMOVA_BOOTSTRAP == doAMOVA) {
+ IO::requireArgStr(formula, "--formula/-f", "-doAMOVA 1");
+ IO::requireArgFile(in_mtd_fn, "--metadata/-m", "-doAMOVA 1");
- } else {
- ERROR("-doAMOVA %d is not a valid option.", doAMOVA);
+ if (0 == doDist && NULL == in_dm_fn) {
+ ERROR("-doAMOVA %d requires either (1) a method to calculate the pairwise distance matrix (-doDist <int>) or (2) a distance matrix file (--in-dm <file>).", doAMOVA);
+ }
+ if (!PROGRAM_WILL_USE_BCF_FMT_GL) {
+ ERROR("-doAMOVA %d requires genotype likelihoods in the input VCF file (--bcf-src %d).", doAMOVA, ARG_INTPLUS_BCFSRC_FMT_GL);
+ }
+ } else if (ARG_DOAMOVA_PERMTEST) {
+ NEVER;//TODO
+ } else {
+ ERROR("-doAMOVA %d is not a valid option.", doAMOVA);
+ }
}
if (0 == doEM) {
@@ -580,12 +696,12 @@ void argStruct::check_arg_dependencies() {
}
//----------------------------------------------------------------------------------//
- // -doDxy
+// -doDxy
// default: 0
// 0: do not estimate dxy
// 1: estimate dxy from the distance matrix for all groups in all hierarchical levels defined in the metadata file (requires: method to obtain distance matrix, metadata file)
- // : (doDist 1,2 or in_ft == IN_DM), (--metadata-file <METADATA_FILE> or doDxyStr)
+ // : (doDist 1,2 or in_ft == IN_DM), (--metadata-file <METADATA_FILE>)
if (0 == doDxy) {
//
@@ -612,21 +728,10 @@ void argStruct::check_arg_dependencies() {
// ERROR("-doPhylo %d requires a distance matrix (either -doDist <int> or --in-dm <file>).", doPhylo);
// }
} else if (2 == doPhylo) {
- if (0 == doDxy && NULL == in_dxy_fn) {
- ERROR("-doPhylo %d requires a dxy matrix (either -doDxy <int> or --in-dxy <file>).", doPhylo);
- }
} else {
ERROR("-doPhylo %d is not a valid option.", doPhylo);
}
-
-
- if (args->rmInvarSites) {
- } else {
- //TODO req for some?
- // ERROR("No analyses including invar sites are implemented yet"); //TODO
- }
-
if (args->bcfSrc & ARG_INTPLUS_BCFSRC_FMT_GL) {
LOG("%s INT+ argument value contains %d. Program will use the GL tag from input VCF file.", "--bcf-src", ARG_INTPLUS_BCFSRC_FMT_GL);
}
@@ -634,7 +739,6 @@ void argStruct::check_arg_dependencies() {
LOG("%s INT+ argument value contains %d. Program will use the GT tag from input VCF file.", "--bcf-src", ARG_INTPLUS_BCFSRC_FMT_GT);
}
-
// -------------------------------
// doMajorMinor
if (args->doMajorMinor == ARG_DOMAJORMINOR_BCF_REFALT1) {
@@ -658,7 +762,7 @@ void argStruct::check_arg_dependencies() {
// - em ngl=10
// - input is not vcf
if ((args->in_vcf_fn != NULL) && (args->doEM != (ARG_DOEM_10GL))) {
- if (args->doMajorMinor == ARG_DOMAJORMINOR_NONE) {
+ if (args->doMajorMinor == ARG_DOMAJORMINOR_UNSET) {
ERROR("Program requires major/minor alleles for the specified analyses. Please provide a method to obtain major/minor alleles using -doMajorMinor <int>.");
}
}
@@ -666,17 +770,6 @@ void argStruct::check_arg_dependencies() {
// -------------------------------
-
-
- // else if (doPhylo == 1 && doDist == 0 && in_dm_fn == NULL) {
- // fprintf(stderr, "\n[ERROR]\t-> -doPhylo %i requires -doDist 1 or --in-dm <file>.", doPhylo);
- // exit(1);
- // }
- // if (doPhylo == 2 && doDxy == 0 && in_dxy_fn == NULL) {
- // fprintf(stderr, "\n[ERROR]\t-> -doPhylo %i requires -doDxy 1 or --in-dxy <file>.", doPhylo);
- // exit(1);
- // }
-
// TODO using regions with block definitions
// TODO handle empty blocks
if (blockSize > 0 && in_blocks_tab_fn != NULL) {
@@ -708,13 +801,16 @@ void argStruct::check_arg_dependencies() {
}
}
- // TODO block bootstrapping cannot be used with gt data (bcfSrc==ARG_INTPLUS_BCFSRC_FMT_GT)
-
-
if (PROGRAM_WILL_USE_BCF_FMT_GL && PROGRAM_WILL_USE_BCF_FMT_GT) {
ERROR("Program cannot use both GL and GT data at the same time.");
}
+ if(PROGRAM_WILL_USE_BCF_FMT_GT){
+ if(PROGRAM_WILL_PERFORM_BLOCK_BOOTSTRAPPING){
+ ERROR("Program cannot use GT data for block bootstrapping.");
+ }
+ }
+
}
@@ -769,12 +865,6 @@ void argStruct_destroy(argStruct* args) {
FREE(args->formula);
}
- if (args->keyCols != NULL) {
- FREE(args->keyCols);
- }
- if (args->doDxyStr != NULL) {
- FREE(args->doDxyStr);
- }
delete args;
}
\ No newline at end of file
diff --git a/argStruct.h b/argStruct.h
index acb1e14..c043ed9 100644
--- a/argStruct.h
+++ b/argStruct.h
@@ -2,6 +2,7 @@
#define __ARG_STRUCT__
#include "shared.h"
+#include "dmat.h"
typedef struct argStruct argStruct;
@@ -16,7 +17,6 @@ extern argStruct* args;
* @field *in_dm_fn input distance matrix filename
* @field *in_mtd_fn input metadata filename
* @field *in_blb_fn input block bed filename
- * @field *in_dxy_fn input dxy filename
*
* @field *out_fnp pointer to output file prefix [angsdput]
*
@@ -70,10 +70,6 @@ extern argStruct* args;
* ind4,Pop2,Reg2
*
*
- * @field keyCols defines the index(es) (1-based) of the
- * column(s) in Metadata file 'in_mtd_fn'
- * to be used to define hierarchical stratification levels
- *
* @field doAMOVA [0]
* 1 use 10 genotype likelihoods (GL)
* 2 use genotypes (GT) (NOTE: Only for benchmark purposes)
@@ -154,6 +150,8 @@ struct argStruct {
int doJGTM = 0;
int doDist = 0;
+ int dm_method = -1;
+ int dm_transform = -1;
int doEM = 0;
@@ -202,15 +200,15 @@ struct argStruct {
int nThreads = 1;
int rmInvarSites = 0;
int seed = -1;
- int nBootstraps = 0;
+ int nBootstraps = -1;
+ double bootstrap_ci=-1.0;
void check_arg_dependencies();
- // TODO check below
- char* doDxyStr = NULL; // TODO maybe not needed
int windowSize = 0;
- int* keyCols = NULL;
- int printDev = 0;
+
+ // input file type
+ uint8_t in_ft = 0;
};
diff --git a/bootstrap.cpp b/bootstrap.cpp
index 016c929..5715bef 100644
--- a/bootstrap.cpp
+++ b/bootstrap.cpp
@@ -1,361 +1,489 @@
-#include "amova.h"
#include "bootstrap.h"
#include <algorithm>
-// TODO allow using whole contigs as blocks
-// TODO make it work with region and regions, we need to get ncontigs from the region filtered vcf
+// TODO add test case
-blobStruct* blobStruct_get(vcfData* vcf, paramStruct* pars) {
- fprintf(stderr, "\n\t-> --nBootstraps %d is set, will perform %d bootstraps for AMOVA significance testing.\n", args->nBootstraps, args->nBootstraps);
+void bblocks_destroy(bblocks_t* bblocks) {
+ for (size_t i = 0;i < args->nBootstraps;++i) {
+ FREE(bblocks->rblocks[i]);
+ }
+ FREE(bblocks->rblocks);
+ FREE(bblocks->nsites_per_block);
+ FREE(bblocks->block_start_pos);
+ FREE(bblocks->block_end_pos);
+ FREE(bblocks->block_contig);
+ FREE(bblocks->block_start_siteidx);
+ strArray_destroy(bblocks->contig_names);
+ FREE(bblocks);
+ return;
+}
- blobStruct* blob = NULL;
- if (args->blockSize != 0) {
- fprintf(stderr, "\n\t-> blockSize is set, will perform block bootstrapping with blocks of size %d.\n", args->blockSize);
- blob = blobStruct_populate_blocks_withSize(vcf);
- } else if (args->in_blocks_tab_fn != NULL) {
- blob = blobStruct_read_tab(args->in_blocks_tab_fn);
- } else if (args->in_blocks_bed_fn != NULL) {
- blob = blobStruct_read_bed(args->in_blocks_bed_fn);
- } else {
- ERROR("--nBootstraps %d requires --block-size, --in_blocks_tab_fn or --in_blocks_bed_fn to be set.", args->nBootstraps);
- }
+void bblocks_sample_with_replacement(bblocks_t* bblocks) {
+ DEVASSERT(bblocks != NULL);
- blob->bootstraps = bootstrapDataset_get(vcf, pars, blob);
+ const size_t nReps = args->nBootstraps;
+ const int nBlocks = bblocks->n_blocks;
- blob->print();
- blob->bootstraps->print();
+ for (size_t rep = 0;rep < nReps;++rep) {
+ for (size_t block = 0;block < nBlocks;++block) {
+ bblocks->rblocks[rep][block] = (int)(nBlocks * drand48());
+ }
+ }
- return blob;
}
+void bblocks_print_bootstrap_samples(bblocks_t* bblocks) {
-bootstrapDataset* bootstrapDataset_get(vcfData* vcfd, paramStruct* pars, blobStruct* blobSt) {
- bootstrapDataset* bootstraps = new bootstrapDataset(pars, args->nBootstraps, blobSt->nBlocks);
+ kstring_t* kbuf= kbuf_init();
- int rblock = -1;
+ ksprintf(kbuf, "Replicate,BlockNo,BlockID\n");
- for (int rep = 0; rep < args->nBootstraps; ++rep) {
- for (int block = 0; block < blobSt->nBlocks; ++block) {
- rblock = sample_with_replacement(blobSt->nBlocks);
- bootstraps->replicates[rep]->rBlocks[block] = rblock;
+ for (size_t rep = 0;rep < args->nBootstraps;++rep) {
+ for (size_t block = 0;block < bblocks->n_blocks;++block) {
+ ksprintf(kbuf, "%ld,%ld,%ld\n", rep, block, bblocks->rblocks[rep][block]);
}
}
- return bootstraps;
-}
+ fprintf(stdout,"%s", kbuf->s);
+ kbuf_destroy(kbuf);
-// / @brief sample an index from a set of size n
-/// @param n
-/// @return
-int sample_with_replacement(int n) {
- ASSERT(n > 0);
- return (int)(n * drand48());
-}
-void blobStruct::_print() {
- for (int i = 0; i < nBlocks; ++i) {
- fprintf(stderr, "%s\t%d\t%d\t%d\n", blocks[i]->chr, blocks[i]->start, blocks[i]->end, blocks[i]->len);
- }
+ return;
}
-blobStruct::~blobStruct() {
- delete(bootstraps);
- for (int i = 0; i < nBlocks; ++i) {
- FREE(blocks[i]->chr);
- FREE(blocks[i]);
- FREE(blockPtrs[i]);
- }
- FREE(blocks);
- FREE(blockPtrs);
-}
-void blobStruct::addBlock() {
- ++nBlocks;
- if (nBlocks > 1) {
- REALLOC(blockStruct**, blocks, nBlocks);
- REALLOC(int**, this->blockPtrs, this->nBlocks);
+// blocks tab file = 1-based, [start, end]
+// internal representation = 0-based, [start, end)
+// conversion from internal to blocks tab: start+1,end
+void bblocks_print_blocks_tab(bblocks_t* bblocks) {
+ LOGADD("(%s) Writing blocks tab file: %s", "-printBlocksTab", outFiles->out_blockstab_fs->fn);
- } else if (nBlocks == 1) {
- blocks = (blockStruct**)malloc(nBlocks * sizeof(blockStruct*));
- blockPtrs = (int**)malloc(nBlocks * sizeof(int*));
- } else {
- NEVER;
+ outFiles->out_blockstab_fs->kbuf = kbuf_init();
+
+ size_t ci;
+ for (size_t bi = 0;bi < bblocks->n_blocks;++bi) {
+
+ ci = bblocks->block_contig[bi];
+
+ ksprintf(outFiles->out_blockstab_fs->kbuf, "%s\t%ld\t%ld\n", bblocks->contig_names->d[ci], bblocks->block_start_pos[bi] + 1, bblocks->block_end_pos[bi]);
}
- blocks[nBlocks - 1] = (blockStruct*)malloc(sizeof(blockStruct));
- blockPtrs[nBlocks - 1] = (int*)malloc(sizeof(int));
+ outFiles->out_blockstab_fs->kbuf_write();
+ return;
}
+void bblocks_generate_blocks_with_size(bblocks_t* bblocks, vcfData* vcfd, paramStruct* pars, const uint64_t blockSize) {
-// - 1-based
-// - [start:included, end:included]
-blobStruct* blobStruct_read_tab(const char* fn) {
- FILE* fp = IO::getFile(fn, "r");
- char* firstLine = IO::readFile::getFirstLine(fp);
- int nCols = IO::inspectFile::count_nCols(firstLine, "\t");
- if (nCols != 3) {
- ERROR("Blocks tab file must have 3 columns. Found %d columns.", nCols);
- }
+ BEGIN_LOGSECTION_MSG("Generating blocks for block bootstrapping");
- ASSERT(fseek(fp, 0, SEEK_SET) == 0);
- int nBlocks = 0;
+ LOGADD("(%s) Targeted block size: %d", "--block-size", blockSize);
- char* tok = NULL;
- char chr[100];
- char start[100];
- char end[100];
- blobStruct* blob = new blobStruct();
+ const size_t nInd = pars->names->len;
+ const size_t nContigs = vcfd->nContigs;
- while (EOF != fscanf(fp, "%s\t%s\t%s", chr, start, end)) {
- blob->addBlock();
+ const int targeted_blockSize = args->blockSize;
+ int current_blockSize = targeted_blockSize;
- ASSERTM(strIsNumeric(start), "Start position must be numeric.");
- int start_int = atoi(start);
- ASSERTM(strIsNumeric(end), "End position must be numeric.");
- int end_int = atoi(end);
+ bblocks->n_contigs = nContigs;
+ bblocks->contig_names = strArray_alloc(nContigs);
- IO::validateString(chr);
- blob->blocks[nBlocks]->chr = strdup(chr);
+ uint64_t prev_nBlocks;
+ uint64_t totnBlocks = 0;
+ for (size_t ci = 0;ci < nContigs;++ci) {
- // tab file positions are 1-based, but blockStruct positions are 0-based
- // so -1 to make it 0-based
+ const uint64_t contigSize = vcfd->hdr->id[BCF_DT_CTG][ci].val->info[0];
+ const char* contigName = vcfd->hdr->id[BCF_DT_CTG][ci].key;
- // TODO make it work with region and regions, we need to get ncontigs from the region filtered vcf
- // both tab file start and blockStruct start are inclusive
- blob->blocks[nBlocks]->start = start_int - 1;
- ASSERTM(start_int > 0, "Start position must be greater than 0. Note: Tab files have 1-based indexing with [start:inclusive, end:inclusive].");
+ bblocks->contig_names->add(contigName);
+ LOGADD("Found contig %s with size %d", contigName, contigSize);
- // tab file end is inclusive, but blockStruct end is exclusive
- // +1 to make it exclusive
- // -1+1 = 0
- blob->blocks[nBlocks]->end = end_int;
+ if (targeted_blockSize > contigSize) {
+ LOGADD("Size of the contig %s (%d) is smaller than the targeted block size (%d). Setting the block size to the size of the contig.", contigName, contigSize, targeted_blockSize);
+ current_blockSize = contigSize;
+ } else {
+ current_blockSize = targeted_blockSize;
+ }
+
+ // +1 to account for the remainder (last block)
+ const int nBlocks_perContig = (contigSize % current_blockSize == 0) ? (contigSize / current_blockSize) : ((contigSize / current_blockSize) + 1);
+ DEVASSERT(nBlocks_perContig > 0);
- ASSERTM(end_int > 0, "End position must be greater than 0. Note: Tab files have 1-based indexing with [start:inclusive, end:inclusive].");
+ LOGADD("Generating %d block%c of size %d for contig %s", nBlocks_perContig, (nBlocks_perContig > 1) ? 's' : '\0', current_blockSize, contigName);
- blob->blocks[nBlocks]->len = blob->blocks[nBlocks]->end - blob->blocks[nBlocks]->start;
- ASSERTM(blob->blocks[nBlocks]->len > 0, "Block length must be greater than 0.");
+ prev_nBlocks = totnBlocks;
+ totnBlocks += nBlocks_perContig;
+
+ if (ci == 0) {
+ bblocks->block_start_pos = (size_t*)malloc(totnBlocks * sizeof(size_t));
+ ASSERT(bblocks->block_start_pos != NULL);
+ bblocks->block_end_pos = (size_t*)malloc(totnBlocks * sizeof(size_t));
+ ASSERT(bblocks->block_end_pos != NULL);
+ bblocks->block_contig = (size_t*)malloc(totnBlocks * sizeof(size_t));
+ ASSERT(bblocks->block_contig != NULL);
+ bblocks->block_start_siteidx = (size_t*)malloc(totnBlocks * sizeof(size_t));
+ ASSERT(bblocks->block_start_siteidx != NULL);
+ for(size_t i = 0;i < totnBlocks;++i) {
+ bblocks->block_start_pos[i] = 0;
+ bblocks->block_end_pos[i] = 0;
+ bblocks->block_contig[i] = 0;
+ bblocks->block_start_siteidx[i] = 0;
+ }
+
+ } else {
+ size_t* tmp = NULL;
+ tmp = (size_t*)realloc(bblocks->block_start_pos, totnBlocks * sizeof(size_t));
+ ASSERT(tmp != NULL);
+ bblocks->block_start_pos = tmp;
+ tmp = NULL;
+
+ tmp = (size_t*)realloc(bblocks->block_end_pos, totnBlocks * sizeof(size_t));
+ ASSERT(tmp != NULL);
+ bblocks->block_end_pos = tmp;
+ tmp = NULL;
+
+ tmp = (size_t*)realloc(bblocks->block_contig, totnBlocks * sizeof(size_t));
+ ASSERT(tmp != NULL);
+ bblocks->block_contig =tmp;
+ tmp = NULL;
+
+ tmp= (size_t*)realloc(bblocks->block_start_siteidx, totnBlocks * sizeof(size_t));
+ ASSERT(tmp != NULL);
+ bblocks->block_start_siteidx = tmp;
+ tmp = NULL;
+
+ for(size_t i = prev_nBlocks;i < totnBlocks;++i) {
+ bblocks->block_start_pos[i] = 0;
+ bblocks->block_end_pos[i] = 0;
+ bblocks->block_contig[i] = 0;
+ bblocks->block_start_siteidx[i] = 0;
+ }
+
+ }
+
+ // -> set
+ for (size_t bi = 0;bi < nBlocks_perContig;++bi) {
+ bblocks->block_start_pos[bi + prev_nBlocks] = bi * current_blockSize;
+ bblocks->block_end_pos[bi + prev_nBlocks] = ((bi + 1) * current_blockSize);
+ bblocks->block_contig[bi + prev_nBlocks] = ci;
+ }
+
+ // <- set
- ++nBlocks;
- // fprintf(stderr, "%s\t%d\t%d\n", chr, start_int, end_int);
}
- ASSERT(blob->nBlocks == nBlocks);
+ if (totnBlocks == 1) {
+ ERROR("Cannot perform block bootstrapping with only one block. Please decrease the block size and try again.");
+ }
- FREE(firstLine);
- FREE(tok);
- FCLOSE(fp);
+ bblocks->n_blocks = totnBlocks;
+ bblocks->n_ind = nInd;
- return blob;
+ bblocks->rblocks = (size_t**)malloc(args->nBootstraps * sizeof(size_t*));
+ ASSERT(bblocks->rblocks != NULL);
+ for (size_t i = 0;i < args->nBootstraps;++i) {
+ bblocks->rblocks[i] = (size_t*)malloc(totnBlocks * sizeof(size_t));
+ ASSERT(bblocks->rblocks[i] != NULL);
+ for (size_t j = 0;j < totnBlocks;++j) {
+ bblocks->rblocks[i][j] = 0;
+ }
+ }
+
+ bblocks->nsites_per_block = (uint64_t*)malloc(totnBlocks * sizeof(uint64_t));
+ ASSERT(bblocks->nsites_per_block != NULL);
+ for (size_t i = 0;i < totnBlocks;++i) {
+ bblocks->nsites_per_block[i] = 0;
+ }
+
+ return;
}
-// - 0-based
-// - [start:included, end:excluded)
-blobStruct* blobStruct_read_bed(const char* fn) {
+
+// blocks tab file = 1-based, [start, end]
+// internal representation = 0-based, [start, end)
+void bblocks_read_tab(bblocks_t* bblocks, const char* fn, vcfData* vcfd, paramStruct* pars) {
+
FILE* fp = IO::getFile(fn, "r");
char* firstLine = IO::readFile::getFirstLine(fp);
int nCols = IO::inspectFile::count_nCols(firstLine, "\t");
if (nCols != 3) {
- ERROR("Blocks bed file must have 3 columns. Found %d columns.", nCols);
+ ERROR("Blocks tab file must have 3 columns. Found %d columns.", nCols);
}
+ FREE(firstLine);
ASSERT(fseek(fp, 0, SEEK_SET) == 0);
- int nBlocks = 0;
char* tok = NULL;
- char chr[100];
- char start[100];
- char end[100];
- blobStruct* blob = new blobStruct();
-
- while (EOF != fscanf(fp, "%s\t%s\t%s", chr, start, end)) {
- blob->addBlock();
+ char chr[256];
+ int64_t start;
+ int64_t end;
- ASSERTM(strIsNumeric(start), "Start position must be numeric.");
- int start_int = atoi(start);
- ASSERTM(strIsNumeric(end), "End position must be numeric.");
- int end_int = atoi(end);
+ size_t nContigsFound = 0;
+ char lastChr[256];
+ bool contigChanged = false;
- IO::validateString(chr);
- blob->blocks[nBlocks]->chr = strdup(chr);
+ int nContigsVcf;
+ const char** vcfContigs = bcf_hdr_seqnames(vcfd->hdr, &nContigsVcf);
+ DEVASSERT(nContigsVcf == vcfd->nContigs);
- // both bed file and blockStruct positions are 0-based
+ size_t tmp_nBlocks = 512;
- // both tab file start and blockStruct start are inclusive
- blob->blocks[nBlocks]->start = start_int;
- ASSERTM(start_int > 0, "Start position must be greater than 0. Note: Bed files have 0-based indexing with [start:inclusive, end:exclusive).");
-
- // both tab file end and blockStruct end are exclusive
- blob->blocks[nBlocks]->end = end_int;
- ASSERTM(end_int > 0, "End position must be greater than 0. Note: Bed files have 0-based indexing with [start:inclusive, end:exclusive).");
-
- blob->blocks[nBlocks]->len = blob->blocks[nBlocks]->end - blob->blocks[nBlocks]->start;
- ASSERTM(blob->blocks[nBlocks]->len > 0, "Block length must be greater than 0.");
-
- ++nBlocks;
- // fprintf(stderr, "%s\t%d\t%d\n", chr, start_int, end_int);
+ bblocks->block_start_pos = (size_t*)malloc(tmp_nBlocks * sizeof(size_t));
+ ASSERT(bblocks->block_start_pos != NULL);
+ bblocks->block_end_pos = (size_t*)malloc(tmp_nBlocks * sizeof(size_t));
+ ASSERT(bblocks->block_end_pos != NULL);
+ for (size_t i = 0;i < tmp_nBlocks;++i) {
+ bblocks->block_start_pos[i] = 0;
+ bblocks->block_end_pos[i] = 0;
}
- ASSERT(blob->nBlocks == nBlocks);
+ bblocks->contig_names = strArray_init();
- FREE(firstLine);
- FREE(tok);
- FCLOSE(fp);
- return blob;
-}
+ size_t nBlocks = 0;
+ while (EOF != fscanf(fp, "%s\t%ld\t%ld", chr, &start, &end)) {
-blobStruct* blobStruct_populate_blocks_withSize(vcfData* vcf) {
-
- if (args->in_regions_tab_fn != NULL || args->in_regions_bed_fn != NULL || args->in_region != NULL) {
- ERROR("Block definitions cannot be used with region definitions, yet."); //TODO add feature
- }
+ // tab file positions are 1-based, but bblocks_t positions are 0-based
+ // so use -1 to make it 0-based
+ // both tab file start and bblocks_t start are inclusive
+ if (start <= 0) {
+ ERROR("Start position must be greater than 0. Note: Tab files have 1-based indexing with [start:inclusive, end:inclusive].");
+ }
+ bblocks->block_start_pos[nBlocks] = start - 1;
- const int targeted_blockSize = args->blockSize;
- int current_blockSize = targeted_blockSize;
+ // tab file end is inclusive, but bblocks_t end is exclusive
+ // +1 to make it exclusive
+ // -1+1 = 0 // so no change
+ if (end <= 0) {
+ ERROR("End position must be greater than 0. Note: Tab files have 1-based indexing with [start:inclusive, end:inclusive].");
+ }
+ bblocks->block_end_pos[nBlocks] = end;
- blobStruct* blob = new blobStruct();
+ if (end <= start) {
+ ERROR("End position must be greater than start position. Found start: %ld, end: %ld at line %ld", start, end, nBlocks);
+ }
+ // DEVPRINT("Found block %ld with size %ld at %s:%ld-%ld", nBlocks, end - start+1, chr, start, end);
- int totnBlocks = 0;
- for (int ci = 0; ci < vcf->nContigs; ci++) {
+ if (nContigsFound == 0) {
+ // first loop
+ strcpy(lastChr, chr);
+ ++nContigsFound;
+ contigChanged = true;
- const int contigSize = vcf->hdr->id[BCF_DT_CTG][ci].val->info[0];
- if (targeted_blockSize > contigSize) {
- LOG("Size of the contig %s (%d) is smaller than the targeted block size (%d). Setting the block size to the size of the contig.", vcf->hdr->id[BCF_DT_CTG][ci].key, contigSize, targeted_blockSize);
- current_blockSize = contigSize;
} else {
- current_blockSize = targeted_blockSize;
+ if (strcmp(lastChr, chr) == 0) {
+ contigChanged = false;
+ } else {
+ contigChanged = true;
+ strcpy(lastChr, chr);
+ ++nContigsFound;
+ }
}
- // +1 to account for the remainder (last block)
- const int nBlocks_perContig = (contigSize % current_blockSize == 0) ? contigSize / current_blockSize : (contigSize / current_blockSize) + 1;
- ASSERT(nBlocks_perContig > 0);
+ // if found new contig:
+ // -> check if the contig in input blocks file exists in the VCF file
+ if (contigChanged) {
+ size_t contig_i;
+ for (contig_i = 0; contig_i < nContigsVcf; ++contig_i) {
+ if (strcmp(chr, vcfContigs[contig_i]) == 0) {
+ break;
+ }
+ if (contig_i == nContigsVcf - 1) {
+ ERROR("Contig %s in input blocks file does not exist in the VCF file.", chr);
+ }
+ }
+ bblocks->contig_names->add(chr);
+ size_t* tmp = (size_t*)realloc(bblocks->block_contig, (nBlocks + 1) * sizeof(size_t));
+ ASSERT(tmp != NULL);
+ bblocks->block_contig = tmp;
+ bblocks->block_contig[nBlocks] = contig_i;
- LOG("Generating %d block%c of size %d for contig %s", nBlocks_perContig, (nBlocks_perContig > 1) ? 's' : '\0', current_blockSize, vcf->hdr->id[BCF_DT_CTG][ci].key);
+ }
- for (int bi = 0; bi < nBlocks_perContig; bi++) {
+ ++nBlocks;
+ if (tmp_nBlocks == nBlocks) {
+ tmp_nBlocks += 256;
+ bblocks->block_start_pos = (size_t*)realloc(bblocks->block_start_pos, tmp_nBlocks * sizeof(size_t));
+ ASSERT(bblocks->block_start_pos != NULL);
+ bblocks->block_end_pos = (size_t*)realloc(bblocks->block_end_pos, tmp_nBlocks * sizeof(size_t));
+ ASSERT(bblocks->block_end_pos != NULL);
+ for (size_t i = nBlocks;i < tmp_nBlocks;++i) {
+ bblocks->block_start_pos[i] = 0;
+ bblocks->block_end_pos[i] = 0;
+ }
+ }
+ }
- blob->addBlock();
- int blockStart = bi * current_blockSize;
+ bblocks->n_blocks = nBlocks;
+ bblocks->n_contigs = nContigsFound;
+ const size_t nInd = pars->names->len;
+ bblocks->n_ind = nInd;
+ bblocks->nsites_per_block = (uint64_t*)malloc(nBlocks * sizeof(uint64_t));
+ ASSERT(bblocks->nsites_per_block != NULL);
+ for (size_t i = 0;i < nBlocks;++i) {
+ bblocks->nsites_per_block[i] = 0;
+ }
- blob->blocks[bi]->start = blockStart;
+ FREE(tok);
+ FCLOSE(fp);
+ return;
+}
- if (bi == nBlocks_perContig - 1 && ci == vcf->nContigs - 1) {
- // last block of the last contig
- // make sure it ends at the end of the contig
- // since the size of the last block might be smaller than the block size
- blob->blocks[bi]->end = contigSize;
- } else {
- blob->blocks[bi]->end = blockStart + current_blockSize;
- }
- blob->blocks[bi]->chr = strdup(vcf->hdr->id[BCF_DT_CTG][ci].key);
- }
+void bblocks_read_bed(bblocks_t* bblocks, const char* fn, vcfData* vcfd, paramStruct* pars) {
- totnBlocks += nBlocks_perContig;
+ FILE* fp = IO::getFile(fn, "r");
+ char* firstLine = IO::readFile::getFirstLine(fp);
+ int nCols = IO::inspectFile::count_nCols(firstLine, "\t");
+ if (nCols != 3) {
+ ERROR("Blocks tab file must have 3 columns. Found %d columns.", nCols);
}
+ FREE(firstLine);
+ ASSERT(fseek(fp, 0, SEEK_SET) == 0);
- for (int ci = 0; ci < vcf->nContigs; ci++) {
- }
+ char* tok = NULL;
+ char chr[256];
+ int64_t start;
+ int64_t end;
- ASSERT(totnBlocks == blob->nBlocks);
- ASSERT(totnBlocks != 0);
+ size_t nContigsFound = 0;
+ char lastChr[256];
+ bool contigChanged = false;
+ int nContigsVcf;
+ const char** vcfContigs = bcf_hdr_seqnames(vcfd->hdr, &nContigsVcf);
+ DEVASSERT(nContigsVcf == vcfd->nContigs);
- return blob;
-}
+ size_t tmp_nBlocks = 256;
+
+ bblocks->block_start_pos = (size_t*)malloc(tmp_nBlocks * sizeof(size_t));
+ ASSERT(bblocks->block_start_pos != NULL);
+ bblocks->block_end_pos = (size_t*)malloc(tmp_nBlocks * sizeof(size_t));
+ ASSERT(bblocks->block_end_pos != NULL);
+ for (size_t i = 0;i < tmp_nBlocks;++i) {
+ bblocks->block_start_pos[i] = 0;
+ bblocks->block_end_pos[i] = 0;
+ }
+ bblocks->block_contig = (size_t*)malloc(sizeof(size_t));
+ ASSERT(bblocks->block_contig != NULL);
+ bblocks->contig_names = strArray_init();
-void bootstrapDataset::print() {
- if (1 != DEV)
- return;
+ size_t nBlocks = 0;
+ while (EOF != fscanf(fp, "%s\t%ld\t%ld", chr, &start, &end)) {
- fprintf(stderr, "\n[INFO]\t-> Writing bootstrap info file: %s\n", outFiles->out_v_bootstrapRep_fs->fn);
- outFiles->out_v_bootstrapRep_fs->kbuf = kbuf_init();
+ // both bed file and bblocks_t positions are 0-based
+ // both bed file start and bblocks_t start are inclusive
+ // both bed file end and bblocks_t end are exclusive
+ // so no change
- ksprintf(outFiles->out_v_bootstrapRep_fs->kbuf, "Replicate,ReplicateBlockIndex,BlockName\n");
+ bblocks->block_start_pos[nBlocks] = start;
- for (int b = 0; b < nReplicates; ++b) {
- for (int i = 0; i < nBlocks; ++i) {
- ksprintf(outFiles->out_v_bootstrapRep_fs->kbuf, "%d,%d,%d\n", b, i, replicates[b]->rBlocks[i]);
+ if (end <= 0) {
+ ERROR("End position must be greater than 0. Note: Tab files have 1-based indexing with [start:inclusive, end:inclusive].");
}
- }
- outFiles->out_v_bootstrapRep_fs->kbuf_write();
-}
+ bblocks->block_end_pos[nBlocks] = end;
-bootstrapDataset::bootstrapDataset(paramStruct* pars, int nBootstraps_, int nBlocks_) {
- nReplicates = nBootstraps_;
- nBlocks = nBlocks_;
- replicates = new bootstrapReplicate * [nReplicates];
- for (int b = 0; b < nReplicates; ++b) {
- replicates[b] = new bootstrapReplicate(nBlocks);
- }
-}
+ if (end <= start) {
+ ERROR("End position must be greater than start position. Found start: %ld, end: %ld at line %ld", start, end, nBlocks);
+ }
-bootstrapDataset::~bootstrapDataset() {
- for (int i = 0;i < nReplicates;++i) {
- delete[] replicates[i];
- }
- delete[] replicates;
- for (int i = 0;i < nPhiValues;++i) {
- FREE(phiValues[i]);
- }
-}
+ // DEVPRINT("Found block %ld with size %ld at %s:%ld-%ld", nBlocks, end - start, chr, start, end);
-void bootstrapDataset::print_confidenceInterval(FILE* fp) {
- double mean = 0.0;
- double sd = 0.0;
+ if (nContigsFound == 0) {
+ // first loop
+ strcpy(lastChr, chr);
+ ++nContigsFound;
+ contigChanged = true;
- double ci = 0.95;
- int n_lower = floor((double)((double)(1 - ci) / 2) * nReplicates);
- int n_upper = nReplicates - n_lower - 1;
+ } else {
+ if (strcmp(lastChr, chr) == 0) {
+ contigChanged = false;
+ } else {
+ contigChanged = true;
+ strcpy(lastChr, chr);
+ ++nContigsFound;
+ }
+ }
- fprintf(fp, "PhiStatistic,nReplicates,Mean,SD,CI_lower,CI_upper\n");
- for (int i = 0; i < nPhiValues; i++) {
- std::sort(phiValues[i], phiValues[i] + nReplicates);
+ // if found new contig:
+ // -> check if the contig in input blocks file exists in the VCF file
+ if (contigChanged) {
+ size_t contig_i;
+ for (contig_i = 0; contig_i < nContigsVcf; ++contig_i) {
+ if (strcmp(chr, vcfContigs[contig_i]) == 0) {
+ break;
+ }
+ if (contig_i == nContigsVcf - 1) {
+ ERROR("Contig %s in input blocks file does not exist in the VCF file.", chr);
+ }
+ }
+ bblocks->contig_names->add(chr);
+ size_t* tmp = (size_t*)realloc(bblocks->block_contig, (nBlocks + 1) * sizeof(size_t));
+ ASSERT(tmp != NULL);
+ bblocks->block_contig = tmp;
+ bblocks->block_contig[nBlocks] = contig_i;
- mean = MATH::MEAN(phiValues[i], nReplicates);
- sd = MATH::SD(phiValues[i], nReplicates);
+ }
- fprintf(fp, "%d,%d,%f,%f,%f,%f\n", i, nReplicates, mean, sd, phiValues[i][n_lower], phiValues[i][n_upper]);
+ ++nBlocks;
+ if (tmp_nBlocks == nBlocks) {
+ tmp_nBlocks *= 2;
+ bblocks->block_start_pos = (size_t*)realloc(bblocks->block_start_pos, tmp_nBlocks * sizeof(size_t));
+ ASSERT(bblocks->block_start_pos != NULL);
+ bblocks->block_end_pos = (size_t*)realloc(bblocks->block_end_pos, tmp_nBlocks * sizeof(size_t));
+ ASSERT(bblocks->block_end_pos != NULL);
+ for (size_t i = nBlocks;i < tmp_nBlocks;++i) {
+ bblocks->block_start_pos[i] = 0;
+ bblocks->block_end_pos[i] = 0;
+ }
+ }
}
-}
-bootstrapReplicate::bootstrapReplicate(int nBlocks) {
- rBlocks = (int*)malloc(nBlocks * sizeof(int));
- for (int i = 0; i < nBlocks; ++i) {
- rBlocks[i] = -1;
+ bblocks->n_blocks = nBlocks;
+ bblocks->n_contigs = nContigsFound;
+ const size_t nInd = pars->names->len;
+ bblocks->n_ind = nInd;
+ bblocks->nsites_per_block = (uint64_t*)malloc(nBlocks * sizeof(uint64_t));
+ ASSERT(bblocks->nsites_per_block != NULL);
+ for (size_t i = 0;i < nBlocks;++i) {
+ bblocks->nsites_per_block[i] = 0;
}
-}
-bootstrapReplicate::~bootstrapReplicate() {
- FREE(rBlocks);
- delete amova;
+ FREE(tok);
+ FCLOSE(fp);
+ return;
}
-void blobStruct::print() {
- if (0 == args->printBlocksTab)
- return;
+void bblocks_get(bblocks_t* bblocks, vcfData* vcfd, paramStruct* pars) {
+ DEVASSERT(bblocks != NULL);
+ DEVASSERT(vcfd != NULL);
- fprintf(stderr, "\n[INFO]\t-> Writing blocks tab file: %s\n", outFiles->out_blockstab_fs->fn);
- outFiles->out_blockstab_fs->kbuf = kbuf_init();
- // blocks tab file = 1-based, inclusive start, inclusive end
- // internal representation = 0-based, inclusive start, exclusive end
- // convert internal representation to blocks tab file representation
- for (int bi = 0; bi < nBlocks; ++bi) {
- ksprintf(outFiles->out_blockstab_fs->kbuf, "%s\t%d\t%d\n", blocks[bi]->chr, blocks[bi]->start + 1, blocks[bi]->end);
+ if (PROGRAM_HAS_INPUT_BLOCKS) {
+ if (args->in_blocks_bed_fn != NULL) {
+ bblocks_read_bed(bblocks, args->in_blocks_bed_fn, vcfd, pars);
+ } else if (args->in_blocks_tab_fn != NULL) {
+ bblocks_read_tab(bblocks, args->in_blocks_tab_fn, vcfd, pars);
+ } else {
+ NEVER;
+ }
+ } else if (args->blockSize > 0) {
+ bblocks_generate_blocks_with_size(bblocks, vcfd, pars, args->blockSize);
+ } else {
+ NEVER;
}
- outFiles->out_blockstab_fs->kbuf_write();
-}
\ No newline at end of file
+
+ return;
+}
diff --git a/bootstrap.h b/bootstrap.h
index e579f4d..df80818 100644
--- a/bootstrap.h
+++ b/bootstrap.h
@@ -1,147 +1,94 @@
#ifndef __BOOTSTRAP__
#define __BOOTSTRAP__
-#include "argStruct.h"
#include "dataStructs.h"
-#include "paramStruct.h"
#include "shared.h"
-#include "vcfReader.h"
/* FORWARD DECLARATIONS ----------------------------------------------------- */
-struct distanceMatrixStruct;
-struct metadataStruct;
-struct vcfData;
-struct amovaStruct;
-
-typedef struct blockStruct blockStruct;
-typedef struct blobStruct blobStruct;
-typedef struct bootstrapReplicate bootstrapReplicate;
-typedef struct bootstrapDataset bootstrapDataset;
-
-
-typedef struct block_t block_t;
-typedef struct block_bootstrap_t block_bootstrap_t;
+typedef struct bblocks_t bblocks_t;
+typedef struct strArray strArray;
+void strArray_destroy(strArray* sa);
+typedef struct vcfData vcfData;
+typedef struct paramStruct paramStruct;
/* -------------------------------------------------------------------------- */
-
-/// @brief block_t - structure for storing a single block
-/// @details
-/// positions are 0-based
-/// [start, end) - [inclusive start, exclusive end)
-struct block_t {
- char* chr; // pointer to the chromosome name; not allocated
- int start; // inclusive
- int end; // exclusive
- int id; // block id (index of the block in the blocks array)
- // length = end - start
-};
-
-
-struct block_bootstrap_t {
- block_t* blocks;
- int* blockPtrs;
-};
-
-
-/// @brief blockStruct - structure for storing a single block
+/// @brief bblocks_sample_with_replacement - perform the block bootstrapping sampling with replacement
/// @details
-/// positions are 0-based
-/// [start, end) - [inclusive start, exclusive end)
-struct blockStruct {
- char* chr = NULL;
- // const size_t chr; //TODO chr id instead
- int start = 0; // inclusive
- int end = 0; // exclusive
- int len = 0; // end - start; length of the block
-};
-// to access the data
-// if this block is not the last block in the contig
-// loop: from this_blockStart to next_blockStart
-// if this block is the last block in the contig
-// loop: from this_blockStart to the end of the contig
-// for (int gti = 0; gti < 3; gti++)
-// {
-// double x = vcfd->lngl[0][3 * vb + gti];
-// }
+/// sample the blocks with replacement for all individuals as to preserve the correlation structure
+void bblocks_sample_with_replacement(bblocks_t* bblocks);
-/// @brief blobStruct - structure for storing all blocks
-struct blobStruct {
- // Total number of blocks
- int nBlocks = 0;
+void bblocks_print_blocks_tab(bblocks_t* bblocks);
+void bblocks_print_bootstrap_samples(bblocks_t* bblocks);
- // /def blocks[nBlocks]
- // blocks[i] == pointer to a blockStruct at index i
- blockStruct** blocks = NULL;
+/// @brief bblocks_t - structure for storing bootstrap blocks
+/// @note positions: 0-based, [start, end)
+struct bblocks_t {
- // /def blockPtrs[nBlocks]
- // blockPtrs[i] == pointer to the location of the data for block i
- // e.g. blockPtrs[42] == index of the first site in block 42 in vcf data
- int** blockPtrs = NULL;
+ /// @var n_blocks - number of blocks
+ size_t n_blocks;
- bootstrapDataset* bootstraps = NULL;
+ /// @var n_contigs - number of contigs
+ size_t n_contigs;
- ~blobStruct();
+ /// @var n_ind - number of individuals
+ size_t n_ind;
- void addBlock();
+ /// \def rblocks[nReps][n_blocks] = size_t index of the sampled block
+ size_t** rblocks;
- void _print();
-
- void print();
-
- void get_bootstrap_replicates(vcfData* vcfd, bootstrapReplicate* bRep);
-};
-
-blobStruct* blobStruct_get(vcfData* vcf, paramStruct* pars, distanceMatrixStruct* dMS, metadataStruct* mS);
-blobStruct* blobStruct_get(vcfData* vcf, paramStruct* pars);
-blobStruct* blobStruct_read_bed(const char* fn);
-blobStruct* blobStruct_read_tab(const char* fn);
-blobStruct* blobStruct_populate_blocks_withSize(vcfData* vcf);
-
-void* blobStruct_destroy(blobStruct* blob);
-
-int sample_block_variant(metadataStruct* mtd, const int lvl, const int local_group_idx);
-
-bootstrapReplicate* get_bootstrap_replicate(vcfData* vcfd, blobStruct* blobSt);
-int sample_with_replacement(int n);
-
-struct bootstrapReplicate {
- // /def rBlocks[nBlocks] - set of block ids
- // sampled in the bootstrap replicate bootstrapReplicate
- int* rBlocks = NULL;
-
- distanceMatrixStruct* distanceMatrix = NULL;
-
- amovaStruct* amova = NULL;
-
- bootstrapReplicate(int nBlocks);
- ~bootstrapReplicate();
-};
+ /// @var nsites_per_block[n_blocks]
+ /// nsites_per_block[i] == number of sites in block i
+ uint64_t* nsites_per_block;
-struct bootstrapDataset {
- // /def rep[nBootstraps]
- // rep[i] == pointer to bootstrapData struct for ith bootstrap replicate
- bootstrapReplicate** replicates = NULL;
+ strArray* contig_names;
- int nReplicates = 0;
- int nBlocks = 0;
+ /// @var block_start_pos[n_blocks]
+ /// block_start_pos[i] == position (vcfd->rec->pos) of the start of block i
+ /// @note 0-based, inclusive [start
+ size_t* block_start_pos;
- // \def phiValues[indexOfPhiStatistic][nBootstrapReplicates]
- // nPhiValues == (nLevels * 2)-3
- double** phiValues = NULL;
- int nPhiValues = 0;
+ /// @var block_end_pos[n_blocks]
+ /// block_end_pos[i] == position (vcfd->rec->pos) of the end of block i
+ /// @note 0-based, exclusive end)
+ size_t* block_end_pos;
+ /// @var block_contig[n_blocks]
+ /// block_contig[i] == index of the contig (in contig_names) that block i belongs to
+ size_t* block_contig;
- void print_confidenceInterval(FILE* fp);
- void print();
+ /// @var block_start_siteidx[n_blocks]
+ /// block_start_siteidx[i] == index of the first site that belongs to block i
+ /// @note site index refers to the set of sites the program did not skip and are included in the analysis
+ /// if all sites from all contigs are used, this set is of size for(c in contigs) size+=contignsites[c]
+ /// the max siteidx is ((vcfd->nSites)-1), the min is 0, and it is global through all contigs
+ /// vcfd->nSites==pars->nSites
+ /// this information is collected during sites reading
+ /// @note 0-based siteidx inclusive [start
+ /// @note block_end = (wb == nBlocks - 1) ? max_nsites : block_start_siteidx[wb + 1];
+ size_t* block_start_siteidx;
- bootstrapDataset(paramStruct* pars, int nBootstraps_, int nBlocks_);
- ~bootstrapDataset();
};
-bootstrapDataset* bootstrapDataset_get(vcfData* vcfd, paramStruct* pars, blobStruct* blobSt);
+inline bblocks_t* bblocks_init(void) {
+ bblocks_t* bblocks = (bblocks_t*)malloc(sizeof(bblocks_t));
+ ASSERT(bblocks != NULL);
+ bblocks->n_blocks = 0;
+ bblocks->n_contigs = 0;
+ bblocks->n_ind = 0;
+ bblocks->nsites_per_block = NULL;
+ bblocks->block_start_pos = NULL;
+ bblocks->block_end_pos = NULL;
+ bblocks->block_contig = NULL;
+ bblocks->block_start_siteidx = NULL;
+ bblocks->contig_names = NULL;
+ return(bblocks);
+}
+
+void bblocks_destroy(bblocks_t* bblocks);
+
+void bblocks_get(bblocks_t* bblocks, vcfData* vcfd, paramStruct* pars);
-void get_distanceMatrix_GT(paramStruct* pars, distanceMatrixStruct* distanceMatrix, vcfData* vcfd, blobStruct* blob);
#endif // __BOOTSTRAP__
diff --git a/dataStructs.cpp b/dataStructs.cpp
index 558dfc6..60809af 100644
--- a/dataStructs.cpp
+++ b/dataStructs.cpp
@@ -1,106 +1,6 @@
#include "dataStructs.h"
#include "em.h"
-lnglStruct::lnglStruct(const int nGt, const int nInd) {
- this->size1 = NSITES_BUF_INIT;
- this->size2 = (size_t)nGt * nInd;
-
- this->d = (double**)malloc(this->size1 * sizeof(double*));
- for (size_t i = 0; i < this->size1;++i) {
- this->d[i] = (double*)malloc(this->size2 * sizeof(double));
- for (size_t j = 0;j < this->size2;++j) {
- this->d[i][j] = NEG_INF;
- }
- }
-}
-
-lnglStruct::~lnglStruct() {
- for (size_t i = 0;i < this->size1;++i) {
- FREE(this->d[i]);
- }
- FREE(this->d);
-}
-
-
-void jgtmat_get_srcgl(jgtmat_t* jgtm, paramStruct* pars, vcfData* vcfd, blobStruct* blob) {
- readSites(jgtm, vcfd, pars, blob);
- if (1 == args->doEM) {
- spawnThreads_em_optimize_jgtmat(jgtm, pars, vcfd);
- return;
- } else {
- ERROR("Nothing to do.");
- }
- return;
-}
-
-void jgtmat_get_srcgt(jgtmat_t* jgtm, paramStruct* pars, vcfData* vcfd, blobStruct* blob) {
- readSites(jgtm, vcfd, pars, blob);
-}
-
-
-void dmat_write(dmat_t* dmat) {
- LOG("Writing distance matrix to %s\n", outFiles->out_dm_fs->fn);
- outFiles->out_dm_fs->kbuf = kbuf_init();
- dmat_print(dmat, outFiles->out_dm_fs->kbuf);
- outFiles->out_dm_fs->kbuf_write();
-}
-
-void dmat_print(dmat_t* dmat, kstring_t* kstr) {
- if (0 == args->printDistanceMatrix) {
- return;
- }
-
- double* matrix = NULL;
- if (dmat->n == 1) {
- matrix = dmat->matrix[0];
- } else {
- NEVER; //TODO add multi matrix printing
- }
-
-
- // line 0: number of matrices
- ksprintf(kstr, "%ld\n", dmat->n);
-
-
- // line 1: type
- if (DMAT_TYPE_LTED == dmat->type) {
- ksprintf(kstr, "%s\n", "LTED");
- } else if (DMAT_TYPE_LTID == dmat->type) {
- ksprintf(kstr, "%s\n", "LTID");
- } else if (DMAT_TYPE_UTED == dmat->type) {
- ksprintf(kstr, "%s\n", "UTED");
- } else if (DMAT_TYPE_UTID == dmat->type) {
- ksprintf(kstr, "%s\n", "UTID");
- } else if (DMAT_TYPE_FULL == dmat->type) {
- ksprintf(kstr, "%s\n", "FULL");
- } else {
- NEVER;
- }
-
- // line 2: transformation
- ksprintf(kstr, "%d\n", dmat->transform);
-
- // line 3: method
- ksprintf(kstr, "%d\n", dmat->method);
-
- // line 4: number of dmat->names
- ksprintf(kstr, "%ld\n", dmat->names->len);
-
- // line 5: number of distances
- ksprintf(kstr, "%ld\n", dmat->size);
-
- // lines 6-X: dmat->names
- for (size_t i = 0;i < dmat->names->len;++i) {
- ksprintf(kstr, "%s\n", dmat->names->d[i]);
- }
-
- // lines (X+1)-Y: distances
- for (size_t p = 0; p < dmat->size; ++p) {
- ksprintf(kstr, "%f\n", matrix[p]);
- }
-
- return;
-}
@@ -228,7 +128,7 @@ metadataStruct* metadataStruct_read(paramStruct* pars) {
while (('\n' != full_line[full_line_len - 1]) && (!missing_newline)) {
// line was not fully read
- DEVPRINT("Line was not fully read, increasing buffer size from %ld to %ld\n", buf_size, buf_size * 2);
+ // DEVPRINT("Line was not fully read, increasing buffer size from %ld to %ld\n", buf_size, buf_size * 2);
buf_size *= 2;
REALLOC(char*, full_line, buf_size);
@@ -321,20 +221,18 @@ metadataStruct* metadataStruct_read(paramStruct* pars) {
ERROR("Found %ld columns in metadata file but expected %ld. Please make sure that your metadata file does not contain empty columns.\n", colidx + 1, nLevels);
}
-
-
// -> start with the individual column (lvl=nLevels, lvlidx=nLevels-1)
// check if individual id is already in names
- if (mtd->names->contains(lvlCols[nLevels - 1])) {
+ if (mtd->indNames->contains(lvlCols[nLevels - 1])) {
// it should not be already in names since we populate it here, this means multiple individuals have the same id
ERROR("Individual name %s is duplicated in the metadata file.\n", lvlCols[nLevels - 1]);
}
- size_t x = mtd->names->add(lvlCols[nLevels - 1]);
+ size_t x = mtd->indNames->add(lvlCols[nLevels - 1]);
ASSERT(x == indidx);
- IO::vprint(2, "Found individual with name:%s (index: %d) in the metadata file.", mtd->names->d[mtd->names->len - 1], indidx);
+ IO::vprint(2, "Found individual with name:%s (index: %d) in the metadata file.", mtd->indNames->d[mtd->indNames->len - 1], indidx);
colidx = 0;
@@ -435,53 +333,51 @@ metadataStruct* metadataStruct_read(paramStruct* pars) {
#if 0
-
- fprintf(stderr, "\n\n\n\n--------------------------------------------------------------------------------\n");
- size_t nGroupsTotal = mtd->groupNames->len;
- for (size_t i = 0;i < nGroupsTotal;++i) {
- size_tArray* tmp = mtd->group2subgroupIndices[i];
- if (tmp != NULL) {
- fprintf(stderr, "Group %ld (%s) has %ld subgroups\n", i, mtd->groupNames->d[i], tmp->len);
- for (size_t j = 0;j < tmp->len;++j) {
- if (j == 0)
- fprintf(stderr, "Subgroups are:\n");
- fprintf(stderr, "\t%ld. %s (from level %ld)\n", j + 1, mtd->groupNames->d[tmp->d[j]], mtd->group2levelIndices->d[tmp->d[j]] + 1);
- }
- }
- }
-
- fprintf(stderr, "\n\n\n\n--------------------------------------------------------------------------------\n");
-
#endif
mtd->nInd = indidx;
- //TODO verbose print this
- // if (VERBOSE) {
- // VRUN(
+ if (PROGRAM_VERBOSITY_LEVEL > 2) {
- fprintf(stderr, "\n\n\n\n--------------------------------------------------------------------------------\n");
+ fprintf(stderr, "\n\n\n\n--------------------------------------------------------------------------------\n");
+
+ fprintf(stderr, "In total, we have %ld levels, %d groups, and %d individuals\n", nLevels, mtd->nGroups, mtd->nInd);
+
+ for (size_t i = 0;i < (size_t)nLevels - 1;++i) {
+ fprintf(stderr, "\n\n// mtd->level2groupIndices->d[%ld] is an array of group indices at level %ld; it has %ld elements\n", i, i, mtd->level2groupIndices[i]->len);
+ fprintf(stderr, "-> Level %ld with name %s has %ld groups\n", i, mtd->levelNames->d[i], mtd->level2groupIndices[i]->len);
+ // the names of these groups are:
+ for (size_t j = 0;j < mtd->level2groupIndices[i]->len;++j) {
+ fprintf(stderr, "\t\t%ld. group idx=%ld with name %s\n", j + 1, mtd->level2groupIndices[i]->d[j], mtd->groupNames->d[mtd->level2groupIndices[i]->d[j]]);
+ fprintf(stderr, "\t\t\t-> this group has %ld individuals:\n", mtd->group2indIndices[mtd->level2groupIndices[i]->d[j]]->len);
+ for (size_t k = 0;k < mtd->group2indIndices[mtd->level2groupIndices[i]->d[j]]->len;++k) {
+ fprintf(stderr, "\t\t\t\t%ld. ind idx=%ld with name %s\n", k + 1, mtd->group2indIndices[mtd->level2groupIndices[i]->d[j]]->d[k], mtd->indNames->d[mtd->group2indIndices[mtd->level2groupIndices[i]->d[j]]->d[k]]);
+ }
+ }
- fprintf(stderr, "In total, we have %ld levels, %d groups, and %d individuals\n", nLevels, mtd->nGroups, mtd->nInd);
+ }
+ fprintf(stderr, "\n\n--------------------------------------------------------------------------------\n");
- for (size_t i = 0;i < (size_t)nLevels - 1;++i) {
- fprintf(stderr, "\n\n// mtd->level2groupIndices->d[%ld] is an array of group indices at level %ld; it has %ld elements\n", i, i, mtd->level2groupIndices[i]->len);
- fprintf(stderr, "-> Level %ld with name %s has %ld groups\n", i, mtd->levelNames->d[i], mtd->level2groupIndices[i]->len);
- // the names of these groups are:
- for (size_t j = 0;j < mtd->level2groupIndices[i]->len;++j) {
- fprintf(stderr, "\t\t%ld. group idx=%ld with name %s\n", j + 1, mtd->level2groupIndices[i]->d[j], mtd->groupNames->d[mtd->level2groupIndices[i]->d[j]]);
- fprintf(stderr, "\t\t\t-> this group has %ld individuals:\n", mtd->group2indIndices[mtd->level2groupIndices[i]->d[j]]->len);
- for (size_t k = 0;k < mtd->group2indIndices[mtd->level2groupIndices[i]->d[j]]->len;++k) {
- fprintf(stderr, "\t\t\t\t%ld. ind idx=%ld with name %s\n", k + 1, mtd->group2indIndices[mtd->level2groupIndices[i]->d[j]]->d[k], mtd->names->d[mtd->group2indIndices[mtd->level2groupIndices[i]->d[j]]->d[k]]);
+ fprintf(stderr, "\n\n\n\n--------------------------------------------------------------------------------\n");
+ size_t nGroupsTotal = mtd->groupNames->len;
+ for (size_t i = 0;i < nGroupsTotal;++i) {
+ size_tArray* tmp = mtd->group2subgroupIndices[i];
+ if (tmp != NULL) {
+ fprintf(stderr, "Group %ld (%s) has %ld subgroups\n", i, mtd->groupNames->d[i], tmp->len);
+ for (size_t j = 0;j < tmp->len;++j) {
+ if (j == 0)
+ fprintf(stderr, "Subgroups are:\n");
+ fprintf(stderr, "\t%ld. %s (from level %ld)\n", j + 1, mtd->groupNames->d[tmp->d[j]], mtd->group2levelIndices->d[tmp->d[j]] + 1);
+ }
}
}
+ fprintf(stderr, "\n\n\n\n--------------------------------------------------------------------------------\n");
+
}
- fprintf(stderr, "\n\n--------------------------------------------------------------------------------\n");
- // );
size_t n;
@@ -517,285 +413,6 @@ metadataStruct* metadataStruct_read(paramStruct* pars) {
}
-dmat_t* dmat_read(const char* in_dm_fn, const uint32_t required_transform, metadataStruct* metadata) {
-
- FILE* fp = NULL;
-
- bool isGzFile = false;
- if (IO::isGzFile(args->in_dm_fn) == 1) {
- isGzFile = true;
- kstring_t cmd = KS_INITIALIZE;
- ksprintf(&cmd, "zcat %s", args->in_dm_fn);
- fp = popen(cmd.s, "r");
- } else {
- fp = IO::getFile(args->in_dm_fn, "r");
- }
-
- dmat_t* ret = NULL;
- ret = (dmat_t*)malloc(sizeof(dmat_t));
- ASSERT(ret != NULL);
- ret->type = 0;
- ret->transform = 0;
- ret->method = 0;
- ret->size = 0;
- ret->names = NULL;
- ret->names_src = DMAT_NAMES_SRC_IN_DM_FILE;
- ret->matrix = NULL;
- ret->n = 0;
-
- int lineno = 0;
-
- BEGIN_LOGSECTION;
-
- // line 0: number of matrices
- if (fscanf(fp, "%ld", &ret->n) != 1) {
- // ERROR()
- ERROR("Could not read the number of matrices line (line 1) from the distance matrix input file %s", args->in_dm_fn);
- }
- if (0 == ret->n) {
- ERROR("Distance matrix input file must contain at least 1 distance matrices (line 1 > 0)");
- }
-
- ++lineno;
-
- // line 1: type
- char in_type[5] = { '\0' };
-
- if (fscanf(fp, "%s", in_type) != 1) {
- ERROR("Could not read the type line (line 2) from the distance matrix input file %s", args->in_dm_fn);
- }
- if (in_type[4] != '\0') {
- ERROR("Unrecognized distance matrix type: %s", in_type);
- }
-
- if (in_type[0] == 'F' && in_type[1] == 'U' && in_type[2] == 'L' && in_type[2] == 'L') {
- ret->type = DMAT_TYPE_FULL;
- LOG("Input distance matrix type is detected as FULL: Full Matrix");
- } else if (in_type[0] == 'L' && in_type[1] == 'T' && in_type[2] == 'E' && in_type[3] == 'D') {
- ret->type = DMAT_TYPE_LTED;
- LOG("Input distance matrix type is detected as LTED: Lower Triangular Matrix (Excluding Diagonal)");
- } else if (in_type[0] == 'L' && in_type[1] == 'T' && in_type[2] == 'I' && in_type[3] == 'D') {
- ret->type = DMAT_TYPE_LTID;
- LOG("Input distance matrix type is detected as LTID: Lower Triangular Matrix (Including Diagonal)");
- } else if (in_type[0] == 'U' && in_type[1] == 'T' && in_type[2] == 'E' && in_type[3] == 'D') {
- ret->type = DMAT_TYPE_UTED;
- LOG("Input distance matrix type is detected as UTED: Upper Triangular Matrix (Excluding Diagonal)");
- } else if (in_type[0] == 'U' && in_type[1] == 'T' && in_type[2] == 'I' && in_type[3] == 'D') {
- ret->type = DMAT_TYPE_UTID;
- LOG("Input distance matrix type is detected as UTID: Upper Triangular Matrix (Including Diagonal)");
- } else {
- ERROR("Unrecognized distance matrix type: %s", in_type);
- }
-
- ++lineno;
-
- // line 2: transformation
- if (fscanf(fp, "%d", &ret->transform) != 1) {
- ERROR("Could not read the transform line (line 3) from the distance matrix input file %s", args->in_dm_fn);
- }
-
- if (DMAT_TRANSFORM_NONE == ret->transform) {
- LOG("Input distance matrix transform is detected as: None");
- } else if (DMAT_TRANSFORM_SQUARE == ret->transform) {
- LOG("Input distance matrix transform is detected as: Squared");
- } else {
- ERROR("Unrecognized distance matrix transformation: %d", ret->transform);
- }
-
-
- ++lineno;
-
- // line 3: method
- if (fscanf(fp, "%d", &ret->method) != 1) {
- ERROR("Could not read the method line (line 4) from the distance matrix input file %s", args->in_dm_fn);
- }
-
- if (DMAT_METHOD_DIJ == ret->method) {
- LOG("Input distance matrix method is detected as: Dij");
- } else if (DMAT_METHOD_FIJ == ret->method) {
- LOG("Input distance matrix method is detected as: Fij");
- } else {
- ERROR("Unrecognized distance matrix method: %d", ret->method);
- }
-
-
- ++lineno;
-
- // line 4: number of names
- int in_nInd;
- if (fscanf(fp, "%d", &in_nInd) != 1) {
- ERROR("Could not read the second line from the distance matrix input file %s", args->in_dm_fn);
- }
-
- if (in_nInd > 0) {
- LOG("Found %d names in the distance matrix file", in_nInd);
- } else {
- ERROR("Number of names in the distance matrix file detected as %d. It must be greater than 0.", in_nInd);
- }
-
- ++lineno;
-
- // line 5: number of distances
- int in_nIndCmb;
- if (fscanf(fp, "%d", &in_nIndCmb) != 1) {
- ERROR("Could not read the third line from the distance matrix input file %s", args->in_dm_fn);
- }
-
-
- if (in_nIndCmb > 0) {
- LOG("Found %d distances in the distance matrix file", in_nIndCmb);
- } else {
- ERROR("Number of distances in the distance matrix file detected as %d. It must be greater than 0.", in_nIndCmb);
- }
-
- ret->size = in_nIndCmb;
-
- if (ret->type == DMAT_TYPE_LTED) {
- if ((int)ret->size != ((in_nInd * (in_nInd - 1)) / 2)) {
- ERROR("Number of distances in the input distance matrix (%ld) does not match the expected number of pairwise distances given the number of individuals (%d) and the distance matrix type (LTED).", ret->size, in_nInd);
- }
- } else if (ret->type == DMAT_TYPE_UTED) {
- if ((int)ret->size != ((in_nInd * (in_nInd - 1)) / 2)) {
- ERROR("Number of distances in the input distance matrix (%ld) does not match the expected number of pairwise distances given the number of individuals (%d) and the distance matrix type (UTED).", ret->size, in_nInd);
- }
- } else if (ret->type == DMAT_TYPE_LTID) {
- if ((int)ret->size != ((in_nInd * (in_nInd + 1)) / 2)) {
- ERROR("Number of distances in the input distance matrix (%ld) does not match the expected number of pairwise distances given the number of individuals (%d) and the distance matrix type (LTID).", ret->size, in_nInd);
- }
- } else if (ret->type == DMAT_TYPE_UTID) {
- if ((int)ret->size != ((in_nInd * (in_nInd + 1)) / 2)) {
- ERROR("Number of distances in the input distance matrix (%ld) does not match the expected number of pairwise distances given the number of individuals (%d) and the distance matrix type (UTID).", ret->size, in_nInd);
- }
- } else if (ret->type == DMAT_TYPE_FULL) {
- if ((int)ret->size != (in_nInd * in_nInd)) {
- ERROR("Number of distances in the input distance matrix (%ld) does not match the expected number of pairwise distances given the number of individuals (%d) and the distance matrix type (FULL).", ret->size, in_nInd);
- }
- } else {
- NEVER;
- }
-
- ++lineno;
-
- // lines 6:(6+nInd-1) names
- ret->names = strArray_alloc(in_nInd);
-
- char tmp[1024] = { '\0' };
- while (lineno < 6 + in_nInd) {
- tmp[0] = '\0';
- fscanf(fp, "%s\n", tmp);
- if ('\0' == tmp[0]) {
- ERROR("Found bad name at line %d of distance matrix file %s", lineno + 1, args->in_dm_fn);
- }
- ret->names->add(tmp);
- ++lineno;
- }
-
- ASSERT((int)ret->names->len == in_nInd);
- LOG("Read %ld names from distance matrix file", ret->names->len);
-
- ++lineno;
- // lines (6+nInd):(6+nInd+nIndCmb-1) distances
-
- ret->matrix = (double**)malloc(ret->n * sizeof(dmat_t*));
- ASSERT(ret->matrix != NULL);
-
- for (size_t i = 0; i < ret->n;++i) {
- ret->matrix[i] = NULL;
- ret->matrix[i] = (double*)malloc(ret->size * sizeof(double));
- ASSERT(ret->matrix[i] != NULL);
- for (size_t j = 0;j < ret->size;++j) {
- ret->matrix[i][j] = 0.0;
- }
- }
-
- size_t j = 0;
- if (ret->n == 1) {
- while (lineno < 7 + in_nInd + in_nIndCmb) {
- fscanf(fp, "%lf\n", &ret->matrix[0][j]);
- ++lineno;
- ++j;
- }
- } else {
- NEVER;//TODO
- }
-
-
- ASSERT((int)ret->size == in_nIndCmb);
- LOG("Read %ld distances from distance matrix file", ret->size);
-
-
-
- if (required_transform != ret->transform) {
-
- if (required_transform == DMAT_TRANSFORM_NONE) {
- // -> we need: no transform
-
- if (ret->transform == DMAT_TRANSFORM_SQUARE) {
- // -> we have: squared
- LOG("Input distance matrix transform (Squared, %d) is not the same as the required transform (None, %d). Will take square roots of the input values to obtain the required transform.", ret->transform, required_transform);
- for (size_t m = 0;m < ret->n;++m) {
- for (size_t i = 0;i < ret->size;++i) {
- ret->matrix[m][i] = sqrt(ret->matrix[m][i]);
- }
- }
- ret->transform = DMAT_TRANSFORM_NONE;
- } else {
- NEVER;
- }
-
- } else if (required_transform == DMAT_TRANSFORM_SQUARE) {
- // -> we need: squared
-
- if (ret->transform == DMAT_TRANSFORM_NONE) {
- // -> we have: no transform
- LOG("Input distance matrix transform (None, %d) is not the same as the required transform (Squared, %d). Will take squares of the input values to obtain the required transform.", ret->transform, required_transform);
- for (size_t m = 0;m < ret->n;++m) {
- for (size_t i = 0;i < ret->size;++i) {
- ret->matrix[m][i] *= ret->matrix[m][i];
- }
- }
- ret->transform = DMAT_TRANSFORM_SQUARE;
- } else {
- NEVER;
- }
- } else {
- NEVER;
- }
- }
-
- if (isGzFile) {
- PCLOSE(fp);
- } else {
- FCLOSE(fp);
-
- }
-
- // -> finished reading the distance matrix file
-
- if (metadata != NULL) {
- // -> reorder ret->names according to metadata->names
-
- // first, compare if the names in ret->names are the same as in metadata->names
- if (ret->names->len != metadata->names->len) {
- ERROR("Number of names in the distance matrix file (%ld) does not match the number of individuals in the metadata file (%ld).", ret->names->len, metadata->names->len);
- }
-
- for (size_t i = 0; i < ret->names->len; ++i) {
- if (strcmp(ret->names->d[i], metadata->names->d[i]) != 0) {
- break; // possibly the order is different
- //TODO HERE
- }
- }
-
-
- }
-
-
- END_LOGSECTION;
-
-
- return(ret);
-}
strArray* read_formula_str(const char* formula) {
@@ -889,19 +506,3 @@ strArray* read_formula_str(const char* formula) {
return (formulaTokens);
}
-
-
-//TODO eval dmat
-// void eval_distanceMatrixStruct(paramStruct *pars, distanceMatrixStruct *dm) {
-// for (int i = 0; i < dm->nIndCmb; i++) {
-// if (dm->M[i] == 0) {
-// // should we allow individuals with '0' distance?
-// // fprintf(stderr, "\n[ERROR]\tDistance between individuals (i1:%d,i2:%d,pair_index:%d) is %f. Please check your analysis settings and make sure you have enough data.\n", pars->lut_idxToInds[i][0], pars->lut_idxToInds[i][1], i, dm->M[i]);
-// exit(1);
-// }
-// if (dm->M[i] < 0) {
-// // fprintf(stderr, "\n[ERROR]\tDistance between individuals (i1:%d,i2:%d,pair_index:%d) is %f. Please check your analysis settings and make sure you have enough data.\n", pars->lut_idxToInds[i][0], pars->lut_idxToInds[i][1], i, dm->M[i]);
-// exit(1);
-// }
-// }
-// }
\ No newline at end of file
diff --git a/dataStructs.h b/dataStructs.h
index f7e994a..c3e6d04 100644
--- a/dataStructs.h
+++ b/dataStructs.h
@@ -7,11 +7,7 @@
#include "vcfReader.h"
-/// ----------------------------------------------------------------------- ///
-
-
-/// TODO !! MATCH THE INDNAMES IN THE VCF FILE WITH THE INDNAMES IN THE METADATA FILE
/* FORWARD DECLARATIONS ----------------------------------------------------- */
@@ -26,12 +22,13 @@ namespace IO {
struct blobStruct;
-typedef struct lnglStruct lnglStruct;
typedef struct vcfData vcfData;
typedef struct indPairThreads indPairThreads;
typedef struct intArray intArray;
+/// ----------------------------------------------------------------------- ///
+
struct intArray {
/// @var d - array of int data
@@ -504,29 +501,6 @@ inline void strArray_destroy(strArray* sa) {
/* -------------------------------------------------------------------------- */
-struct lnglStruct {
-
- /*
- * lngl[nSites][nInd*nGT*double]
- * genotype likelihoods in natural log
- * for each individual at each site
- *
- *
- */
- // d[size1][size2]
- double** d = NULL;
-
- // size1 follows pars->nSites_arrays_size
- size_t size1 = -1;
-
- size_t size2 = -1;
-
- lnglStruct(const int nGt, const int nInd);
- ~lnglStruct();
-
- void expand();
-
-};
/// @brief read_formula_str - read the formula string into a strArray of formula tokens
/// @param formula formula string
@@ -572,9 +546,10 @@ struct metadataStruct {
// levelNames = {"Region","Population","Subpopulation","Individual"}
strArray* levelNames;
- // \def names->d[nInds]
- // names->d[i] = name of individual i (char *)
- strArray* names;
+ // \def indNames->d[nInds]
+ // indNames->d[i] = name of individual i (char *)
+ /// @note free iff pars->metadata==NULL && PROGRAM_HAS_INPUT_VCF ; else it is ptr to pars->metadata->indNames
+ strArray* indNames;
// \def groupNames->d[nGroups]
// N.B. does not contain individual names
@@ -605,8 +580,8 @@ inline metadataStruct* metadataStruct_init(const int in_nLevels) {
ret->levelNames = NULL;
ret->levelNames = strArray_alloc(in_nLevels);
- ret->names = NULL;
- ret->names = strArray_init();
+ ret->indNames = NULL;
+ ret->indNames = strArray_init();
ret->groupNames = NULL;
ret->groupNames = strArray_init();
@@ -658,7 +633,7 @@ metadataStruct* metadataStruct_read(paramStruct* pars);
inline void metadataStruct_destroy(metadataStruct* mtd) {
strArray_destroy(mtd->levelNames);
- strArray_destroy(mtd->names);
+ strArray_destroy(mtd->indNames);
strArray_destroy(mtd->groupNames);
@@ -739,221 +714,11 @@ inline void trimSpaces(char* str) {
}
}
-/// ----------------------------------------------------------------------- ///
-/// DISTANCE MATRIX
-
-/// @brief DMAT_TYPE_* - type of the distance matrix
-///
-/// ___ ___ ___ (exclude=0,include=1)
-/// . . [0|1] -> diagonal
-/// . [0|1] . -> lower triangular
-/// [0|1] . . -> upper triangular
-///
-#define DMAT_TYPE_INCLUDE_DIAG (1<<0)
-#define DMAT_TYPE_INCLUDE_LOWER_TRI (1<<1)
-#define DMAT_TYPE_INCLUDE_UPPER_TRI (1<<2)
-/// LTED: lower triangular, excluding the diagonal
-/// (0<<2)|(1<<1)|(0<<0)
-#define DMAT_TYPE_LTED 2
-/// LTID: lower triangular, including the diagonal
-/// (0<<2)|(1<<1)|(1<<0)
-#define DMAT_TYPE_LTID 3
-/// UTED: upper triangular, excluding the diagonal
-/// (1<<2)|(0<<1)|(0<<0)
-#define DMAT_TYPE_UTED 4
-/// UTID: upper triangular, including the diagonal
-/// (1<<2)|(0<<1)|(1<<0)
-#define DMAT_TYPE_UTID 5
-/// FULL: full matrix
-/// (1<<2)|(1<<1)|(1<<0)
-#define DMAT_TYPE_FULL 7
-
-
-/// @brief DMAT_TRANSFORM_* - transformation applied to the distances in the matrix
-///
-/// NONE: not transformed
-#define DMAT_TRANSFORM_NONE 0
-/// SQUARE: val^2
-#define DMAT_TRANSFORM_SQUARE 1
-/// SQRT: sqrt(val) //TODO
-// #define DMAT_TRANSFORM_SQRT 2
-/// ABS: absolute value (|val|) //TODO
-// #define DMAT_TRANSFORM_ABS 3
-/// PSEUDO_EUCLIDEAN: // TODO
-// #define DMAT_TRANSFORM_PSEUDO_EUCLIDEAN 4
-
-
-/// @brief DMAT_METHOD_* - method used in distance calculation (i.e. distance metric)
-#define DMAT_METHOD_DIJ 0
-#define DMAT_METHOD_SIJ 1
-#define DMAT_METHOD_FIJ 2
-#define DMAT_METHOD_IBS0 3
-#define DMAT_METHOD_IBS1 4
-#define DMAT_METHOD_IBS2 5
-#define DMAT_METHOD_KIN 6
-#define DMAT_METHOD_R0 7
-#define DMAT_METHOD_R1 8
-#define DMAT_METHOD_DXY 9
-
-/// @brief DMAT_NAMES_SRC_* - source of the names in the distance matrix
-/// NONE: no names (names=NULL)
-#define DMAT_NAMES_SRC_NONE 0
-/// IN_DM_FILE: names is allocated and read from the distance matrix input file
-/// <requires cleaning>
-#define DMAT_NAMES_SRC_IN_DM_FILE 1
-/// IN_VCF_FILE: names is pointer to the strArray* names in pars which was read from the VCF file
-/// <no cleaning>
-#define DMAT_NAMES_SRC_IN_VCF_PARS_PTR 2
-/// IN_METADATA_FILE: names is pointer to the strArray* names in metadataStruct
-/// <no cleaning>
-/// NOTE: currently not used
-#define DMAT_NAMES_SRC_METADATA_NAMES_PTR 3
-/// PRIVATE: names is allocated and used internally in the program
-#define DMAT_NAMES_SRC_PRIVATE 4
-
-typedef struct dmat_t dmat_t;
-
-/// @struct dmat_t - distance matrix struct
-/// @brief struct for n distance matrices
-struct dmat_t {
-
- /// @var type - type of the distance matrix
- /// @details bitset
- uint8_t type;
-
- /// @var transform - transformation applied to the distances in the matrix
- uint32_t transform;
-
- /// @var method - method used to calculate the distances in the matrix
- /// @details
- /// 0: Dij
- /// 1: Sij
- /// 2: Fij
- /// 3: IBS0
- /// 4: IBS1
- /// 5: IBS2
- /// 6: Kin
- /// 7: R0
- /// 8: R1
- /// 9: Dxy
- uint32_t method;
-
- /// @var size - size of a each matrix matrix[n]
- /// @details typically nIndCmb
- size_t size;
-
- /// @var names - array of names of the individuals in the distance matrix
- /// @details
- /// - if names_src == DMAT_NAMES_SRC_IN_DM_FILE, names is allocated and read from the distance matrix input file
- /// - if names_src == DMAT_NAMES_SRC_IN_VCF_PARS_PTR, names is pointer to the strArray* names in pars which was read from the VCF file
- /// - if names_src == DMAT_NAMES_SRC_METADATA_NAMES_PTR, names is pointer to the strArray* names in metadataStruct
- /// - if names_src == DMAT_NAMES_SRC_NONE, names is NULL
- strArray* names;
- uint8_t names_src;
-
- /// @var n - number of distance matrices
- size_t n;
-
- /// @var matrix - array of n 1D distance matrices of size 'size'
- // matrix[n][size]
- double** matrix;
-
-};
-
-
-inline dmat_t* dmat_init(const size_t nInd, const uint8_t type, const uint32_t method, const uint32_t transform, strArray* names, const uint8_t names_src) {
-
- dmat_t* ret = NULL;
- ret = (dmat_t*)malloc(sizeof(dmat_t));
- ASSERT(ret != NULL);
-
- switch (type) {
- case DMAT_TYPE_LTED:
- case DMAT_TYPE_UTED:
- ret->size = (nInd * (nInd - 1)) / 2;
- break;
- case DMAT_TYPE_LTID:
- case DMAT_TYPE_UTID:
- ret->size = (nInd * (nInd + 1)) / 2;
- break;
- case DMAT_TYPE_FULL:
- ret->size = nInd * nInd;
- break;
- default:
- NEVER;
- }
-
- ret->type = type;
- ret->transform = transform;
- ret->method = method;
-
- ret->n = (args->nBootstraps > 0) ? (1 + args->nBootstraps) : 1;
-
- ret->names_src = names_src;
- if (ret->names_src == DMAT_NAMES_SRC_IN_VCF_PARS_PTR) {
- ret->names = names;
- } else if (ret->names_src == DMAT_NAMES_SRC_METADATA_NAMES_PTR) {
- ret->names = names;
- } else if (ret->names_src == DMAT_NAMES_SRC_PRIVATE) {
- // names is allocated and used internally in the program
- } else {
- NEVER;
- }
-
- ret->matrix = NULL;
- ret->matrix = (double**)malloc(ret->n * sizeof(dmat_t*));
- ASSERT(ret->matrix != NULL);
-
- for (size_t i = 0; i < ret->n;++i) {
- ret->matrix[i] = NULL;
- ret->matrix[i] = (double*)malloc(ret->size * sizeof(double));
- ASSERT(ret->matrix[i] != NULL);
- for (size_t j = 0;j < ret->size;++j) {
- ret->matrix[i][j] = 0.0;
- }
- }
- return(ret);
-}
-
-
-
-inline void dmat_destroy(dmat_t* d) {
- for (size_t i = 0; i < d->n;++i) {
- FREE(d->matrix[i]);
- }
- FREE(d->matrix);
-
- if (d->names_src == DMAT_NAMES_SRC_IN_DM_FILE) {
- strArray_destroy(d->names);
- } else if (d->names_src == DMAT_NAMES_SRC_IN_VCF_PARS_PTR) {
- d->names = NULL;
- } else if (d->names_src == DMAT_NAMES_SRC_METADATA_NAMES_PTR) {
- d->names = NULL;
- } else if (d->names_src == DMAT_NAMES_SRC_NONE) {
- NEVER;
- } else if (d->names_src == DMAT_NAMES_SRC_PRIVATE) {
- strArray_destroy(d->names);
- } else {
- NEVER;
- }
-
- FREE(d);
- return;
-}
-
-dmat_t* dmat_read(const char* in_dm_fn, const uint32_t required_transform, metadataStruct* metadata);
-
-void dmat_write(dmat_t* dmat);
-void dmat_print(dmat_t* dmat, kstring_t* kstr);
-
-
-
-
-
typedef struct jgtmat_t jgtmat_t;
struct jgtmat_t {
// typically: nIndCmb (one matrix per ind pair)
+ // or nIndCmb*(args->nBootstraps+1) (for each bootstrap replicate+the orig)
size_t n; // number of matrices
// - 9 (3x3) for MM Mm mm
@@ -964,79 +729,65 @@ struct jgtmat_t {
// pm[n][size]
double** pm;
- // expected counts (from doEM with gl source)
- // em[n][size]
- double** em;
-
// counts
// m[n][size]
- int** m;
+ uint64_t** m;
};
-inline jgtmat_t* jgtmat_init(const size_t nIndCmb) {
+inline jgtmat_t* jgtmat_init(const size_t nmat) {
jgtmat_t* ret = NULL;
ret = (jgtmat_t*)malloc(sizeof(jgtmat_t));
ASSERT(ret != NULL);
- ret->n = nIndCmb;
+ ret->n = nmat;
ret->pm = NULL;
- ret->em = NULL;
ret->m = NULL;
-
- if (args->doJGTM == 1) {
+ if (ARG_DOJGTM_3GT == args->doJGTM) {
ret->size = 9;
- } else if (args->doJGTM == 2) {
+ } else if (ARG_DOJGTM_10GT == args->doJGTM) {
ret->size = 100;
+ } else {
+ NEVER;
}
- ret->pm = (double**)malloc(ret->n * sizeof(double*));
- ASSERT(ret->pm != NULL);
- for (size_t i = 0;i < ret->n;++i) {
- ret->pm[i] = NULL;
- ret->pm[i] = (double*)malloc(ret->size * sizeof(double));
- ASSERT(ret->pm[i] != NULL);
- for (size_t ii = 0;ii < ret->size;++ii) {
- ret->pm[i][ii] = 0.0;
- }
- }
if (args->doEM) {
- // TODO are counts really necessary??
- ret->em = (double**)malloc(ret->n * sizeof(double*));
- ASSERT(ret->em != NULL);
+ ret->pm = (double**)malloc(ret->n * sizeof(double*));
+ ASSERT(ret->pm != NULL);
for (size_t i = 0;i < ret->n;++i) {
- ret->em[i] = NULL;
- ret->em[i] = (double*)malloc(ret->size * sizeof(double));
- ASSERT(ret->em[i] != NULL);
+ ret->pm[i] = NULL;
+ ret->pm[i] = (double*)malloc(ret->size * sizeof(double));
+ ASSERT(ret->pm[i] != NULL);
if (9 == ret->size) {
// use flat prior
- ret->em[i][0] = FRAC_1_9;
- ret->em[i][1] = FRAC_1_9;
- ret->em[i][2] = FRAC_1_9;
- ret->em[i][3] = FRAC_1_9;
- ret->em[i][4] = FRAC_1_9;
- ret->em[i][5] = FRAC_1_9;
- ret->em[i][6] = FRAC_1_9;
- ret->em[i][7] = FRAC_1_9;
- ret->em[i][8] = FRAC_1_9;
+ ret->pm[i][0] = FRAC_1_9;
+ ret->pm[i][1] = FRAC_1_9;
+ ret->pm[i][2] = FRAC_1_9;
+ ret->pm[i][3] = FRAC_1_9;
+ ret->pm[i][4] = FRAC_1_9;
+ ret->pm[i][5] = FRAC_1_9;
+ ret->pm[i][6] = FRAC_1_9;
+ ret->pm[i][7] = FRAC_1_9;
+ ret->pm[i][8] = FRAC_1_9;
} else if (100 == ret->size) {
// use flat prior
for (size_t j = 0;j < ret->size;++j) {
- ret->em[i][j] = 0.01;
+ ret->pm[i][j] = 0.01;
}
}
}
} else {
- ret->m = (int**)malloc(ret->n * sizeof(int*));
+ ret->m = (uint64_t**)malloc(ret->n * sizeof(uint64_t*));
ASSERT(ret->m != NULL);
for (size_t i = 0;i < ret->n;++i) {
ret->m[i] = NULL;
- ret->m[i] = (int*)malloc(ret->size * sizeof(int));
+ ret->m[i] = (uint64_t*)malloc(ret->size * sizeof(uint64_t));
ASSERT(ret->m[i] != NULL);
- for (size_t ii = 0;ii < ret->size;++ii) {
- ret->m[i][ii] = 0;
+ for (size_t j = 0;j < ret->size;++j) {
+ ret->m[i][j] = 0;
}
+
}
}
@@ -1044,12 +795,6 @@ inline jgtmat_t* jgtmat_init(const size_t nIndCmb) {
}
inline void jgtmat_destroy(jgtmat_t* d) {
- if (d->em != NULL) {
- for (size_t i = 0;i < d->n;++i) {
- FREE(d->em[i]);
- }
- FREE(d->em);
- }
if (d->m != NULL) {
for (size_t i = 0;i < d->n;++i) {
FREE(d->m[i]);
@@ -1066,8 +811,8 @@ inline void jgtmat_destroy(jgtmat_t* d) {
return;
}
-void jgtmat_get_srcgl(jgtmat_t* jgtm, paramStruct* pars, vcfData* vcfd, blobStruct* blob);
-void jgtmat_get_srcgt(jgtmat_t* jgtm, paramStruct* pars, vcfData* vcfd, blobStruct* blob);
+void jgtmat_get_srcgl(jgtmat_t* jgtm, paramStruct* pars, vcfData* vcfd);
+void jgtmat_get_srcgt(jgtmat_t* jgtm, paramStruct* pars, vcfData* vcfd);
diff --git a/dev.h b/dev.h
index f32957b..5b1b6bc 100644
--- a/dev.h
+++ b/dev.h
@@ -1,7 +1,7 @@
#ifndef __DEV__
#define __DEV__
-#define DEV 0
+#define DEV 1
#if 1 == DEV
diff --git a/dmat.cpp b/dmat.cpp
new file mode 100644
index 0000000..4a2dbc2
--- /dev/null
+++ b/dmat.cpp
@@ -0,0 +1,599 @@
+#include "dmat.h"
+#include "dataStructs.h"
+#include "jgtmat.h"
+
+dmat_t* dmat_init(const size_t nInd, const uint8_t type, const uint32_t method, const uint32_t transform, strArray* names, const uint8_t names_src) {
+
+ dmat_t* ret = NULL;
+ ret = (dmat_t*)malloc(sizeof(dmat_t));
+ ASSERT(ret != NULL);
+
+ switch (type) {
+ case DMAT_TYPE_LTED:
+ case DMAT_TYPE_UTED:
+ ret->size = (nInd * (nInd - 1)) / 2;
+ break;
+ case DMAT_TYPE_LTID:
+ case DMAT_TYPE_UTID:
+ ret->size = (nInd * (nInd + 1)) / 2;
+ break;
+ case DMAT_TYPE_FULL:
+ ret->size = nInd * nInd;
+ break;
+ default:
+ NEVER;
+ }
+
+ ret->type = type;
+ ret->transform = transform;
+ ret->method = method;
+
+ ret->n = (args->nBootstraps > 0) ? (1 + args->nBootstraps) : 1;
+
+ ret->names_src = names_src;
+ if (ret->names_src == DMAT_NAMES_SRC_IN_VCF_PARS_PTR) {
+ ret->names = names;
+ } else if (ret->names_src == DMAT_NAMES_SRC_IN_METADATA_NAMES_PTR) {
+ ret->names = names;
+ } else if (ret->names_src == DMAT_NAMES_SRC_PRIVATE) {
+ // names is allocated and used internally in the program
+ } else {
+ NEVER;
+ }
+
+ ret->matrix = NULL;
+ ret->matrix = (double**)malloc(ret->n * sizeof(dmat_t*));
+ ASSERT(ret->matrix != NULL);
+
+ for (size_t i = 0; i < ret->n;++i) {
+ ret->matrix[i] = NULL;
+ ret->matrix[i] = (double*)malloc(ret->size * sizeof(double));
+ ASSERT(ret->matrix[i] != NULL);
+ for (size_t j = 0;j < ret->size;++j) {
+ ret->matrix[i][j] = 0.0;
+ }
+ }
+ return(ret);
+}
+
+void dmat_destroy(dmat_t* d) {
+ for (size_t i = 0; i < d->n;++i) {
+ FREE(d->matrix[i]);
+ }
+ FREE(d->matrix);
+
+ if (d->names_src == DMAT_NAMES_SRC_IN_DM_FILE) {
+ strArray_destroy(d->names);
+ } else if (d->names_src == DMAT_NAMES_SRC_IN_VCF_PARS_PTR) {
+ d->names = NULL;
+ } else if (d->names_src == DMAT_NAMES_SRC_IN_METADATA_NAMES_PTR) {
+ d->names = NULL;
+ } else if (d->names_src == DMAT_NAMES_SRC_NONE) {
+ NEVER;
+ } else if (d->names_src == DMAT_NAMES_SRC_PRIVATE) {
+ strArray_destroy(d->names);
+ } else {
+ NEVER;
+ }
+
+ FREE(d);
+ return;
+}
+
+dmat_t* dmat_read(const char* in_dm_fn, const uint32_t required_transform, metadataStruct* metadata) {
+
+ FILE* fp = NULL;
+
+ bool isGzFile = false;
+ if (IO::isGzFile(args->in_dm_fn) == 1) {
+ isGzFile = true;
+ kstring_t cmd = KS_INITIALIZE;
+ ksprintf(&cmd, "zcat %s", args->in_dm_fn);
+ fp = popen(cmd.s, "r");
+ } else {
+ fp = IO::getFile(args->in_dm_fn, "r");
+ }
+
+ dmat_t* ret = NULL;
+ ret = (dmat_t*)malloc(sizeof(dmat_t));
+ ASSERT(ret != NULL);
+ ret->type = 0;
+ ret->transform = 0;
+ ret->method = 0;
+ ret->size = 0;
+ ret->names = NULL;
+
+ if (metadata != NULL) {
+ ret->names_src = DMAT_NAMES_SRC_IN_METADATA_NAMES_PTR;
+ } else {
+ ret->names_src = DMAT_NAMES_SRC_IN_DM_FILE;
+ }
+ ret->matrix = NULL;
+ ret->n = 0;
+
+ int lineno = 0;
+
+ BEGIN_LOGSECTION;
+
+ // line 0: number of matrices
+ if (fscanf(fp, "%ld", &ret->n) != 1) {
+ // ERROR()
+ ERROR("Could not read the number of matrices line (line 1) from the distance matrix input file %s", args->in_dm_fn);
+ }
+ if (0 == ret->n) {
+ ERROR("Distance matrix input file must contain at least 1 distance matrices (line 1 > 0)");
+ }
+
+ ++lineno;
+
+ // line 1: type
+ char in_type[5] = { '\0' };
+
+ if (fscanf(fp, "%s", in_type) != 1) {
+ ERROR("Could not read the type line (line 2) from the distance matrix input file %s", args->in_dm_fn);
+ }
+ if (in_type[4] != '\0') {
+ ERROR("Unrecognized distance matrix type: %s", in_type);
+ }
+
+ if (in_type[0] == 'F' && in_type[1] == 'U' && in_type[2] == 'L' && in_type[2] == 'L') {
+ ret->type = DMAT_TYPE_FULL;
+ LOG("Input distance matrix type is detected as FULL: Full Matrix");
+ } else if (in_type[0] == 'L' && in_type[1] == 'T' && in_type[2] == 'E' && in_type[3] == 'D') {
+ ret->type = DMAT_TYPE_LTED;
+ LOG("Input distance matrix type is detected as LTED: Lower Triangular Matrix (Excluding Diagonal)");
+ } else if (in_type[0] == 'L' && in_type[1] == 'T' && in_type[2] == 'I' && in_type[3] == 'D') {
+ ret->type = DMAT_TYPE_LTID;
+ LOG("Input distance matrix type is detected as LTID: Lower Triangular Matrix (Including Diagonal)");
+ } else if (in_type[0] == 'U' && in_type[1] == 'T' && in_type[2] == 'E' && in_type[3] == 'D') {
+ ret->type = DMAT_TYPE_UTED;
+ LOG("Input distance matrix type is detected as UTED: Upper Triangular Matrix (Excluding Diagonal)");
+ } else if (in_type[0] == 'U' && in_type[1] == 'T' && in_type[2] == 'I' && in_type[3] == 'D') {
+ ret->type = DMAT_TYPE_UTID;
+ LOG("Input distance matrix type is detected as UTID: Upper Triangular Matrix (Including Diagonal)");
+ } else {
+ ERROR("Unrecognized distance matrix type: %s", in_type);
+ }
+
+ ++lineno;
+
+ // line 2: transformation
+ if (fscanf(fp, "%d", &ret->transform) != 1) {
+ ERROR("Could not read the transform line (line 3) from the distance matrix input file %s", args->in_dm_fn);
+ }
+
+ if (DMAT_TRANSFORM_NONE == ret->transform) {
+ LOG("Input distance matrix transform is detected as: None");
+ } else if (DMAT_TRANSFORM_SQUARE == ret->transform) {
+ LOG("Input distance matrix transform is detected as: Squared");
+ } else {
+ ERROR("Unrecognized distance matrix transformation: %d", ret->transform);
+ }
+
+
+ ++lineno;
+
+ // line 3: method
+ if (fscanf(fp, "%d", &ret->method) != 1) {
+ ERROR("Could not read the method line (line 4) from the distance matrix input file %s", args->in_dm_fn);
+ }
+
+ if (DMAT_METHOD_DIJ == ret->method) {
+ LOG("Input distance matrix method is detected as: Dij");
+ } else if (DMAT_METHOD_FIJ == ret->method) {
+ LOG("Input distance matrix method is detected as: Fij");
+ } else {
+ ERROR("Unrecognized distance matrix method: %d", ret->method);
+ }
+
+
+ ++lineno;
+
+ // line 4: number of names
+ int in_nInd;
+ if (fscanf(fp, "%d", &in_nInd) != 1) {
+ ERROR("Could not read the second line from the distance matrix input file %s", args->in_dm_fn);
+ }
+
+ if (in_nInd > 0) {
+ LOG("Found %d names in the distance matrix file", in_nInd);
+ } else {
+ ERROR("Number of names in the distance matrix file detected as %d. It must be greater than 0.", in_nInd);
+ }
+
+ ++lineno;
+
+ // line 5: number of distances
+ int in_nIndCmb;
+ if (fscanf(fp, "%d", &in_nIndCmb) != 1) {
+ ERROR("Could not read the third line from the distance matrix input file %s", args->in_dm_fn);
+ }
+
+
+ if (in_nIndCmb > 0) {
+ LOG("Found %d distances in the distance matrix file", in_nIndCmb);
+ } else {
+ ERROR("Number of distances in the distance matrix file detected as %d. It must be greater than 0.", in_nIndCmb);
+ }
+
+ ret->size = in_nIndCmb;
+
+ if (ret->type == DMAT_TYPE_LTED) {
+ if ((int)ret->size != ((in_nInd * (in_nInd - 1)) / 2)) {
+ ERROR("Number of distances in the input distance matrix (%ld) does not match the expected number of pairwise distances given the number of individuals (%d) and the distance matrix type (LTED).", ret->size, in_nInd);
+ }
+ } else if (ret->type == DMAT_TYPE_UTED) {
+ if ((int)ret->size != ((in_nInd * (in_nInd - 1)) / 2)) {
+ ERROR("Number of distances in the input distance matrix (%ld) does not match the expected number of pairwise distances given the number of individuals (%d) and the distance matrix type (UTED).", ret->size, in_nInd);
+ }
+ } else if (ret->type == DMAT_TYPE_LTID) {
+ if ((int)ret->size != ((in_nInd * (in_nInd + 1)) / 2)) {
+ ERROR("Number of distances in the input distance matrix (%ld) does not match the expected number of pairwise distances given the number of individuals (%d) and the distance matrix type (LTID).", ret->size, in_nInd);
+ }
+ } else if (ret->type == DMAT_TYPE_UTID) {
+ if ((int)ret->size != ((in_nInd * (in_nInd + 1)) / 2)) {
+ ERROR("Number of distances in the input distance matrix (%ld) does not match the expected number of pairwise distances given the number of individuals (%d) and the distance matrix type (UTID).", ret->size, in_nInd);
+ }
+ } else if (ret->type == DMAT_TYPE_FULL) {
+ if ((int)ret->size != (in_nInd * in_nInd)) {
+ ERROR("Number of distances in the input distance matrix (%ld) does not match the expected number of pairwise distances given the number of individuals (%d) and the distance matrix type (FULL).", ret->size, in_nInd);
+ }
+ } else {
+ NEVER;
+ }
+
+ ++lineno;
+
+
+ /// ------------------------------------------------------------
+ // lines 6:(6+nInd-1) names
+
+ if (ret->names_src == DMAT_NAMES_SRC_IN_DM_FILE) {
+ ret->names = strArray_alloc(in_nInd);
+
+ char tmp[1024] = { '\0' };
+ while (lineno < 6 + in_nInd) {
+ tmp[0] = '\0';
+ fscanf(fp, "%s\n", tmp);
+ if ('\0' == tmp[0]) {
+ ERROR("Found bad name at line %d of distance matrix file %s", lineno + 1, args->in_dm_fn);
+ }
+ ret->names->add(tmp);
+ ++lineno;
+ }
+
+ ASSERT((int)ret->names->len == in_nInd);
+ LOG("Read %ld names from distance matrix file", ret->names->len);
+
+ ++lineno;
+
+ } else if (ret->names_src == DMAT_NAMES_SRC_IN_METADATA_NAMES_PTR) {
+
+ if (in_nInd != metadata->indNames->len) {
+ ERROR("Number of names in the distance matrix file (%ld) does not match the number of individuals in the metadata file (%ld). Please edit your metadata file to match the names in the distance matrix file.", in_nInd, metadata->indNames->len);
+ }
+
+ char tmp[1024] = { '\0' };
+ bool badorder = false;
+ size_t mtdidx;
+ while (lineno < 6 + in_nInd) {
+ tmp[0] = '\0';
+ fscanf(fp, "%s\n", tmp);
+ if ('\0' == tmp[0]) {
+ ERROR("Found bad name at line %d of distance matrix file %s", lineno + 1, args->in_dm_fn);
+ }
+ if (metadata->indNames->find(tmp, &mtdidx)) {
+ if (mtdidx != lineno - 6) {
+ DEVPRINT("name:%s idx:%ld mtdidx:%ld mtdname:%s", tmp, lineno - 6, mtdidx, metadata->indNames->d[mtdidx]);
+ badorder = true;
+ break;
+ }
+ } else {
+ ERROR("Name %s in the distance matrix file is not found in the metadata file.", tmp);
+ }
+ ++lineno;
+ }
+ if (badorder) {
+ ERROR("Names in the distance matrix file are not in the same order as in the metadata file. Please edit your metadata file to match the names in the distance matrix file.");
+ }
+
+ ret->names = metadata->indNames;
+
+ } else {
+ NEVER;
+ }
+
+ /// ------------------------------------------------------------
+ // lines (6+nInd):(6+nInd+nIndCmb-1) distances
+
+ ret->matrix = (double**)malloc(ret->n * sizeof(dmat_t*));
+ ASSERT(ret->matrix != NULL);
+
+ for (size_t i = 0; i < ret->n;++i) {
+ ret->matrix[i] = NULL;
+ ret->matrix[i] = (double*)malloc(ret->size * sizeof(double));
+ ASSERT(ret->matrix[i] != NULL);
+ for (size_t j = 0;j < ret->size;++j) {
+ ret->matrix[i][j] = 0.0;
+ }
+ }
+
+ size_t j = 0;
+ if (ret->n == 1) {
+ while (lineno < 7 + in_nInd + in_nIndCmb) {
+ fscanf(fp, "%lf\n", &ret->matrix[0][j]);
+ ++lineno;
+ ++j;
+ }
+ } else {
+ NEVER;//TODO
+ }
+
+
+ ASSERT((int)ret->size == in_nIndCmb);
+ LOG("Read %ld distances from distance matrix file", ret->size);
+
+
+
+ if (required_transform != ret->transform) {
+
+ if (required_transform == DMAT_TRANSFORM_NONE) {
+ // -> we need: no transform
+
+ if (ret->transform == DMAT_TRANSFORM_SQUARE) {
+ // -> we have: squared
+ LOG("Input distance matrix transform (Squared, %d) is not the same as the required transform (None, %d). Will take square roots of the input values to obtain the required transform.", ret->transform, required_transform);
+ for (size_t m = 0;m < ret->n;++m) {
+ for (size_t i = 0;i < ret->size;++i) {
+ ret->matrix[m][i] = sqrt(ret->matrix[m][i]);
+ }
+ }
+ ret->transform = DMAT_TRANSFORM_NONE;
+ } else {
+ NEVER;
+ }
+
+ } else if (required_transform == DMAT_TRANSFORM_SQUARE) {
+ // -> we need: squared
+
+ if (ret->transform == DMAT_TRANSFORM_NONE) {
+ // -> we have: no transform
+ LOG("Input distance matrix transform (None, %d) is not the same as the required transform (Squared, %d). Will take squares of the input values to obtain the required transform.", ret->transform, required_transform);
+ for (size_t m = 0;m < ret->n;++m) {
+ for (size_t i = 0;i < ret->size;++i) {
+ ret->matrix[m][i] *= ret->matrix[m][i];
+ }
+ }
+ ret->transform = DMAT_TRANSFORM_SQUARE;
+ } else {
+ NEVER;
+ }
+ } else {
+ NEVER;
+ }
+ }
+
+ if (isGzFile) {
+ PCLOSE(fp);
+ } else {
+ FCLOSE(fp);
+
+ }
+
+ // -> finished reading the distance matrix file
+
+ END_LOGSECTION;
+ return(ret);
+}
+
+void dmat_write(dmat_t* dmat) {
+ LOG("Writing distance matrix to %s\n", outFiles->out_dm_fs->fn);
+ outFiles->out_dm_fs->kbuf = kbuf_init();
+ dmat_print(dmat, outFiles->out_dm_fs->kbuf);
+ outFiles->out_dm_fs->kbuf_write();
+}
+
+
+void dmat_print(dmat_t* dmat, kstring_t* kstr) {
+
+ // line 0: number of matrices
+ ksprintf(kstr, "%ld\n", dmat->n);
+
+ // line 1: type
+ if (DMAT_TYPE_LTED == dmat->type) {
+ ksprintf(kstr, "%s\n", "LTED");
+ } else if (DMAT_TYPE_LTID == dmat->type) {
+ ksprintf(kstr, "%s\n", "LTID");
+ } else if (DMAT_TYPE_UTED == dmat->type) {
+ ksprintf(kstr, "%s\n", "UTED");
+ } else if (DMAT_TYPE_UTID == dmat->type) {
+ ksprintf(kstr, "%s\n", "UTID");
+ } else if (DMAT_TYPE_FULL == dmat->type) {
+ ksprintf(kstr, "%s\n", "FULL");
+ } else {
+ NEVER;
+ }
+
+ // line 2: transformation
+ ksprintf(kstr, "%d\n", dmat->transform);
+
+ // line 3: method
+ ksprintf(kstr, "%d\n", dmat->method);
+
+ // line 4: number of dmat->names
+ ksprintf(kstr, "%ld\n", dmat->names->len);
+
+ // line 5: number of distances
+ ksprintf(kstr, "%ld\n", dmat->size);
+
+ // lines 6-X: dmat->names
+ for (size_t i = 0;i < dmat->names->len;++i) {
+ ksprintf(kstr, "%s\n", dmat->names->d[i]);
+ }
+
+ double* matrix = NULL;
+
+ for(size_t mi=0;mi<dmat->n;++mi){
+ matrix=dmat->matrix[mi];
+
+ // lines (X+1)-Y: distances
+ for (size_t p = 0; p < dmat->size; ++p) {
+ // ksprintf(kstr, "%f\n", matrix[p]);
+ ksprintf(kstr,"%.17g\n",matrix[p]);
+ }
+ }
+
+ return;
+}
+
+void dmat_calculate_distances(jgtmat_t* jgtmat, dmat_t* dmat) {
+
+
+ double* dm = NULL;
+
+ uint32_t transform = dmat->transform;
+ uint32_t method= dmat->method;
+ const size_t nPairs = dmat->size;
+
+
+ size_t idx;
+ for (size_t i = 0;i < dmat->n;++i) {
+ // N.B. i==0 original run in dmat
+
+ idx = i * nPairs;
+ dm = dmat->matrix[i];
+
+ if (PROGRAM_WILL_USE_BCF_FMT_GT) {
+
+ double sum, x, A, B, C, D, E, F, G, H, I;
+ uint64_t** m = jgtmat->m;
+ uint64_t* pm = NULL;
+
+ for (size_t p = 0; p < nPairs; ++p) {
+
+ pm = m[idx + p];
+ A = pm[JGTMAT_LINEAR_IDXOF_A];
+ B = pm[JGTMAT_LINEAR_IDXOF_B];
+ C = pm[JGTMAT_LINEAR_IDXOF_C];
+ D = pm[JGTMAT_LINEAR_IDXOF_D];
+ E = pm[JGTMAT_LINEAR_IDXOF_E];
+ F = pm[JGTMAT_LINEAR_IDXOF_F];
+ G = pm[JGTMAT_LINEAR_IDXOF_G];
+ H = pm[JGTMAT_LINEAR_IDXOF_H];
+ I = pm[JGTMAT_LINEAR_IDXOF_I];
+ sum = A + B + C + D + E + F + G + H + I;
+
+ switch (method) {
+ case DMAT_METHOD_DIJ:
+ x = JGTMAT_GET_DIJ;
+ break;
+ case DMAT_METHOD_SIJ:
+ x = JGTMAT_GET_SIJ;
+ break;
+ case DMAT_METHOD_FIJ:
+ x = JGTMAT_GET_FIJ;
+ break;
+ case DMAT_METHOD_IBS0:
+ x = JGTMAT_GET_IBS0;
+ break;
+ case DMAT_METHOD_IBS1:
+ x = JGTMAT_GET_IBS1;
+ break;
+ case DMAT_METHOD_IBS2:
+ x = JGTMAT_GET_IBS2;
+ break;
+ case DMAT_METHOD_R0:
+ x = JGTMAT_GET_R0;
+ break;
+ case DMAT_METHOD_R1:
+ x = JGTMAT_GET_R1;
+ break;
+ case DMAT_METHOD_KIN:
+ x = JGTMAT_GET_KIN;
+ break;
+ default:
+ NEVER;
+ }
+ x /= sum;
+
+
+ if (DMAT_TRANSFORM_NONE != transform) {
+ if (DMAT_TRANSFORM_SQUARE == transform) {
+ x *= x;
+ }
+ }
+ dm[p] = x;
+ }
+
+ } else if (PROGRAM_WILL_USE_BCF_FMT_GL) {
+
+
+ double x, A, B, C, D, E, F, G, H, I;
+ double** m = jgtmat->pm;
+ double* pm = NULL; // pair pm
+
+ for (size_t p = 0; p < nPairs; ++p) {
+
+ pm = m[idx + p];
+ A = pm[JGTMAT_LINEAR_IDXOF_A];
+ B = pm[JGTMAT_LINEAR_IDXOF_B];
+ C = pm[JGTMAT_LINEAR_IDXOF_C];
+ D = pm[JGTMAT_LINEAR_IDXOF_D];
+ E = pm[JGTMAT_LINEAR_IDXOF_E];
+ F = pm[JGTMAT_LINEAR_IDXOF_F];
+ G = pm[JGTMAT_LINEAR_IDXOF_G];
+ H = pm[JGTMAT_LINEAR_IDXOF_H];
+ I = pm[JGTMAT_LINEAR_IDXOF_I];
+
+
+ switch (method) {
+ case DMAT_METHOD_DIJ:
+ x = JGTMAT_GET_DIJ;
+ break;
+ case DMAT_METHOD_SIJ:
+ x = JGTMAT_GET_SIJ;
+ break;
+ case DMAT_METHOD_FIJ:
+ x = JGTMAT_GET_FIJ;
+ break;
+ case DMAT_METHOD_IBS0:
+ x = JGTMAT_GET_IBS0;
+ break;
+ case DMAT_METHOD_IBS1:
+ x = JGTMAT_GET_IBS1;
+ break;
+ case DMAT_METHOD_IBS2:
+ x = JGTMAT_GET_IBS2;
+ break;
+ case DMAT_METHOD_R0:
+ x = JGTMAT_GET_R0;
+ break;
+ case DMAT_METHOD_R1:
+ x = JGTMAT_GET_R1;
+ break;
+ case DMAT_METHOD_KIN:
+ x = JGTMAT_GET_KIN;
+ break;
+ default:
+ NEVER;
+ }
+
+ if (DMAT_TRANSFORM_NONE != transform) {
+ if (DMAT_TRANSFORM_SQUARE == transform) {
+ x *= x;
+ }
+ }
+
+ dm[p] = x;
+ }
+
+
+ } else {
+ NEVER;
+ }
+
+ }
+
+
+ return;
+}
\ No newline at end of file
diff --git a/dmat.h b/dmat.h
new file mode 100644
index 0000000..3cf0ac6
--- /dev/null
+++ b/dmat.h
@@ -0,0 +1,142 @@
+#ifndef __DMAT_H__
+#define __DMAT_H__
+
+/* INCLUDES ----------------------------------------------------------------- */
+#include "shared.h"
+/* -------------------------------------------------------------------------- */
+
+/* FORWARD DECLARATIONS ----------------------------------------------------- */
+typedef struct metadataStruct metadataStruct;
+typedef struct strArray strArray;
+typedef struct jgtmat_t jgtmat_t;
+
+/* -------------------------------------------------------------------------- */
+
+
+
+/// ----------------------------------------------------------------------- ///
+/// DISTANCE MATRIX
+
+/// @brief DMAT_TYPE_* - type of the distance matrix
+///
+/// ___ ___ ___ (exclude=0,include=1)
+/// . . [0|1] -> diagonal
+/// . [0|1] . -> lower triangular
+/// [0|1] . . -> upper triangular
+///
+#define DMAT_TYPE_INCLUDE_DIAG (1<<0)
+#define DMAT_TYPE_INCLUDE_LOWER_TRI (1<<1)
+#define DMAT_TYPE_INCLUDE_UPPER_TRI (1<<2)
+/// LTED: lower triangular, excluding the diagonal
+/// (0<<2)|(1<<1)|(0<<0)
+#define DMAT_TYPE_LTED 2
+/// LTID: lower triangular, including the diagonal
+/// (0<<2)|(1<<1)|(1<<0)
+#define DMAT_TYPE_LTID 3
+/// UTED: upper triangular, excluding the diagonal
+/// (1<<2)|(0<<1)|(0<<0)
+#define DMAT_TYPE_UTED 4
+/// UTID: upper triangular, including the diagonal
+/// (1<<2)|(0<<1)|(1<<0)
+#define DMAT_TYPE_UTID 5
+/// FULL: full matrix
+/// (1<<2)|(1<<1)|(1<<0)
+#define DMAT_TYPE_FULL 7
+
+
+/// @brief DMAT_TRANSFORM_* - transformation applied to the distances in the matrix
+///
+/// NONE: not transformed
+#define DMAT_TRANSFORM_NONE 0
+/// SQUARE: val^2
+#define DMAT_TRANSFORM_SQUARE 1
+
+
+/// @brief DMAT_METHOD_* - method used in distance calculation (i.e. distance metric)
+#define DMAT_METHOD_DIJ 0
+#define DMAT_METHOD_SIJ 1
+#define DMAT_METHOD_FIJ 2
+#define DMAT_METHOD_IBS0 3
+#define DMAT_METHOD_IBS1 4
+#define DMAT_METHOD_IBS2 5
+#define DMAT_METHOD_KIN 6
+#define DMAT_METHOD_R0 7
+#define DMAT_METHOD_R1 8
+#define DMAT_METHOD_DXY 9
+
+/// @brief DMAT_NAMES_SRC_* - source of the names in the distance matrix
+/// NONE: no names (names=NULL)
+#define DMAT_NAMES_SRC_NONE 0
+/// IN_DM_FILE: names is allocated and read from the distance matrix input file, implies input is distance matrix AND metadata is NOT provided
+/// <requires cleaning>
+#define DMAT_NAMES_SRC_IN_DM_FILE 1
+/// IN_VCF_FILE: names is pointer to the strArray* names in pars which was read from the VCF file, implies no metadata is provided
+/// <no cleaning>
+#define DMAT_NAMES_SRC_IN_VCF_PARS_PTR 2
+/// IN_METADATA_FILE: names is pointer to the strArray* names in metadataStruct, implies input is distance matrix and metadata file is provided
+/// <no cleaning>
+#define DMAT_NAMES_SRC_IN_METADATA_NAMES_PTR 3
+/// PRIVATE: names is allocated and used internally in the program
+#define DMAT_NAMES_SRC_PRIVATE 4
+
+typedef struct dmat_t dmat_t;
+
+/// @struct dmat_t - distance matrix struct
+/// @brief struct for n distance matrices
+struct dmat_t {
+
+ /// @var n - number of distance matrices
+ size_t n;
+
+ /// @var size - size of a each matrix matrix[n]
+ /// @details typically nIndCmb
+ size_t size;
+
+ /// @var type - type of the distance matrix
+ /// @details bitset
+ uint8_t type;
+
+ /// @var transform - transformation applied to the distances in the matrix
+ uint32_t transform;
+
+ /// @var method - method used to calculate the distances in the matrix
+ /// @details
+ /// 0: Dij
+ /// 1: Sij
+ /// 2: Fij
+ /// 3: IBS0
+ /// 4: IBS1
+ /// 5: IBS2
+ /// 6: Kin
+ /// 7: R0
+ /// 8: R1
+ /// 9: Dxy
+ uint32_t method;
+
+ /// @var names - array of names of the individuals in the distance matrix
+ /// @details
+ /// - if names_src == DMAT_NAMES_SRC_IN_DM_FILE, names is allocated and read from the distance matrix input file
+ /// - if names_src == DMAT_NAMES_SRC_IN_VCF_PARS_PTR, names is pointer to the strArray* names in pars which was read from the VCF file, used when metadata is null
+ /// - if names_src == DMAT_NAMES_SRC_IN_METADATA_NAMES_PTR, names is pointer to the strArray* names in metadataStruct
+ /// - if names_src == DMAT_NAMES_SRC_NONE, names is NULL
+ strArray* names;
+ uint8_t names_src;
+
+ /// @var matrix - array of n 1D distance matrices of size 'size'
+ // matrix[n][size]
+ double** matrix;
+
+};
+
+
+dmat_t* dmat_init(const size_t nInd, const uint8_t type, const uint32_t method, const uint32_t transform, strArray* names, const uint8_t names_src);
+void dmat_destroy(dmat_t* d);
+
+dmat_t* dmat_read(const char* in_dm_fn, const uint32_t required_transform, metadataStruct* metadata);
+
+void dmat_write(dmat_t* dmat);
+void dmat_print(dmat_t* dmat, kstring_t* kstr);
+
+void dmat_calculate_distances(jgtmat_t* jgtmat, dmat_t* dmat);
+
+#endif // __DMAT_H__
\ No newline at end of file
diff --git a/doc/internal.MD b/doc/internal.MD
index 0c8f10b..ba11560 100644
--- a/doc/internal.MD
+++ b/doc/internal.MD
@@ -10,24 +10,90 @@ If distance matrix input, reorder names in dmat_t based on metad_t names.
```mermaid
graph TD;
+
+ DATAIN[Main data input]
+
+ subgraph INPUT
+ INDM[Distance matrix file]
+ INVCF[VCF file]
+ DATAIN --- INDM --- DISTNAMES
+ DATAIN --- INVCF --- VCFNAMES
+ end
+
MTD --- MTDNAMES
MTD[Metadata file]
- DATAIN[Data input file]
MTDNAMES[metad_t \n strArray names];
- INDM[Distance matrix file]
DISTNAMES[dmat_t \n strArray names];
NAMES_TO_USE[strArray names_to_use];
VCFNAMES[vcfd_t \n hdr->samples];
- JOIN1[join]
- JOIN2[join]
- MTDNAMES -.-> JOIN1
- MTDNAMES -.-> JOIN2
- DATAIN --- INDM --- DISTNAMES
- DATAIN --- INVCF --- VCFNAMES
- DISTNAMES --- JOIN1 ---> NAMES_TO_USE
- VCFNAMES --- JOIN2 ---> NAMES_TO_USE
+ MTDNAMES -.- DISTNAMES
+ MTDNAMES -.- VCFNAMES
+
+ DISTNAMES ---> NAMES_TO_USE
+ VCFNAMES ---> NAMES_TO_USE
+
+
+```
+
+```
+ pars->names = ptrto metadata->names
+
+
+
+if input is distance matrix
+ if metadata input is provided
+ filter and reorder dmat_t based on metad_t names
+ dmat->names = ptrto metadata->names
+ pars->names = ptrto metadata->names
+ else
+ dmat->names = allocated and filled from dmat input
+ pars->names = ptrto dmat->names
+
+else if input is VCF
+
+ if metadata input is provided
+ exclude samples from VCF that are not in metadata (set bool vcfd->includeSamples[bcf_hdr_nsamples(vcfd->hdr)]), no need to reorder
+ pars->names = ptrto metadata->names
+ else
+ pars->names = allocated and filled from vcfd->hdr->samples
+
+ if doDist
+ dmat->names = allocated and filled from pars->names
+
+```
+
+<!-- -doJGTM <int> : get joint genotypes matrix for each individual pair
+ ret: jgtmat
+-doDist <int> : estimate pairwise distance matrix
+ req: jgtmat
+ ret: dmat
+
+-doAMOVA <int> : perform AMOVA analysis
+ req: dmat, metadata, formula
+
+-doEM <int> : perform EM optimization
+ req: DATASOURCE_GL, lngl_t
+ ret: jgtmat
+
+TODO add doEM 2 for 10gls optim
+-doDxy <int> : estimate Dxy
+ req: dmat
+-doPhylo <int> : do neighbor-joining
+
+-doIbd <int> : detect IBD segments
+
+-doMajorMinor <int> : get major and minor alleles for each site -->
+
+```mermaid
+graph TD;
+ jgtmat[jgtmat]
+ dmat[dmat]
+ metadata[metadata]
+ formula[formula]
+ INPUT_VCF_GL[vcfd_t \n gl_t]
+
```
diff --git a/dxy.cpp b/dxy.cpp
index d811467..261077a 100644
--- a/dxy.cpp
+++ b/dxy.cpp
@@ -1,7 +1,23 @@
#include "dxy.h"
#include "dataStructs.h"
+#include "dmat.h"
-static dxy_t* dxy_init(metadataStruct* mtd) {
+void dxy_destroy(dxy_t* dxy) {
+ FREE(dxy->d);
+ FREE(dxy->g1names_p);
+ FREE(dxy->g2names_p);
+ FREE(dxy->levelnames_p);
+ for (size_t i = 0;i < dxy->nLevels;++i) {
+ if (dxy->dm[i] != NULL) {
+ dmat_destroy(dxy->dm[i]);
+ }
+ }
+ FREE(dxy->dm);
+ FREE(dxy);
+ return;
+}
+
+ dxy_t* dxy_init(metadataStruct* mtd) {
dxy_t* dxy = NULL;
dxy = (dxy_t*)malloc(sizeof(dxy_t));
ASSERT(dxy != NULL);
diff --git a/dxy.h b/dxy.h
index bc54159..537cc49 100644
--- a/dxy.h
+++ b/dxy.h
@@ -6,6 +6,9 @@
#include "mathUtils.h"
+//TODO use dmats only, rm dxy data d labels etc
+// ur not using the dmats right now
+//TODO add test
/// @brief dxy_t - structure for dxy analysis results
///
@@ -61,20 +64,7 @@ typedef struct dxy_t {
dxy_t* dxy_read(paramStruct* pars, dmat_t* dmat, metadataStruct* mtd);
dxy_t* dxy_get(paramStruct* pars, dmat_t* dmat, metadataStruct* mtd);
-inline void dxy_destroy(dxy_t* dxy) {
- FREE(dxy->d);
- FREE(dxy->g1names_p);
- FREE(dxy->g2names_p);
- FREE(dxy->levelnames_p);
- for (size_t i = 0;i < dxy->nLevels;++i) {
- if (dxy->dm[i] != NULL) {
- dmat_destroy(dxy->dm[i]);
- }
- }
- FREE(dxy->dm);
- FREE(dxy);
- return;
-}
+void dxy_destroy(dxy_t* dxy);
void dxy_print(dxy_t* dxy);
diff --git a/em.cpp b/em.cpp
index 7ce5fd2..8795ad0 100755
--- a/em.cpp
+++ b/em.cpp
@@ -1,286 +1,299 @@
#include "em.h"
-typedef struct ptdata_t ptdata_t;
-struct ptdata_t {
- size_t i1pos;
- size_t i2pos;
- size_t pairidx;
- double* em;
+static double tole;
+static size_t max_nsites;
+static int max_iter;
+static size_t* block_start_siteidx;
+static size_t nBlocks;
+static bool doBlocks;
+static int maxnThreads;
+
+typedef struct ptdata_em_t ptdata_em_t;
+struct ptdata_em_t {
+ size_t i1;
+ size_t i2;
double* pm;
- vcfData* vcfd;
+ gldata_t* gldata;
};
-int em_optimize_jgtmat9(ptdata_t* tdata) {
+/// @return reason for termination (1:tole, 2:max_iter)
+static int em_optim_jgtmat9(const size_t i1, const size_t i2, double* pm, gldata_t* gldata) {
- vcfData* vcfd = tdata->vcfd;
- const size_t i1pos = tdata->i1pos;
- const size_t i2pos = tdata->i2pos;
- const size_t pairidx = tdata->pairidx;
+ uint8_t reason = -1;
- double* em = tdata->em;
- double* pm = tdata->pm;
+ double** i1sites = NULL;
+ double** i2sites = NULL;
- double* sitelngl = NULL;
+ i1sites = gldata->d[i1];
+ i2sites = gldata->d[i2];
- const double tole = args->tole;
+ bool* mis1 = gldata->mis[i1];
+ bool* mis2 = gldata->mis[i2];
+ int shared_nSites = 0;
- const int max_iter = args->maxEmIter;
+ double TMP[9] = { 0.0 };
+ double ESFS[9] = { 0.0 };
- double sum = 0.0;
- double d = 0.0;
- double tmp = 0.0;
+ double sum, d, tmp;
+ size_t s, k, ki, k1, k2;
int n_iter = 0;
- // #shared sites for this pair
- int shared_nSites = 0;
- double TMP[9] = { 0.0 };
- double ESFS[9] = { 0.0 };
+ double* i1p;
+ double* i2p;
- double i1gl0 = NEG_INF;
- double i1gl1 = NEG_INF;
- double i1gl2 = NEG_INF;
- double i2gl0 = NEG_INF;
- double i2gl1 = NEG_INF;
- double i2gl2 = NEG_INF;
-
- // em iteration 0
- // starts before while to get shared_nSites without loop cost
- for (size_t s = 0; s < (size_t)vcfd->nSites; ++s) {
-
- sitelngl = vcfd->lngl->d[s];
- // 0 <- (a1,a1)
- // 1 <- (a1,a2) or (a2,a1)
- // 2 <- (a2,a2)
- //
- // ind2
- // 0 1 2
- // -----------------
- // ind1 0 | 0 3 6
- // 1 | 1 4 7
- // 2 | 2 5 8
- //
- //
-
- if (NEG_INF == (i1gl0 = sitelngl[i1pos])) {
- continue;
- } else if (NEG_INF == (i1gl1 = sitelngl[i1pos + 1])) {
- continue;
- } else if (NEG_INF == (i1gl2 = sitelngl[i1pos + 2])) {
- continue;
- } else if (NEG_INF == (i2gl0 = sitelngl[i2pos])) {
- continue;
- } else if (NEG_INF == (i2gl1 = sitelngl[i2pos + 1])) {
- continue;
- } else if (NEG_INF == (i2gl2 = sitelngl[i2pos + 2])) {
+
+ // --> first loop
+ for (s = 0;s < max_nsites;++s) {
+
+ if (mis1[s] || mis2[s]) {
continue;
}
- shared_nSites++;
+ i1p = i1sites[s];
+ i2p = i2sites[s];
- // fprintf(stdout, "\n");
- // for (int i = 0;i < 9;++i) {
- // fprintf(stdout, "%f", TMP[i]);
- // if (i != 8)
- // fprintf(stdout, ",");
- // }
-
- // expectation
- TMP[0] = em[0] * exp(i1gl0 + i2gl0);
- TMP[1] = em[1] * exp(i1gl1 + i2gl0);
- TMP[2] = em[2] * exp(i1gl2 + i2gl0);
- TMP[3] = em[3] * exp(i1gl0 + i2gl1);
- TMP[4] = em[4] * exp(i1gl1 + i2gl1);
- TMP[5] = em[5] * exp(i1gl2 + i2gl1);
- TMP[6] = em[6] * exp(i1gl0 + i2gl2);
- TMP[7] = em[7] * exp(i1gl1 + i2gl2);
- TMP[8] = em[8] * exp(i1gl2 + i2gl2);
- sum = TMP[0] + TMP[1] + TMP[2] + TMP[3] + TMP[4] + TMP[5] + TMP[6] + TMP[7] + TMP[8];
- ESFS[0] += TMP[0] / sum;
- ESFS[1] += TMP[1] / sum;
- ESFS[2] += TMP[2] / sum;
- ESFS[3] += TMP[3] / sum;
- ESFS[4] += TMP[4] / sum;
- ESFS[5] += TMP[5] / sum;
- ESFS[6] += TMP[6] / sum;
- ESFS[7] += TMP[7] / sum;
- ESFS[8] += TMP[8] / sum;
+ ki = 0;
+ sum = 0.0;
+ for (k1 = 0; k1 < 3;++k1) {
+ for (k2 = 0; k2 < 3;++k2) {
+ TMP[ki] = pm[ki] * exp(i1p[k1] + i2p[k2]);
+ sum += TMP[ki];
+ ++ki;
+ }
+ }
+
+ for (k = 0;k < 9;++k) {
+ ESFS[k] += TMP[k] / sum;
+ }
+
+ shared_nSites++;
} // end sites loop
+ // <- first loop
+
+
+ if (shared_nSites == 0) {
+ ERROR("Individuals %ld and %ld have no sites shared. This is currently not allowed.", i1, i2);
+ //TODO check for this in GT too
+ }
- // maximization
- d = 0.0;
- tmp = ESFS[0] / shared_nSites;
- d += fabs(tmp - em[0]);
- em[0] = tmp;
- ESFS[0] = 0.0;
- tmp = ESFS[1] / shared_nSites;
- d += fabs(tmp - em[1]);
- em[1] = tmp;
- ESFS[1] = 0.0;
- tmp = ESFS[2] / shared_nSites;
- d += fabs(tmp - em[2]);
- em[2] = tmp;
- ESFS[2] = 0.0;
- tmp = ESFS[3] / shared_nSites;
- d += fabs(tmp - em[3]);
- em[3] = tmp;
- ESFS[3] = 0.0;
- tmp = ESFS[4] / shared_nSites;
- d += fabs(tmp - em[4]);
- em[4] = tmp;
- ESFS[4] = 0.0;
- tmp = ESFS[5] / shared_nSites;
- d += fabs(tmp - em[5]);
- em[5] = tmp;
- ESFS[5] = 0.0;
- tmp = ESFS[6] / shared_nSites;
- d += fabs(tmp - em[6]);
- em[6] = tmp;
- ESFS[6] = 0.0;
- tmp = ESFS[7] / shared_nSites;
- d += fabs(tmp - em[7]);
- em[7] = tmp;
- ESFS[7] = 0.0;
- tmp = ESFS[8] / shared_nSites;
- d += fabs(tmp - em[8]);
- em[8] = tmp;
- ESFS[8] = 0.0;
-
- ++n_iter;
-
-
-
- while (d > tole) {
+
+ while (1) {
+
+ // -> maximization
d = 0.0;
+ for (k = 0;k < 9;++k) {
+ tmp = ESFS[k] / shared_nSites;
+ d += fabs(tmp - pm[k]);
+ pm[k] = tmp;
+ ESFS[k] = 0.0;
+ }
+ ++n_iter;
+ if (d <= tole) {
+ reason = EM_TERM_REASON_TOLE;
+ break;
+ }
if (n_iter == max_iter) {
+ reason = EM_TERM_REASON_ITER;
break;
}
- for (size_t s = 0; s < (size_t)vcfd->nSites; ++s) {
+ // -> expectation
+ for (s = 0;s < max_nsites;++s) {
- sitelngl = vcfd->lngl->d[s];
- if (NEG_INF == (i1gl0 = sitelngl[i1pos])) {
- continue;
- } else if (NEG_INF == (i1gl1 = sitelngl[i1pos + 1])) {
- continue;
- } else if (NEG_INF == (i1gl2 = sitelngl[i1pos + 2])) {
- continue;
- } else if (NEG_INF == (i2gl0 = sitelngl[i2pos])) {
- continue;
- } else if (NEG_INF == (i2gl1 = sitelngl[i2pos + 1])) {
- continue;
- } else if (NEG_INF == (i2gl2 = sitelngl[i2pos + 2])) {
+ if (mis1[s] || mis2[s]) {
continue;
}
+ i1p = i1sites[s];
+ i2p = i2sites[s];
+
+ ki = 0;
+ sum = 0.0;
+ for (k1 = 0; k1 < 3;++k1) {
+ for (k2 = 0; k2 < 3;++k2) {
+ TMP[ki] = pm[ki] * exp(i1p[k1] + i2p[k2]);
+ sum += TMP[ki];
+ ++ki;
+ }
+ }
- // expectation
- TMP[0] = em[0] * exp(i1gl0 + i2gl0);
- TMP[1] = em[1] * exp(i1gl1 + i2gl0);
- TMP[2] = em[2] * exp(i1gl2 + i2gl0);
- TMP[3] = em[3] * exp(i1gl0 + i2gl1);
- TMP[4] = em[4] * exp(i1gl1 + i2gl1);
- TMP[5] = em[5] * exp(i1gl2 + i2gl1);
- TMP[6] = em[6] * exp(i1gl0 + i2gl2);
- TMP[7] = em[7] * exp(i1gl1 + i2gl2);
- TMP[8] = em[8] * exp(i1gl2 + i2gl2);
- sum = TMP[0] + TMP[1] + TMP[2] + TMP[3] + TMP[4] + TMP[5] + TMP[6] + TMP[7] + TMP[8];
- ESFS[0] += TMP[0] / sum;
- ESFS[1] += TMP[1] / sum;
- ESFS[2] += TMP[2] / sum;
- ESFS[3] += TMP[3] / sum;
- ESFS[4] += TMP[4] / sum;
- ESFS[5] += TMP[5] / sum;
- ESFS[6] += TMP[6] / sum;
- ESFS[7] += TMP[7] / sum;
- ESFS[8] += TMP[8] / sum;
+ for (k = 0;k < 9;++k) {
+ ESFS[k] += TMP[k] / sum;
+ }
} // end sites loop
+ }
+ return(reason);
+
+}
- // maximization
- d = 0.0;
- tmp = ESFS[0] / shared_nSites;
- d += fabs(tmp - em[0]);
- em[0] = tmp;
- ESFS[0] = 0.0;
- tmp = ESFS[1] / shared_nSites;
- d += fabs(tmp - em[1]);
- em[1] = tmp;
- ESFS[1] = 0.0;
- tmp = ESFS[2] / shared_nSites;
- d += fabs(tmp - em[2]);
- em[2] = tmp;
- ESFS[2] = 0.0;
- tmp = ESFS[3] / shared_nSites;
- d += fabs(tmp - em[3]);
- em[3] = tmp;
- ESFS[3] = 0.0;
- tmp = ESFS[4] / shared_nSites;
- d += fabs(tmp - em[4]);
- em[4] = tmp;
- ESFS[4] = 0.0;
- tmp = ESFS[5] / shared_nSites;
- d += fabs(tmp - em[5]);
- em[5] = tmp;
- ESFS[5] = 0.0;
- tmp = ESFS[6] / shared_nSites;
- d += fabs(tmp - em[6]);
- em[6] = tmp;
- ESFS[6] = 0.0;
- tmp = ESFS[7] / shared_nSites;
- d += fabs(tmp - em[7]);
- em[7] = tmp;
- ESFS[7] = 0.0;
- tmp = ESFS[8] / shared_nSites;
- d += fabs(tmp - em[8]);
- em[8] = tmp;
- ESFS[8] = 0.0;
- ++n_iter;
+typedef struct ptdata_em_bootstrap_rep_t ptdata_em_bootstrap_rep_t;
+struct ptdata_em_bootstrap_rep_t {
+ size_t i1;
+ size_t i2;
+ double* pm;
+ gldata_t* gldata;
+ bblocks_t* bblocks;
+
+ size_t* rblocks;
+
+ uint8_t reason; // reason for termination. 1:tole, 2:max_iter
+};
+
+
+
+static int em_optim_jgtmat9_bootstrap_rep(const size_t i1, const size_t i2, double* pm, gldata_t* gldata, const size_t* const rblocks) {
+
+ uint8_t reason = -1;
+
+ double** i1sites = NULL;
+ double** i2sites = NULL;
+ double* i1p = NULL;
+ double* i2p = NULL;
+
+ i1sites = gldata->d[i1];
+ i2sites = gldata->d[i2];
+
+ const bool* const mis1 = gldata->mis[i1];
+ const bool* const mis2 = gldata->mis[i2];
+
+
+ double TMP[9] = { 0.0 };
+ double ESFS[9] = { 0.0 };
+
+ double sum, d, tmp;
+ size_t b, s, wb, k, ki, k1, k2, n_iter;
+
+ size_t block_start, block_end;
+
+ int shared_nSites = 0;
+ n_iter = 0;
+
+
+ // -> first loop
+ for (b = 0;b < nBlocks;++b) {
+ wb = rblocks[b];
+ block_start = block_start_siteidx[wb];
+ block_end = (wb == nBlocks - 1) ? max_nsites : block_start_siteidx[wb + 1];
+ // DEVPRINT("%ld-th sampled block (%ld) = [%ld, %ld)", b, wb, block_start, block_end);
+
+ for (s = block_start;s < block_end;++s) {
+
+ if (mis1[s] || mis2[s]) {
+ continue;
+ }
+ i1p = i1sites[s];
+ i2p = i2sites[s];
+
+
+
+ ki = 0;
+ sum = 0.0;
+ for (k1 = 0; k1 < 3;++k1) {
+ for (k2 = 0; k2 < 3;++k2) {
+ TMP[ki] = pm[ki] * exp(i1p[k1] + i2p[k2]);
+ sum += TMP[ki];
+ ++ki;
+ }
+ }
+
+ for (k = 0;k < 9;++k) {
+ ESFS[k] += TMP[k] / sum;
+ }
+
+ shared_nSites++;
+
+
+ } // end sites loop
+ // <- first loop
}
if (shared_nSites == 0) {
- ERROR("Pair %ld has no sites shared. This is currently not allowed.", pairidx);
- //TODO drop pair instead?
- //TODO check for this in GT too
+ ERROR("Individuals %ld and %ld have no sites shared. This is currently not allowed.", i1, i2);
+ }
+
+
+ while (1) {
+
+ // -> maximization
+ d = 0.0;
+ for (k = 0;k < 9;++k) {
+ tmp = ESFS[k] / shared_nSites;
+ d += fabs(tmp - pm[k]);
+ pm[k] = tmp;
+ ESFS[k] = 0.0;
+ }
+ ++n_iter;
+
+ if (d <= tole) {
+ reason = EM_TERM_REASON_TOLE;
+ break;
+ }
+ if (n_iter == max_iter) {
+ reason = EM_TERM_REASON_ITER;
+ break;
+ }
+
+
+ for (b = 0;b < nBlocks;++b) {
+ wb = rblocks[b];
+ block_start = block_start_siteidx[wb];
+ block_end = (wb == nBlocks - 1) ? max_nsites : block_start_siteidx[wb + 1];
+
+ for (s = block_start;s < block_end;++s) {
+
+ if (mis1[s] || mis2[s]) {
+ continue;
+ }
+ i1p = i1sites[s];
+ i2p = i2sites[s];
+
+ ki = 0;
+ sum = 0.0;
+ for (k1 = 0; k1 < 3;++k1) {
+ for (k2 = 0; k2 < 3;++k2) {
+ TMP[ki] = pm[ki] * exp(i1p[k1] + i2p[k2]);
+ sum += TMP[ki];
+ ++ki;
+ }
+ }
+
+ for (k = 0;k < 9;++k) {
+ ESFS[k] += TMP[k] / sum;
+ }
+
+ } // end in-block sites loop
+ } // end blocks loop
+
}
+ return(reason);
+
+}
- pm[0] = em[0];
- em[0] = em[0] * shared_nSites;
- pm[1] = em[1];
- em[1] = em[1] * shared_nSites;
- pm[2] = em[2];
- em[2] = em[2] * shared_nSites;
- pm[3] = em[3];
- em[3] = em[3] * shared_nSites;
- pm[4] = em[4];
- em[4] = em[4] * shared_nSites;
- pm[5] = em[5];
- em[5] = em[5] * shared_nSites;
- pm[6] = em[6];
- em[6] = em[6] * shared_nSites;
- pm[7] = em[7];
- em[7] = em[7] * shared_nSites;
- pm[8] = em[8];
- em[8] = em[8] * shared_nSites;
-
- return(0);
+void* t_em_optim_jgtmat9(void* data) {
+ ptdata_em_t* tdata = (ptdata_em_t*)data;
+ if (-1 == em_optim_jgtmat9(tdata->i1, tdata->i2, tdata->pm, tdata->gldata)) {
+ NEVER;
+ }
+ return(NULL);
}
-void* t_em_optimize_jgtmat9(void* data) {
- ptdata_t* tdata = (ptdata_t*)data;
- if (0 != em_optimize_jgtmat9(tdata)) {
+void* t_em_optim_jgtmat9_bootstrap_rep(void* data) {
+ ptdata_em_bootstrap_rep_t* tdata = (ptdata_em_bootstrap_rep_t*)data;
+ if (-1 == em_optim_jgtmat9_bootstrap_rep(tdata->i1, tdata->i2, tdata->pm, tdata->gldata, tdata->rblocks)) {
NEVER;
}
return(NULL);
@@ -288,40 +301,43 @@ void* t_em_optimize_jgtmat9(void* data) {
-void spawnThreads_em_optimize_jgtmat(jgtmat_t* jgtm, paramStruct* pars, vcfData* vcfd) {
+void jgtmat_get_run_em_optim(jgtmat_t* jgtm, paramStruct* pars, vcfData* vcfd, bblocks_t* bblocks) {
+ tole = args->tole;
+ max_iter = args->maxEmIter;
+ max_nsites = vcfd->gldata->size;
+ doBlocks = (bblocks != NULL);
+ maxnThreads = (args->nThreads > 0) ? args->nThreads : 1;
+ const int nInd = pars->names->len;
+ size_t nPairs = (size_t)((nInd * (nInd - 1)) / 2);
- const int nInd = pars->nInd;
- const size_t nRuns = (size_t)(nInd * (nInd - 1) / 2);
+ // -> main run
- pthread_t threads[nRuns];
- ptdata_t tdata[nRuns];
- const int ngt = vcfd->nGT;
+ pthread_t threads[nPairs];
+ ptdata_em_t tdata[nPairs];
size_t pairidx = 0;
- for (size_t i1 = 0; i1 < (size_t)pars->nInd;++i1) {
+
+ for (size_t i1 = 0; i1 < nInd;++i1) {
+
for (size_t i2 = 0;i2 < i1;++i2) {
- tdata[pairidx].vcfd = vcfd;
- tdata[pairidx].pairidx = pairidx;
- tdata[pairidx].i1pos = ngt * i1;
- tdata[pairidx].i2pos = ngt * i2;
- tdata[pairidx].em = jgtm->em[pairidx];
+
+ tdata[pairidx].i1 = i1;
+ tdata[pairidx].i2 = i2;
tdata[pairidx].pm = jgtm->pm[pairidx];
+ tdata[pairidx].gldata = vcfd->gldata;
+
++pairidx;
}
}
- const int maxnThreads = (args->nThreads == 0) ? 1 : args->nThreads;
-
int nJobsAlive = 0;
size_t run_to_wait = 0;
size_t runidx = 0;
- ASSERT(maxnThreads > 0);
-
- while (runidx < nRuns) {
+ while (runidx < nPairs) {
while (1) {
if (nJobsAlive < maxnThreads) {
@@ -337,7 +353,8 @@ void spawnThreads_em_optimize_jgtmat(jgtmat_t* jgtm, paramStruct* pars, vcfData*
}
}
- if (0 != pthread_create(&threads[runidx], NULL, t_em_optimize_jgtmat9, &tdata[runidx])) {
+
+ if (0 != pthread_create(&threads[runidx], NULL, t_em_optim_jgtmat9, &tdata[runidx])) {
ERROR("Problem with the spawning thread.");
}
nJobsAlive++;
@@ -354,10 +371,111 @@ void spawnThreads_em_optimize_jgtmat(jgtmat_t* jgtm, paramStruct* pars, vcfData*
}
+ pairidx = 0;
+
return;
}
+void jgtmat_get_run_em_optim_bootstrap_reps(jgtmat_t* jgtm, paramStruct* pars, vcfData* vcfd, bblocks_t* bblocks) {
+
+
+ tole = args->tole;
+ max_iter = args->maxEmIter;
+ max_nsites = vcfd->gldata->size;
+ doBlocks = (bblocks != NULL);
+ nBlocks = (doBlocks) ? bblocks->n_blocks : 1;
+
+ block_start_siteidx = (doBlocks) ? bblocks->block_start_siteidx : NULL;
+
+ maxnThreads = (args->nThreads > 0) ? args->nThreads : 1;
+ const int nInd = pars->names->len;
+ size_t nPairs = (size_t)((nInd * (nInd - 1)) / 2);
+
+ size_t pairidx;
+
+ // -> brep runs
+
+
+ const size_t nReps = args->nBootstraps;
+ ASSERT(args->nBootstraps > 0);
+
+ const size_t nRuns = nPairs * nReps;
+
+ pthread_t brepthreads[nRuns];
+ ptdata_em_bootstrap_rep_t breptdata[nRuns];
+
+ size_t repidx;
+ size_t allidx; // idx among all (incl. the original run)
+ for (size_t rep = 0;rep < nReps;++rep) {
+
+ repidx = rep * nPairs;
+ allidx = (rep + 1) * nPairs;
+
+ pairidx = 0;
+
+ for (size_t i1 = 0; i1 < nInd;++i1) {
+
+ for (size_t i2 = 0;i2 < i1;++i2) {
+
+
+ breptdata[repidx + pairidx].i1 = i1;
+ breptdata[repidx + pairidx].i2 = i2;
+ breptdata[repidx + pairidx].pm = jgtm->pm[allidx + pairidx];
+ breptdata[repidx + pairidx].gldata = vcfd->gldata;
+
+ breptdata[repidx + pairidx].bblocks = bblocks;
+ breptdata[repidx + pairidx].rblocks = bblocks->rblocks[rep];
+ breptdata[repidx + pairidx].reason = 0;
+
+
+ ++pairidx;
+ }
+ }
+ }
+
+
+ int nJobsAlive = 0;
+ size_t run_to_wait = 0;
+ size_t runidx = 0;
+
+
+ while (runidx < nRuns) {
+
+ while (1) {
+ if (nJobsAlive < maxnThreads) {
+ break;
+ }
+ while (nJobsAlive >= maxnThreads) {
+ // wait for the run that was sent first
+ if (0 != pthread_join(brepthreads[run_to_wait], NULL)) {
+ ERROR("Problem with joining the thread.");
+ }
+ ++run_to_wait;
+ nJobsAlive--;
+ }
+ }
+
+
+ if (0 != pthread_create(&brepthreads[runidx], NULL, t_em_optim_jgtmat9_bootstrap_rep, &breptdata[runidx])) {
+ ERROR("Problem with the spawning thread.");
+ }
+ nJobsAlive++;
+
+ ++runidx;
+ }
+
+ while (nJobsAlive > 0) {
+ if (0 != pthread_join(brepthreads[run_to_wait], NULL)) {
+ ERROR("Problem with joining the thread.");
+ }
+ ++run_to_wait;
+ nJobsAlive--;
+ }
+
+
+ return;
+}
diff --git a/em.h b/em.h
index be54aa3..1c7f4bc 100755
--- a/em.h
+++ b/em.h
@@ -6,9 +6,10 @@
#include "argStruct.h"
#include "mathUtils.h"
-void spawnThreads_em_optimize_jgtmat(jgtmat_t* jgtmat, paramStruct* pars, vcfData* vcfd);
-
-
+#define EM_TERM_REASON_TOLE 1
+#define EM_TERM_REASON_ITER 2
+void jgtmat_get_run_em_optim(jgtmat_t* jgtm, paramStruct* pars, vcfData* vcfd, bblocks_t* bblocks);
+void jgtmat_get_run_em_optim_bootstrap_reps(jgtmat_t* jgtm, paramStruct* pars, vcfData* vcfd, bblocks_t* bblocks);
#endif // __EM__
\ No newline at end of file
diff --git a/ibd.cpp b/ibd.cpp
index 50d9477..be1621e 100644
--- a/ibd.cpp
+++ b/ibd.cpp
@@ -8,7 +8,7 @@ const int dosePairIndex[3][3] = { {0,1,5}, {1,2,3}, {5,3,4} };
void readSites_doIbd(vcfData* vcfd, paramStruct* pars) {
- const int nInd = pars->nInd;
+ const int nInd = pars->names->len;
int skip_site = 0;
int contig_i = -1;
@@ -157,7 +157,7 @@ void readSites_doIbd(vcfData* vcfd, paramStruct* pars) {
if (n_values <= 0) {
ERROR("Could not read GL tag from the VCF file.");
}
- n_gls = n_values / pars->nInd;
+ n_gls = n_values / pars->names->len;
// if (n_gls < ngt_to_use) {
@@ -490,7 +490,7 @@ int get_ibd_segments(const char* contigName, paramStruct* pars, int* site2pos) {
double thisSum = 0.0;
double thisScore = 0.0;
int pair_idx = -1;
- const int nInd = pars->nInd;
+ const int nInd = pars->names->len;
for (int i1 = 0; i1 < nInd; ++i1) {
for (int i2 = i1 + 1; i2 < nInd; ++i2) {
@@ -598,7 +598,7 @@ int trimmedStart(paramStruct* pars, const int i1, const int i2, const int start,
NEVER;
pair_idx = -1;
} else {
- pair_idx = nCk_idx(pars->nInd, i1, i2);
+ pair_idx = nCk_idx(pars->names->len, i1, i2);
}
while (index <= end && sum < args->ibdseq_ibdtrim) {
@@ -626,7 +626,7 @@ int trimmedEnd(paramStruct* pars, const int i1, const int i2, const int start, c
NEVER;
pair_idx = -1;
} else {
- pair_idx = nCk_idx(pars->nInd, i1, i2);
+ pair_idx = nCk_idx(pars->names->len, i1, i2);
}
while (index >= start && sum < args->ibdseq_ibdtrim) {
@@ -972,7 +972,7 @@ double ibdStruct::hbdScore(int dose, double fB) {
ibdStruct::ibdStruct(vcfData* vcfd, paramStruct* pars) {
- this->nInd = pars->nInd;
+ this->nInd = pars->names->len;
if (args->doIbd == 2) {
this->maxErrorArray = errorArray(args->ibdseq_errormax);
}
@@ -991,8 +991,8 @@ ibdStruct::ibdStruct(vcfData* vcfd, paramStruct* pars) {
this->pairScores[i] = (double*)malloc(BUF_NSITES * sizeof(double));
}
- // this->selfScores = (double**) malloc(pars->nInd * sizeof(double*));
- // for (size_t i=0;i<pars->nInd;++i){
+ // this->selfScores = (double**) malloc(pars->names->len * sizeof(double*));
+ // for (size_t i=0;i<pars->names->len;++i){
// this->selfScores[i]=(double*)malloc(BUF_NSITES*sizeof(double));
// }
diff --git a/io.cpp b/io.cpp
index f7057cf..a212569 100644
--- a/io.cpp
+++ b/io.cpp
@@ -605,11 +605,6 @@ void IO::outFilesStruct_init(IO::outFilesStruct* ofs) {
ofs->out_v_bootstrapRep_fs = new IO::outputStruct(args->out_fnp, ".verbose_bootstrap_replicates.csv", OUTFC::NONE);
}
}
-
- // TODO DEPREC
- if (args->printDev == 1) {
- ofs->out_dev_fs = new IO::outputStruct(args->out_fnp, ".dev.csv", 1);
- }
}
void IO::outFilesStruct_destroy(IO::outFilesStruct* ofs) {
diff --git a/jgtmat.cpp b/jgtmat.cpp
new file mode 100644
index 0000000..e69de29
diff --git a/jgtmat.h b/jgtmat.h
new file mode 100644
index 0000000..8305717
--- /dev/null
+++ b/jgtmat.h
@@ -0,0 +1,50 @@
+#ifndef __JGTMAT_H__
+#define __JGTMAT_H__
+
+/* INCLUDES ----------------------------------------------------------------- */
+#include "dataStructs.h"
+/* -------------------------------------------------------------------------- */
+
+
+/* FORWARD DECLARATIONS ----------------------------------------------------- */
+typedef struct dmat_t dmat_t;
+/* -------------------------------------------------------------------------- */
+
+// JGTMAT9
+// calculations based on 3x3 matrix
+// of pairwise genotype categories
+// -------------------------------------
+//
+// MM Mm mm
+// MM A D G
+// Mm B E H
+// mm C F I
+//
+// linearized version:
+// A D G B E H C F I
+// 0 1 2 3 4 5 6 7 8
+//
+//
+#define JGTMAT_LINEAR_IDXOF_A 0
+#define JGTMAT_LINEAR_IDXOF_B 3
+#define JGTMAT_LINEAR_IDXOF_C 6
+#define JGTMAT_LINEAR_IDXOF_D 1
+#define JGTMAT_LINEAR_IDXOF_E 4
+#define JGTMAT_LINEAR_IDXOF_F 7
+#define JGTMAT_LINEAR_IDXOF_G 2
+#define JGTMAT_LINEAR_IDXOF_H 5
+#define JGTMAT_LINEAR_IDXOF_I 8
+
+
+#define JGTMAT_GET_DIJ (C + G + ((B + D + E + F + H) / 2.0))
+#define JGTMAT_GET_SIJ (A + I + ((B + D + E + F + H) / 2.0))
+#define JGTMAT_GET_FIJ (((2 * C) + (2 * G) - E) / ((2 * C) + (2 * G) + B + D + E + F + H))
+#define JGTMAT_GET_IBS0 (C + G)
+#define JGTMAT_GET_IBS1 (B + D + F + H)
+#define JGTMAT_GET_IBS2 (A + E + I)
+#define JGTMAT_GET_R0 ((C + G) / E)
+#define JGTMAT_GET_R1 (E / (C + G + B + D + F + H))
+#define JGTMAT_GET_KIN ((E - ((2 * C) + (2 * G))) / (B + D + F + H + (2 * E)))
+
+
+#endif // __JGTMAT_H__
\ No newline at end of file
diff --git a/jgtmat_dist.h b/jgtmat_dist.h
deleted file mode 100644
index cc58533..0000000
--- a/jgtmat_dist.h
+++ /dev/null
@@ -1,93 +0,0 @@
-#ifndef __JGTMAT_DIST_H__
-#define __JGTMAT_DIST_H__
-/// jgtmat_dist.h: calculate distances_from_jgtmat from jgtmat
-
-#include "dataStructs.h"
-//todo instead of datastructs, use:
-// jgtmat.h
-// dmat.h
-// etc main data types in sep hdrs
-
-
-// #include <math.h>
-// #include <stdio.h>
-// #include <limits>
-
-// typedef struct jgtmat_t jgtmat_t;
-
-
-// JGTMAT9
-// calculations based on 3x3 matrix
-// of pairwise genotype categories
-// -------------------------------------
-//
-// MM Mm mm
-// MM A D G
-// Mm B E H
-// mm C F I
-//
-// linearized version:
-// A D G B E H C F I
-// 0 1 2 3 4 5 6 7 8
-//
-//
-#define JGTMAT_LINEAR_IDXOF_A 0
-#define JGTMAT_LINEAR_IDXOF_B 3
-#define JGTMAT_LINEAR_IDXOF_C 6
-#define JGTMAT_LINEAR_IDXOF_D 1
-#define JGTMAT_LINEAR_IDXOF_E 4
-#define JGTMAT_LINEAR_IDXOF_F 7
-#define JGTMAT_LINEAR_IDXOF_G 2
-#define JGTMAT_LINEAR_IDXOF_H 5
-#define JGTMAT_LINEAR_IDXOF_I 8
-
-inline void dmat_get_distances_from_jgtmat(jgtmat_t* jgtmat, dmat_t* dmat) {
-
- double** pm = jgtmat->pm;
-
- double* dm = NULL;
- if (dmat->n == 1) {
- dm = dmat->matrix[0];
- } else {
- NEVER; //TODO add multi matrix handling? either add matrix idx to use or loop through all n matrices in here
- }
-
-
- uint32_t transform = dmat->transform;
-
- double x, A, B, D, E, F, H, I;
- size_t p; // pair idx
- double* ppm = NULL; // pair pm
- for (p = 0; p < dmat->size; ++p) {
-
- x = 0.0;
- ppm = pm[p];
- A = ppm[JGTMAT_LINEAR_IDXOF_A];
- I = ppm[JGTMAT_LINEAR_IDXOF_I];
- B = ppm[JGTMAT_LINEAR_IDXOF_B];
- D = ppm[JGTMAT_LINEAR_IDXOF_D];
- E = ppm[JGTMAT_LINEAR_IDXOF_E];
- F = ppm[JGTMAT_LINEAR_IDXOF_F];
- H = ppm[JGTMAT_LINEAR_IDXOF_H];
-
- x = 1.0 - (A + I + ((B + D + E + F + H) / 2.0));
-
- if (DMAT_TRANSFORM_NONE != transform) {
- if (DMAT_TRANSFORM_SQUARE == transform) {
- x *= x;
- } else {
- NEVER;//TODO investigate other transformation methods
- }
- }
-
- dm[p] = x;
- }
- return;
-}
-
-
-
-
-
-
-#endif // __JGTMAT_DIST_H__
diff --git a/mathUtils.cpp b/mathUtils.cpp
deleted file mode 100755
index 52eed35..0000000
--- a/mathUtils.cpp
+++ /dev/null
@@ -1,578 +0,0 @@
-#include "mathUtils.h"
-
-double MATH::MEAN(double* arr, int size) {
- double mean = 0.0;
- for (int x = 0; x < size; x++) {
- mean += arr[x];
- }
- return ((double)mean / (double)size);
-}
-
-double MATH::SD(double* arr, int size) {
- double mean = MATH::MEAN(arr, size);
- double sd = 0.0;
- for (int x = 0; x < size; x++) {
- sd += pow(arr[x] - mean, 2);
- }
- return sqrt(sd / size);
-}
-
-int nCk(int n, int k) {
- if (k == 0) {
- return 1;
- }
- return (n * nCk(n - 1, k - 1)) / k;
-}
-
-int nCk_idx(int nInd, int i1, int i2) {
- ASSERT(i1 < nInd && i2 < nInd); // safeguard for wrong order of arguments
- if (i2 > i1) {
- return (nCk(nInd, 2) - nCk((nInd - i1), 2)) + (i2 - i1) - 1;
- } else {
- return (nCk(nInd, 2) - nCk((nInd - i2), 2)) + (i1 - i2) - 1;
- }
-}
-
-int find_n_given_nC2(int nC2_res) {
- int n = 0;
-
- if (NC2_LUT[300] < nC2_res) {
- // cannot be found in LUT
- n = 301;
- while (nCk(n, 2) < nC2_res) {
- n++;
- ASSERT(n < 500);
- }
- return n;
- }
-
- while (NC2_LUT[n] < nC2_res) {
- n++;
- }
- if (NC2_LUT[n] != nC2_res) {
- ERROR("nC2_res:%d not found in NC2_LUT[]\n", nC2_res);
- }
- return n;
-}
-
-int nC2(const int n) {
- if (n < 301) {
- return (NC2_LUT[n]);
- } else {
- return (nCk(n, 2));
- }
-}
-
-// for (int i1 = 0; i1 < nInd - 1; i1++)
-// {
-// for (int i2 = i1 + 1; i2 < nInd; i2++)
-// {
-// int idx = nCk_idx(nInd, i1, i2);
-// LUTs[0][i1][i2] = idx;
-// LUTs[1][idx][0] = i1;
-// LUTs[1][idx][1] = i2;
-// }
-// }
-
-// calculations based on 3x3 matrix
-// of pairwise genotype categories
-// -------------------------------------
-//
-//
-// MM Mm mm
-// MM A D G
-// Mm B E H
-// mm C F I
-//
-//
-// 1x9 matrix equivalent:
-// A D G B E H C F I
-//
-//
-// A = M[0][0] = M[0]
-// D = M[0][1] = M[1]
-// G = M[0][2] = M[2]
-// B = M[1][0] = M[3]
-// E = M[1][1] = M[4]
-// H = M[1][2] = M[5]
-// C = M[2][0] = M[6]
-// F = M[2][1] = M[7]
-// I = M[2][2] = M[8]
-
-double MATH::SUM(double* M) {
- double sum = 0.0;
- for (int x = 0; x < 9; x++) {
- sum += M[x];
- }
- return sum;
-}
-
-double MATH::SUM(int* M) {
- double sum = 0.0;
- for (int x = 0; x < 9; x++) {
- sum += M[x];
- }
- return sum;
-}
-
-double MATH::MEAN(double M[3][3]) {
- double mean = 0.0;
- double N = 9.0;
- for (int x = 0; x < 3; x++) {
- for (int y = 0; y < 3; y++) {
- mean += M[x][y];
- }
- }
- mean = (double)mean / (double)N;
- return mean;
-}
-
-double MATH::MEAN(double* M) {
- double mean = 0.0;
- for (int x = 0; x < 9; x++) {
- mean += M[x];
- }
- return (double)mean / 9.0;
-}
-
-double MATH::MEAN(int* M) {
- double mean = 0.0;
- for (int x = 0; x < 9; x++) {
- mean += M[x];
- }
- return (double)mean / 9.0;
-}
-
-// VAR and SD is for sample (N-1)
-double MATH::VAR(double M[3][3]) {
- double i = 0.0;
- double N = 9.0;
- for (int x = 0; x < 3; x++) {
- for (int y = 0; y < 3; y++) {
- i += SQUARE((double)M[x][y] - (double)MATH::MEAN(M));
- }
- }
- return (double)i / (double)(N - 1);
-}
-
-double MATH::VAR(double* M) {
- double i = 0.0;
- double N = 9.0;
- for (int x = 0; x < N; x++) {
- i += SQUARE((double)M[x] - (double)MATH::MEAN(M));
- }
- return (double)i / (double)(N - 1);
-}
-
-double MATH::VAR(int* M) {
- double i = 0.0;
- double N = 9.0;
- for (int x = 0; x < N; x++) {
- i += SQUARE((double)M[x] - (double)MATH::MEAN(M));
- }
- return (double)i / (double)(N - 1);
-}
-
-double MATH::SD(double M[3][3]) {
- return sqrt(MATH::VAR(M));
-}
-
-double MATH::SD(double* M) {
- return sqrt(MATH::VAR(M));
-}
-
-double MATH::SD(int* M) {
- return sqrt(MATH::VAR(M));
-}
-
-double MATH::Sij(double M[3][3]) {
- double A = M[0][0];
- double I = M[2][2];
- double B = M[1][0];
- double D = M[0][1];
- double E = M[1][1];
- double F = M[2][1];
- double H = M[1][2];
-
- double x = A + I + ((B + D + E + F + H) / 2.0);
-
- return x;
-}
-
-double MATH::Sij(double* M) {
- double A = (double)M[0];
- double I = (double)M[8];
- double B = (double)M[3];
- double D = (double)M[1];
- double E = (double)M[4];
- double F = (double)M[7];
- double H = (double)M[5];
-
- double x = A + I + ((B + D + E + F + H) / 2.0);
-
- return x;
-}
-
-double MATH::Sij(int* M, int S) {
- double x = 0.0;
-
- double A = (double)M[0] / (double)S;
- double I = (double)M[8] / (double)S;
- double B = (double)M[3] / (double)S;
- double D = (double)M[1] / (double)S;
- double E = (double)M[4] / (double)S;
- double F = (double)M[7] / (double)S;
- double H = (double)M[5] / (double)S;
-
- x = A + I + ((B + D + E + F + H) / 2.0);
-
- return x;
-}
-
-// Dij = 1-Sij
-double MATH::Dij(double M[3][3]) {
- double A = M[0][0];
- double I = M[2][2];
- double B = M[1][0];
- double D = M[0][1];
- double E = M[1][1];
- double F = M[2][1];
- double H = M[1][2];
-
- double x = 1.0 - (A + I + ((B + D + E + F + H) / 2.0));
-
- return x;
-}
-
-double MATH::Dij(double* M) {
- double A = (double)M[0];
- double I = (double)M[8];
- double B = (double)M[3];
- double D = (double)M[1];
- double E = (double)M[4];
- double F = (double)M[7];
- double H = (double)M[5];
-
- double x = 1.0 - (A + I + ((B + D + E + F + H) / 2.0));
-
- return x;
-}
-
-//TODO ??
-double MATH::Dij(int* M, int S) {
- double x = 0.0;
-
- double A = (double)M[0] / (double)S;
- double I = (double)M[8] / (double)S;
- double B = (double)M[3] / (double)S;
- double D = (double)M[1] / (double)S;
- double E = (double)M[4] / (double)S;
- double F = (double)M[7] / (double)S;
- double H = (double)M[5] / (double)S;
-
- x = 1.0 - (A + I + ((B + D + E + F + H) / 2.0));
-
- return x;
-}
-
-double MATH::Fij(double M[3][3]) {
- double x = 0.0;
-
- double C = M[2][0];
- double G = M[0][2];
- double E = M[1][1];
- double B = M[1][0];
- double D = M[0][1];
- double F = M[2][1];
- double H = M[1][2];
-
- x = ((2 * C) + (2 * G) - E) / ((2 * C) + (2 * G) + B + D + E + F + H);
-
- return x;
-}
-
-double MATH::Fij(double* M) {
- double x = 0.0;
-
- double C = (double)M[6];
- double G = (double)M[2];
- double E = (double)M[4];
- double B = (double)M[3];
- double D = (double)M[1];
- double F = (double)M[7];
- double H = (double)M[5];
-
- x = ((2 * C) + (2 * G) - E) / ((2 * C) + (2 * G) + B + D + E + F + H);
-
- return x;
-}
-
-double MATH::Fij(int* M, int S) {
- double x = 0.0;
-
- double C = (double)M[6] / (double)S;
- double G = (double)M[2] / (double)S;
- double E = (double)M[4] / (double)S;
- double B = (double)M[3] / (double)S;
- double D = (double)M[1] / (double)S;
- double F = (double)M[7] / (double)S;
- double H = (double)M[5] / (double)S;
-
- x = ((2 * C) + (2 * G) - E) / ((2 * C) + (2 * G) + B + D + E + F + H);
-
- return x;
-}
-
-double MATH::IBS0(double M[3][3]) {
- double x = 0.0;
-
- double C = M[2][0];
- double G = M[0][2];
-
- x = C + G;
-
- return x;
-}
-
-double MATH::IBS0(double* M) {
- double x = 0.0;
-
- double C = (double)M[6];
- double G = (double)M[2];
-
- x = C + G;
-
- return x;
-}
-
-double MATH::IBS0(int* M, int S) {
- double x = 0.0;
-
- double C = (double)M[6] / (double)S;
- double G = (double)M[2] / (double)S;
-
- x = C + G;
-
- return x;
-}
-
-double MATH::IBS1(double M[3][3]) {
- double x = 0.0;
-
- double B = M[1][0];
- double D = M[0][1];
- double F = M[2][1];
- double H = M[1][2];
-
- x = B + D + F + H;
-
- return x;
-}
-
-double MATH::IBS1(double* M) {
- double x = 0.0;
-
- double B = (double)M[3];
- double D = (double)M[1];
- double F = (double)M[7];
- double H = (double)M[5];
-
- x = B + D + F + H;
-
- return x;
-}
-
-double MATH::IBS1(int* M, int S) {
- double x = 0.0;
-
- double B = (double)M[3] / (double)S;
- double D = (double)M[1] / (double)S;
- double F = (double)M[7] / (double)S;
- double H = (double)M[5] / (double)S;
-
- x = B + D + F + H;
-
- return x;
-}
-
-double MATH::IBS2(double M[3][3]) {
- double x = 0.0;
-
- double A = M[0][0];
- double E = M[1][1];
- double I = M[2][2];
-
- x = A + E + I;
-
- return x;
-}
-
-double MATH::IBS2(double* M) {
- double x = 0.0;
-
- double A = (double)M[0];
- double E = (double)M[4];
- double I = (double)M[8];
-
- x = A + E + I;
-
- return x;
-}
-
-double MATH::IBS2(int* M, int S) {
- double x = 0.0;
-
- double A = (double)M[0] / (double)S;
- double E = (double)M[4] / (double)S;
- double I = (double)M[8] / (double)S;
-
- x = A + E + I;
-
- return x;
-}
-
-double MATH::R0(double M[3][3]) {
- double x = 0.0;
-
- double C = M[2][0];
- double G = M[0][2];
- double E = M[1][1];
-
- x = (C + G) / E;
- // double xi=MATH::IBS0(M)/E;
- // fprintf(stderr,"\nr0->\t %f%f\n",x,xi);
-
- return x;
-}
-
-double MATH::R0(double* M) {
- double x = 0.0;
-
- double C = (double)M[6];
- double G = (double)M[2];
- double E = (double)M[4];
-
- x = (C + G) / E;
-
- return x;
-}
-
-double MATH::R0(int* M, int S) {
- double x = 0.0;
-
- double C = (double)M[6] / (double)S;
- double G = (double)M[2] / (double)S;
- double E = (double)M[4] / (double)S;
-
- x = (C + G) / E;
- // double xi=MATH::IBS0(M)/E;
- // fprintf(stderr,"\nr0->\t %f%f\n",x,xi);
-
- return x;
-}
-
-double MATH::R1(double M[3][3]) {
- double x = 0.0;
-
- double E = M[1][1];
- double C = M[2][0];
- double G = M[0][2];
- double B = M[1][0];
- double D = M[0][1];
- double F = M[2][1];
- double H = M[1][2];
-
- x = E / (C + G + B + D + F + H);
- // double xi=E/(MATH::IBS0(M) + MATH::IBS1(M));
- // fprintf(stderr,"\nr1->\t %f%f\n",x,xi);
-
- return x;
-}
-
-double MATH::R1(double* M) {
- double x = 0.0;
-
- double E = (double)M[4];
- double C = (double)M[6];
- double G = (double)M[2];
- double B = (double)M[3];
- double D = (double)M[1];
- double F = (double)M[7];
- double H = (double)M[5];
-
- x = E / (C + G + B + D + F + H);
-
- return x;
-}
-
-double MATH::R1(int* M, int S) {
- double x = 0.0;
-
- double E = (double)M[4] / (double)S;
- double C = (double)M[6] / (double)S;
- double G = (double)M[2] / (double)S;
- double B = (double)M[3] / (double)S;
- double D = (double)M[1] / (double)S;
- double F = (double)M[7] / (double)S;
- double H = (double)M[5] / (double)S;
-
- x = E / (C + G + B + D + F + H);
- // double xi = E / (MATH::IBS0(M) + MATH::IBS1(M));
- // fprintf(stderr, "\nr1->\t %f%f\n", x, xi);
-
- return x;
-}
-
-double MATH::Kin(double M[3][3]) {
- double x = 0.0;
-
- double E = M[1][1];
-
- double C = M[2][0];
- double G = M[0][2];
-
- double B = M[1][0];
- double D = M[0][1];
- double F = M[2][1];
- double H = M[1][2];
-
- x = (E - ((2 * C) + (2 * G))) / (B + D + F + H + (2 * E));
- // double xi= (E -(2*MATH::IBS0(M)))/(MATH::IBS1(M) + (2*E));
- // fprintf(stderr,"\nkin->\t %f%f\n",x,xi);
-
- return x;
-}
-
-double MATH::Kin(double* M) {
- double x = 0.0;
-
- double E = (double)M[4];
- double C = (double)M[6];
- double G = (double)M[2];
- double B = (double)M[3];
- double D = (double)M[1];
- double F = (double)M[7];
- double H = (double)M[5];
-
- x = (E - ((2 * C) + (2 * G))) / (B + D + F + H + (2 * E));
-
- return x;
-}
-
-double MATH::Kin(int* M, int S) {
- double x = 0.0;
-
- double E = (double)M[4] / (double)S;
- double C = (double)M[6] / (double)S;
- double G = (double)M[2] / (double)S;
- double B = (double)M[3] / (double)S;
- double D = (double)M[1] / (double)S;
- double F = (double)M[7] / (double)S;
- double H = (double)M[5] / (double)S;
-
- x = (E - ((2 * C) + (2 * G))) / (B + D + F + H + (2 * E));
- // double xi= (E -(2*MATH::IBS0(M)))/(MATH::IBS1(M) + (2*E));
- // fprintf(stderr,"\nkin->\t %f%f\n",x,xi);
-
- return x;
-}
diff --git a/mathUtils.h b/mathUtils.h
index b883f5f..68121bb 100755
--- a/mathUtils.h
+++ b/mathUtils.h
@@ -9,7 +9,7 @@
#include "shared.h"
#include "vcfReader.h"
-#define SQUARE(n) ((n) * (n))
+#define SQUARE(n) ((((n)) * ((n))))
/*
* Macro:[LOG2LN]
@@ -19,8 +19,6 @@
#define LN2LOG(x) ((((x) * (M_LOG10E))))
-void gl_log10(int base, double errate, double* like);
-
/*
* Binomial coefficient: n choose k
*
@@ -29,7 +27,14 @@ void gl_log10(int base, double errate, double* like);
* @param k
* @return choose(n,k)
*/
-int nCk(int n, int k);
+
+inline int nCk(int n, int k) {
+ if (k == 0) {
+ return 1;
+ }
+ return (n * nCk(n - 1, k - 1)) / k;
+}
+
/*
* [nCk_idx]
@@ -66,7 +71,16 @@ int nCk(int n, int k);
* Index of pair: (nCk(nInd, 2) - nCk((nInd - i1), 2)) + (i2 - i1) - 1;
*
*/
-int nCk_idx(int nInd, int i1, int i2);
+
+inline int nCk_idx(int nInd, int i1, int i2) {
+ ASSERT(i1 < nInd && i2 < nInd); // safeguard for wrong order of arguments
+ if (i2 > i1) {
+ return (nCk(nInd, 2) - nCk((nInd - i1), 2)) + (i2 - i1) - 1;
+ } else {
+ return (nCk(nInd, 2) - nCk((nInd - i2), 2)) + (i1 - i2) - 1;
+ }
+}
+
/*
* rand()
@@ -81,96 +95,6 @@ int nCk_idx(int nInd, int i1, int i2);
*
*/
-int find_n_given_nC2(int nC2_res);
-int nC2(int n);
-
-namespace MATH {
-
- double SUM(double M[3][3]);
- double SUM(double* M);
-
- double MEAN(double M[3][3]);
- double MEAN(double* M);
-
- double MEAN(double* arr, int size);
-
- double VAR(double M[3][3]);
- double VAR(double* M);
-
- double SD(double M[3][3]);
- double SD(double* M);
- double SD(double* arr, int size);
-
- double SUM(int* M);
- double MEAN(int* M);
- double VAR(int* M);
- double SD(int* M);
-
- /*
- *
- * [S_ij Similarity index]
- *
- * Similarity index (S_ij) based on the
- * probability of identity by descent
- *
- * 00 01 02 10 11 12 20 21 22
- * A D G B E H C F I
- *
- * A + I + ((B+D+E+F+H)/2)
- *
- */
-
- /*
- * Reference for Fij IBS0 IBS1 IBS2 R0 R1 Kin
- * https://doi.org/10.1111/mec.14954
- */
-
- /*
- *
- * [F_ij F statistic]
- *
- * 00 01 02 10 11 12 20 21 22
- * A D G B E H C F I
- *
- * (2C+2G-E) / (2C+2G+B+D+E+F+H)
- *
- */
- double Sij(double M[3][3]);
- double Sij(double* M);
- double Sij(int* M, int S);
-
- double Fij(double* M);
- double Fij(int* M, int S);
- double Fij(double M[3][3]);
-
- double IBS0(double* M);
- double IBS0(int* M, int S);
- double IBS0(double M[3][3]);
-
- double IBS1(double* M);
- double IBS1(int* M, int S);
- double IBS1(double M[3][3]);
-
- double IBS2(double* M);
- double IBS2(int* M, int S);
- double IBS2(double M[3][3]);
-
- double R0(double* M);
- double R0(int* M, int S);
- double R0(double M[3][3]);
-
- double R1(double* M);
- double R1(int* M, int S);
- double R1(double M[3][3]);
-
- double Kin(double* M);
- double Kin(int* M, int S);
- double Kin(double M[3][3]);
-
- double Dij(int* M, int S);
- double Dij(double* M);
- double Dij(double M[3][3]);
-
-} // namespace MATH
+
#endif // __MATH_UTILS__
diff --git a/neighborJoining.cpp b/neighborJoining.cpp
index 28dfb98..456bbb6 100644
--- a/neighborJoining.cpp
+++ b/neighborJoining.cpp
@@ -1,4 +1,5 @@
#include "neighborJoining.h"
+#include "dmat.h"
nj_t* nj_init(dmat_t* dmat, const size_t which_dmat) {
@@ -114,7 +115,7 @@ void nj_destroy(nj_t* nj) {
}
-static void nj_add_edge(nj_t* nj, int parentNode, int childNode, double edgeLength) {
+ void nj_add_edge(nj_t* nj, int parentNode, int childNode, double edgeLength) {
DEVASSERT(parentNode > childNode);
// DEVPRINT("adding edge. parentNode: %d childNode: %d edgeLength: %f nj->L: %d nj->nEdges: %d", parentNode, childNode, edgeLength, nj->L, nj->nEdges);
@@ -157,7 +158,7 @@ static void nj_add_edge(nj_t* nj, int parentNode, int childNode, double edgeLeng
// edgeNodes[edge_index][0] = parent node
// edgeNodes[edge_index][1] = child node
-static void nj_print_leaf_newick(nj_t* nj, int node, kstring_t* kbuf) {
+ void nj_print_leaf_newick(nj_t* nj, int node, kstring_t* kbuf) {
// node - index of the node in the list of all nodes
// L - number of leaf nodes
@@ -165,13 +166,13 @@ static void nj_print_leaf_newick(nj_t* nj, int node, kstring_t* kbuf) {
// <0 if node is a leaf node (i.e. not in the list of parent nodes)
// >=0 if node is an internal node (i.e. in the list of parent nodes)
- static const int L = nj->L;
+ const int L = nj->L;
const int nodeIdxInParents = node - L;
- static int* nEdgesPerParentNode = nj->nEdgesPerParentNode;
- static int** parentToEdgeIdx = nj->parentToEdgeIdx;
- static double* edgeLengths = nj->edgeLengths;
- static int** edgeNodes = nj->edgeNodes;
+ int* nEdgesPerParentNode = nj->nEdgesPerParentNode;
+ int** parentToEdgeIdx = nj->parentToEdgeIdx;
+ double* edgeLengths = nj->edgeLengths;
+ int** edgeNodes = nj->edgeNodes;
if (nodeIdxInParents < 0) {
// node := leaf node
diff --git a/ngsAMOVA.cpp b/ngsAMOVA.cpp
index 477ea8a..e434449 100755
--- a/ngsAMOVA.cpp
+++ b/ngsAMOVA.cpp
@@ -9,7 +9,6 @@
#include "dataStructs.h"
#include "dev.h"
#include "dxy.h"
-#include "em.h"
#include "io.h"
#include "mathUtils.h"
#include "neighborJoining.h"
@@ -17,12 +16,34 @@
#include "shared.h"
#include "vcfReader.h"
#include "ibd.h"
-#include "jgtmat_dist.h"
+#include "jgtmat.h"
+#include "dmat.h"
+#include "em.h"
void input_VCF(paramStruct* pars) {
- vcfData* vcfd = vcfData_init(pars);
+ // ---- READ VCF ------------------------------------------------------------- //
+
+ vcfData* vcfd = vcfData_init(pars, pars->metadata);
+
+
+ bblocks_t* bblocks = NULL;
+ if (PROGRAM_WILL_PERFORM_BLOCK_BOOTSTRAPPING) {
+ bblocks = bblocks_init();
+ bblocks_get(bblocks, vcfd, pars);
+
+ if (args->printBlocksTab) {
+ bblocks_print_blocks_tab(bblocks);
+ }
+
+ bblocks_sample_with_replacement(bblocks);
+ if(PROGRAM_VERBOSITY_LEVEL > 0){
+ bblocks_print_bootstrap_samples(bblocks);
+ }
+ }
+
+
if (0 != args->doIbd) {
pars->ibd = new ibdStruct(vcfd, pars);
@@ -30,72 +51,43 @@ void input_VCF(paramStruct* pars) {
return;
}
- metadataStruct* metadata = NULL;
- if (PROGRAM_NEEDS_METADATA) {
- if (NULL == args->in_mtd_fn) {
- ERROR("Requested analyses requiring metadata but no metadata file was provided.");
- } else {
- if (NULL != args->formula) {
- metadata = metadataStruct_read(pars);
- } else {
- NEVER; // this should already be checked in PROGRAM_NEEDS_FORMULA
- }
- }
- }
+ // ---- GET JOINT GENOTYPE MATRIX ------------------------------------------- //
- blobStruct* blobs = NULL;
- if (0 < args->nBootstraps) {
- blobs = blobStruct_get(vcfd, pars);
- }
+ if (args->doJGTM) {
- // ---- GET JGTM ------------------------------------------------------------ //
+ const size_t nIndCmb = (pars->names->len * (pars->names->len - 1)) / 2;
- if (args->doJGTM) {
+ const size_t nRuns = (args->nBootstraps > 0) ? (args->nBootstraps + 1) : 1;
- const size_t nIndCmb = (pars->nInd * (pars->nInd - 1)) / 2;
- pars->jgtm = jgtmat_init(nIndCmb);
- if (PROGRAM_WILL_USE_BCF_FMT_GL) {
- jgtmat_get_srcgl(pars->jgtm, pars, vcfd, blobs);
- } else if (PROGRAM_WILL_USE_BCF_FMT_GT) {
- jgtmat_get_srcgt(pars->jgtm, pars, vcfd, blobs);
- } else {
- NEVER;
+ pars->jgtm = jgtmat_init(nIndCmb * nRuns);
+
+ readSites(pars->jgtm, bblocks, vcfd, pars);
+ if (1 == args->doEM) {
+ jgtmat_get_run_em_optim(pars->jgtm, pars, vcfd, bblocks);
+ }
+
+ if (PROGRAM_WILL_PERFORM_BLOCK_BOOTSTRAPPING) {
+ jgtmat_get_run_em_optim_bootstrap_reps(pars->jgtm, pars, vcfd, bblocks);
}
+
}
// ---- GET DISTANCE MATRIX ------------------------------------------------- //
if (args->doDist) {
- uint8_t dm_type = DMAT_TYPE_LTED;
- uint32_t dm_method = DMAT_METHOD_DIJ;
- uint32_t dm_transform;
- //TODO
- dm_transform = DMAT_TRANSFORM_SQUARE;
-
-
- if (args->nBootstraps > 0) {
- // pars->multidm = multidmat_init(pars->nInd, dm_type, dm_method, dm_transform);
- // multidmat_get_distances_from_jgtmat(pars->jgtm, pars->multidm);
- // if (args->printDistanceMatrix) {
- // multidmat_print(pars->multidm);
- //TODO also allow for printing dmat for all boots reps
- // }
+ if (pars->metadata != NULL) {
+ pars->dm = dmat_init(pars->names->len, DMAT_TYPE_LTED, args->dm_method, args->dm_transform, pars->metadata->indNames, DMAT_NAMES_SRC_IN_METADATA_NAMES_PTR);
} else {
+ pars->dm = dmat_init(pars->names->len, DMAT_TYPE_LTED, args->dm_method, args->dm_transform, pars->names, DMAT_NAMES_SRC_IN_VCF_PARS_PTR);
+ }
- if (metadata != NULL) {
- pars->dm = dmat_init(pars->nInd, dm_type, dm_method, dm_transform, metadata->names, DMAT_NAMES_SRC_METADATA_NAMES_PTR);
- } else {
- pars->dm = dmat_init(pars->nInd, dm_type, dm_method, dm_transform, pars->names, DMAT_NAMES_SRC_IN_VCF_PARS_PTR);
- }
-
- dmat_get_distances_from_jgtmat(pars->jgtm, pars->dm);
- if (args->printDistanceMatrix) {
- dmat_write(pars->dm);
- }
-
+ dmat_calculate_distances(pars->jgtm, pars->dm);
+ if (args->printDistanceMatrix) {
+ dmat_write(pars->dm);
}
+
}
@@ -104,18 +96,18 @@ void input_VCF(paramStruct* pars) {
// ---- AMOVA
if (args->doAMOVA != 0) {
- amovaStruct* amova = amovaStruct_get(pars, metadata, blobs);
+ amovaStruct* amova = amovaStruct_get(pars, pars->metadata);
amovaStruct_destroy(amova);
}
// ---- dXY
- dxy_t* dxySt = NULL;
+ dxy_t* dxy = NULL;
if (args->doDxy > 0) {
if (args->in_dxy_fn != NULL) {
- // dxySt = dxy_read(pars, dmat, metadata);
+ // dxy = dxy_read(pars, dmat, pars->metadata);
//TODO
} else {
- dxySt = dxy_get(pars, pars->dm, metadata);
+ dxy = dxy_get(pars, pars->dm, pars->metadata);
}
// dont free yet, may be needed for dophylo
}
@@ -133,10 +125,10 @@ void input_VCF(paramStruct* pars) {
nj_print(nj);
} else if (args->doPhylo == 2) {
//TODO
- if (dxySt->nLevels > 1) {
+ if (dxy->nLevels > 1) {
ERROR("Neighbor joining is not supported for dxy with more than one level.");
}
- nj = nj_init(dxySt->dm[0], 0);
+ nj = nj_init(dxy->dm[0], 0);
nj_run(nj);
// nj_print(nj);
}
@@ -144,81 +136,46 @@ void input_VCF(paramStruct* pars) {
}
vcfData_destroy(vcfd);
- if (metadata != NULL) {
- metadataStruct_destroy(metadata);
+ if (dxy != NULL) {
+ dxy_destroy(dxy);
}
- if (dxySt != NULL) {
- dxy_destroy(dxySt);
+ if (bblocks != NULL) {
+ bblocks_destroy(bblocks);
}
- if (blobs != NULL) {
- delete(blobs);
- }
-
+ return;
}
void input_DM(paramStruct* pars) {
- metadataStruct* metadata = NULL;
- if (PROGRAM_NEEDS_METADATA) {
- if (NULL == args->in_mtd_fn) {
- ERROR("Requested analyses requiring metadata but no metadata file was provided.");
- } else {
- if (NULL != args->formula) {
- metadata = metadataStruct_read(pars);
- } else {
- NEVER; // this should already be checked in PROGRAM_NEEDS_FORMULA
- }
- }
- }
-
- if (args->doDist) {
- uint8_t required_transform;
- // if (args->doAMOVA) {
- // TODO
- required_transform = DMAT_TRANSFORM_SQUARE;
- // } else {
- // required_transform = DMAT_TRANSFORM_NONE;
- // }
-
+ uint8_t required_transform;
+ required_transform = DMAT_TRANSFORM_NONE;
+ pars->dm = dmat_read(args->in_dm_fn, required_transform, pars->metadata);
- if (args->nBootstraps > 0) {
- // pars->multidm=multidmat_read(args->in_dm_fn);
-
- // pars->multidm = multidmat_init(pars->nInd, dm_type, dm_method, dm_transform);
- } else {
- pars->dm = dmat_read(args->in_dm_fn, required_transform, metadata);
- //TODO match metadata->names dmat_t->names vcfd->hdr->samples
-
- if (args->printDistanceMatrix) {
- dmat_write(pars->dm);
- }
-
- // pars->dm = dmat_init(pars->nInd, dm_type, dm_method, dm_transform);
- // dmat_get_distances_from_jgtmat(pars->jgtm, pars->dm);
- }
+ //TODO match pars->metadata->indNames dmat_t->names vcfd->hdr->samples
+ if (args->printDistanceMatrix) {
+ dmat_write(pars->dm);
}
-
if (args->doAMOVA != 0) {
if (0 < args->nBootstraps) {
ERROR("Bootstrapping is not supported for distance matrix input.");
}
- amovaStruct* amova = amovaStruct_get(pars, metadata, NULL);
+ amovaStruct* amova = amovaStruct_get(pars, pars->metadata);
amovaStruct_destroy(amova);
}
- dxy_t* dxySt = NULL;
+ dxy_t* dxy = NULL;
if (args->doDxy > 0) {
if (args->in_dxy_fn == NULL) {
- dxySt = dxy_get(pars, pars->dm, metadata);
+ dxy = dxy_get(pars, pars->dm, pars->metadata);
} else {
//TODO
- // dxySt = dxy_read(pars, distanceMatrix, metadata);
+ // dxy = dxy_read(pars, distanceMatrix, pars->metadata);
}
- dxy_destroy(dxySt);
+ dxy_destroy(dxy);
}
// ---- NEIGHBOR JOINING
@@ -234,16 +191,13 @@ void input_DM(paramStruct* pars) {
nj_print(nj);
} else if (args->doPhylo == 2) {
//TODO
- // if (dxySt->nLevels > 1) {
+ // if (dxy->nLevels > 1) {
// ERROR("Neighbor joining is not supported for dxy with more than one level.");
// }
- // nj = nj_init(pars->nInd, dxySt->dm[0]);
+ // nj = nj_init(pars->names->len, dxy->dm[0]);
// nj_run(nj);
// nj_print(nj);
}
- if (metadata != NULL) {
- metadataStruct_destroy(metadata);
- }
return;
}
@@ -262,6 +216,7 @@ int run_unit_tests(void) {
int main(int argc, char** argv) {
+
argStruct* args = argStruct_get(--argc, ++argv);
if (args->doUnitTests) {
@@ -270,25 +225,30 @@ int main(int argc, char** argv) {
paramStruct* pars = paramStruct_init(args);
- if (pars->in_ft & IN_VCF) {
- input_VCF(pars);
- }
- if (pars->in_ft & IN_DM) {
- input_DM(pars);
+ if (PROGRAM_HAS_INPUT_METADATA) {
+ if (PROGRAM_NEEDS_METADATA) {
+ if (NULL != args->formula) {
+ pars->metadata = metadataStruct_read(pars);
+ } else {
+ NEVER; // this should already be checked in PROGRAM_NEEDS_FORMULA
+ }
+ }
}
- if (pars->in_ft == 0) {
- ERROR("Input file type not recognized.");
- }
+ if (PROGRAM_HAS_INPUT_VCF) {
+ input_VCF(pars);
+ } else if (PROGRAM_HAS_INPUT_DM) {
+ input_DM(pars);
+ }
fprintf(stderr, "\n[INFO]\tDone.\n\n");
// == CLEANUP ==
- argStruct_destroy(args);
paramStruct_destroy(pars);
IO::outFilesStruct_destroy(outFiles);
+ argStruct_destroy(args);
return 0;
}
diff --git a/paramStruct.cpp b/paramStruct.cpp
index 69de813..3977067 100644
--- a/paramStruct.cpp
+++ b/paramStruct.cpp
@@ -2,14 +2,43 @@
#include "ibd.h"
#include "dataStructs.h"
+#include "dmat.h"
+
+
+// calculate max memory needed for worst case scenario
+// worst case:
+// - no sites skipped
+// - number of sites == the total size of the all contigs in the vcf file
+//TODO
+void estimate_memory_needed(paramStruct* pars, vcfData* vcfd) {
+
+ // if (args->doJGTM) {
+ // // jgtmat_t
+ // }
+ // const size_t nInd = pars->names->len;
+
+
+ // size_t nIndCmb = (size_t)((nInd * (nInd - 1)) / 2);
+ // size_t nSites = pars->nSites;
+ // size_t nSites_arrays_size = pars->nSites_arrays_size;
+
+ // size_t nGT;
+ // if (args->doEM == ARG_DOEM_3GL) {
+ // nGT = 3;
+ // } else if (args->doEM == ARG_DOEM_10GL) {
+ // nGT = 10;
+ // }
+
+ // size_t mem_needed = 0;
+ // mem_needed += nIndCmb * 9 * sizeof(double);
+ // mem_needed += nIndCmb * 9 * sizeof(double);
+ // mem_needed += nIndCmb * 9 * sizeof(int);
+ // mem_needed += nSites * sizeof(double);
+ // mem_needed += nSites * nInd * nGT * sizeof(double);
+}
-// TODO handle :
-// TODO if site in vcf does not exist in alleles file, skip site ?
- // TODO if site in alleles file does not exist in vcf ? ?
-
-
-static alleles_t* alleles_init(void) {
+ alleles_t* alleles_init(void) {
alleles_t* alleles = (alleles_t*)malloc(sizeof(alleles_t));
ASSERT(alleles != NULL);
alleles->d = NULL;
@@ -19,7 +48,7 @@ static alleles_t* alleles_init(void) {
return(alleles);
}
-static void alleles_alloc(alleles_t* alleles, const size_t nSites) {
+ void alleles_alloc(alleles_t* alleles, const size_t nSites) {
// each uint64_t contains 16 packs of 4bit per-site alleles info (ordered pairs of 2bit ACGTs)
// so we need nSites/16 + 1 uint64_t
alleles->pos = size_tArray_alloc(nSites);
@@ -41,7 +70,7 @@ static void alleles_alloc(alleles_t* alleles, const size_t nSites) {
// with 4 columns: chr, pos, a1/major/ancestral, a2/minor/derived
// into alleles
// N.B. alleles file positions are 0-indexed //TODO make sure the user knows
-static alleles_t* alleles_read(const char* fn) {
+ alleles_t* alleles_read(const char* fn) {
alleles_t* alleles = alleles_init();
@@ -356,7 +385,7 @@ int alleles_get(alleles_t* alleles, const char* querychr, const size_t querypos,
}
-static void alleles_destroy(alleles_t* alleles) {
+ void alleles_destroy(alleles_t* alleles) {
FREE(alleles->d);
size_tArray_destroy(alleles->pos);
size_tArray_destroy(alleles->cposidx);
@@ -439,17 +468,15 @@ paramStruct* paramStruct_init(argStruct* args) {
pars->dm = NULL;
pars->jgtm = NULL;
pars->names = NULL; // set in vcfReader
+ pars->metadata = NULL;
pars->nSites = 0;
pars->totSites = 0;
pars->nSites_arrays_size = NSITES_BUF_INIT;
pars->nContigs = 0;
- pars->nInd = 0;
- pars->in_ft = 0;
pars->majorminor = NULL;
pars->ancder = NULL;
pars->alleles_posidx = -1;
pars->alleles_contigidx = -1;
- pars->contig_changed = false;
pars->a1a2[0] = -1;
pars->a1a2[1] = -1;
pars->formulaTokens = NULL;
@@ -463,35 +490,6 @@ paramStruct* paramStruct_init(argStruct* args) {
pars->DATETIME = (char*)malloc(1024 * sizeof(char));
sprintf(pars->DATETIME, "%s", get_time());
- if (NULL != args->in_vcf_fn) {
- fprintf(stderr, "\n[INFO]\tFound input VCF file: %s\n", args->in_vcf_fn);
- pars->in_ft = pars->in_ft | IN_VCF;
- }
-
- if (NULL != args->in_dm_fn) {
- fprintf(stderr, "\n[INFO]\tFound input distance matrix file: %s\n", args->in_dm_fn);
- pars->in_ft = pars->in_ft | IN_DM;
- }
-
- if (NULL != args->in_dxy_fn) {
- //TODO ??
- fprintf(stderr, "\n[INFO]\tFound input dxy file: %s\n", args->in_dxy_fn);
- pars->in_ft = pars->in_ft | IN_DXY;
- }
-
- // TODO check args versus input file type for compatibility
- if (pars->in_ft & IN_DM) {
- if (args->blockSize != 0) {
- ERROR("-blockSize is not supported for distance matrix input.");
- }
- if (args->doEM) {
- ERROR("-doEM is not available for distance matrix input.");
- }
- if (args->doJGTM) {
- ERROR("-doJGTM is not available for distance matrix input.");
- }
- }
-
if (PROGRAM_NEEDS_FORMULA) {
if (NULL != args->formula) {
@@ -539,46 +537,33 @@ void paramStruct_destroy(paramStruct* pars) {
}
if (pars->ibd != NULL) {
- const size_t nIndCmb = (size_t)((pars->nInd * (pars->nInd - 1)) / 2);
- for (size_t i = 0;i < nIndCmb;++i) {
- FREE(pars->ibd->pairScores[i]);
- }
- FREE(pars->ibd->pairScores);
- delete pars->ibd;
+ //TODO
+ // ASSERT(pars->names != NULL);
+ // const size_t nInd = pars->names->len;
+ // const size_t nIndCmb = (size_t)((nInd * (nInd - 1)) / 2);
+ // for (size_t i = 0;i < nIndCmb;++i) {
+ // FREE(pars->ibd->pairScores[i]);
+ // }
+ // FREE(pars->ibd->pairScores);
+ // delete pars->ibd;
}
FREE(pars->DATETIME);
if (NULL != pars->names) {
- strArray_destroy(pars->names);
+ if (PROGRAM_HAS_INPUT_VCF && (!(PROGRAM_HAS_INPUT_METADATA))) {
+ // only allocated if input is vcf && no metadata provided
+ // for all other cases it is ptr to another strArray
+ strArray_destroy(pars->names);
+ }
+ }
+ if (pars->metadata != NULL) {
+ metadataStruct_destroy(pars->metadata);
}
-
delete pars;
}
// VALIDATION - CHECKS BELOW
// --------------------------
-
-/// @brief check_consistency_args_pars - check consistency between arguments and parameters
-/// @param args pointer to argStruct
-/// @param pars pointer to paramStruct
-void check_consistency_args_pars(paramStruct* pars) {
-
- if (args->minInd == pars->nInd) {
- fprintf(stderr, "\n\t-> -minInd %d is equal to the number of individuals found in file: %d. Setting -minInd to 0 (all).\n", args->minInd, pars->nInd);
- args->minInd = 0;
- }
-
- if (pars->nInd == 1) {
- fprintf(stderr, "\n\n[ERROR]\tOnly one sample; will exit\n\n");
- exit(1);
- }
-
- if (pars->nInd < args->minInd) {
- fprintf(stderr, "\n\n[ERROR]\tMinimum number of individuals -minInd is set to %d, but input file contains %d individuals; will exit!\n\n", args->minInd, pars->nInd);
- exit(1);
- }
-}
-
diff --git a/paramStruct.h b/paramStruct.h
index 187b3d9..c929d69 100644
--- a/paramStruct.h
+++ b/paramStruct.h
@@ -15,6 +15,7 @@ typedef struct dmat_t dmat_t;
typedef struct jgtmat_t jgtmat_t;
typedef struct paramStruct paramStruct;
typedef struct alleles_t alleles_t;
+typedef struct metadataStruct metadataStruct;
/// ----------------------------------------------------------------------- ///
@@ -105,6 +106,9 @@ struct paramStruct {
strArray* names;
+
+ metadataStruct* metadata;
+
// number of sites non skipped for all individuals
// nSites may not be !=totSites if minInd is set
// or if a site is missing for all inds
@@ -113,10 +117,6 @@ struct paramStruct {
int nSites_arrays_size;
int nContigs;
- int nInd;
-
- // input file type
- int in_ft;
// a1: major/ref/ancestral allele
// a2: minor/alt/derived allele
@@ -131,7 +131,6 @@ struct paramStruct {
// for search, start from alleles_posidx instead of alleles_posidx+1 to also handle the first ever contig
int64_t alleles_posidx; // current position in the alleles_t->pos
int64_t alleles_contigidx; // current contig in the alleles_t->cnames
- bool contig_changed; // flag to indicate if the contig has changed in vcf file
// for each site,
// a1a2[0] is the index of the allele1/major/ancestral allele in vcfd->rec->d.allele
diff --git a/shared.h b/shared.h
index 90054f2..96ef63d 100644
--- a/shared.h
+++ b/shared.h
@@ -21,14 +21,10 @@
#define ARG_DOEM_3GL 1
#define ARG_DOEM_10GL 2
-#define PROGRAM_NEEDS_INDNAMES \
- ( ((0 != (args->doPhylo))))
-#define INPUT_IS_VCF \
- ( (pars->in_ft & IN_VCF) )
-#define INPUT_IS_DM \
- ( (pars->in_ft & IN_DM) )
+#define PROGRAM_NEEDS_INDNAMES \
+ ( ((0 != (args->doPhylo))))
#define PROGRAM_NEEDS_METADATA \
( ( args->doAMOVA || args->doDxy ))
@@ -36,16 +32,68 @@
#define PROGRAM_NEEDS_FORMULA \
( ( args->doAMOVA || args->doDxy ))
+#define ARG_DOAMOVA_UNSET 0
+#define ARG_DOAMOVA_SINGLERUN 1
+#define ARG_DOAMOVA_BOOTSTRAP 2
+#define ARG_DOAMOVA_PERMTEST 3
+
+
+#define PROGRAM_WILL_PERFORM_BLOCK_BOOTSTRAPPING \
+ ( (args->doAMOVA==ARG_DOAMOVA_BOOTSTRAP))
+//TODO add block bootstrap test for nj and dxy
/// ----------------------------------------------------------------------- ///
// ARGUMENT VALUES
-#define ARG_DOMAJORMINOR_NONE (0)
+#define ARG_DOJGTM_UNSET 0
+#define ARG_DOJGTM_3GT 1
+#define ARG_DOJGTM_10GT 2
-#define ARG_DOMAJORMINOR_BCF_REFALT1 (1)
+#define ARG_INTPLUS_INPUT_UNSET (0<<0)
+#define ARG_INTPLUS_INPUT_VCF (1<<0)
+#define ARG_INTPLUS_INPUT_DM (1<<1)
+#define ARG_INTPLUS_INPUT_MULTIDM (1<<2)
+#define ARG_INTPLUS_INPUT_DXY (1<<3)
+#define ARG_INTPLUS_INPUT_METADATA (1<<4)
+#define ARG_INTPLUS_INPUT_MAJORMINOR (1<<5)
+#define ARG_INTPLUS_INPUT_ANCDER (1<<6)
+#define ARG_INTPLUS_INPUT_BLOCKS (1<<7)
+#define ARG_INTPLUS_INPUT_REGIONS (1<<8)
+
+#define PROGRAM_HAS_INPUT_VCF \
+ ( (args->in_ft & ARG_INTPLUS_INPUT_VCF) )
+
+#define PROGRAM_HAS_INPUT_DM \
+ ( (args->in_ft & ARG_INTPLUS_INPUT_DM) )
+
+#define PROGRAM_HAS_INPUT_MULTIDM \
+ ( (args->in_ft & ARG_INTPLUS_INPUT_MULTIDM) )
+
+#define PROGRAM_HAS_INPUT_DXY \
+ ( (args->in_ft & ARG_INTPLUS_INPUT_DXY) )
+
+#define PROGRAM_HAS_INPUT_METADATA \
+ ( (args->in_ft & ARG_INTPLUS_INPUT_METADATA) )
+
+#define PROGRAM_HAS_INPUT_MAJORMINOR \
+ ( (args->in_ft & ARG_INTPLUS_INPUT_MAJORMINOR) )
+#define PROGRAM_HAS_INPUT_ANCDER \
+ ( (args->in_ft & ARG_INTPLUS_INPUT_ANCDER) )
+
+#define PROGRAM_HAS_INPUT_BLOCKS \
+ ( (args->in_ft & ARG_INTPLUS_INPUT_BLOCKS) )
+
+#define PROGRAM_HAS_INPUT_REGIONS \
+ ( (args->in_ft & ARG_INTPLUS_INPUT_REGIONS) )
+
+
+
+
+#define ARG_DOMAJORMINOR_UNSET (0)
+#define ARG_DOMAJORMINOR_BCF_REFALT1 (1)
#define ARG_DOMAJORMINOR_INFILE (2)
#define PROGRAM_WILL_USE_ALLELES_REF_ALT1 \
@@ -65,6 +113,8 @@
#define PROGRAM_WILL_USE_BCF_FMT_GT \
( (args->bcfSrc & ARG_INTPLUS_BCFSRC_FMT_GT) )
+#define PROGRAM_WILL_USE_RNG \
+ ( (args->doAMOVA==ARG_DOAMOVA_BOOTSTRAP) || (args->doAMOVA==ARG_DOAMOVA_PERMTEST) )
/* ========================================================================== */
/* MACRO DEFINITIONS ======================================================== */
@@ -111,7 +161,7 @@ do { \
\
ERROR("[Bad argument value: '%s %d'] Allowed range is [%d,%d]", (argstr), (argval), (minval), (maxval)); \
} \
-} while (0);
+} while(0)
#define CHECK_ARG_INTERVAL_DBL(argval, minval, maxval, argstr) \
@@ -120,7 +170,7 @@ do { \
\
ERROR("[Bad argument value: '%s %f'] Allowed range is [%f,%f]", (argstr), (argval), (minval), (maxval)); \
} \
-} while (0);
+} while(0)
#define CHECK_ARG_INTERVAL_IE_DBL(argval, minval, maxval, argstr) \
do { \
@@ -128,7 +178,7 @@ do { \
\
ERROR("[Bad argument value: '%s %f'] Allowed range is [%f,%f]", (argstr), (argval), (minval), (maxval)); \
} \
-} while (0);
+} while(0)
#define CHECK_ARG_INTERVAL_II_DBL(argval, minval, maxval, argstr) \
@@ -137,7 +187,7 @@ do { \
\
ERROR("[Bad argument value: '%s %f'] Allowed range is [%f,%f]", (argstr), (argval), (minval), (maxval)); \
} \
-} while (0);
+} while(0)
#define CHECK_ARG_INTERVAL_EI_DBL(argval, minval, maxval, argstr) \
@@ -146,7 +196,7 @@ do { \
\
ERROR("[Bad argument value: '%s %f'] Allowed range is [%f,%f]", (argstr), (argval), (minval), (maxval)); \
} \
-} while (0);
+} while(0)
#define CHECK_ARG_INTERVAL_EE_DBL(argval, minval, maxval, argstr) \
@@ -155,7 +205,7 @@ do { \
\
ERROR("[Bad argument value: '%s %f'] Allowed range is [%f,%f]", (argstr), (argval), (minval), (maxval)); \
} \
-} while (0);
+} while(0)
@@ -165,7 +215,7 @@ do { \
\
ERROR("Argument %s with value %d is out of range. Allowed values are 0 (for on/enable) and 1 (for off/disable)", (argstr), (argval)); \
} \
-} while (0);
+} while(0)
@@ -205,7 +255,7 @@ do { \
fprintf(stderr, __VA_ARGS__); \
fprintf(stderr, "\n*******\n"); \
exit(1); \
- } while (0);
+ } while(0)
/*
* Macro:[NEVER]
@@ -217,7 +267,7 @@ do { \
fprintf(stderr, "Control should never reach this point; please report this to the developers."); \
fprintf(stderr, "\n*******\n"); \
exit(1); \
- } while (0);
+ } while(0)
/*
@@ -235,7 +285,7 @@ do { \
__FILE__, __FUNCTION__, __LINE__, #expr); \
exit(1); \
} \
- } while (0);
+ } while(0)
/*
@@ -248,7 +298,7 @@ do { \
__LINE__); \
fprintf(stderr, __VA_ARGS__); \
fprintf(stderr, "\n"); \
-} while (0);
+} while(0)
/*
* Macro:[VWARN]
@@ -262,14 +312,14 @@ do { \
fprintf(stderr, __VA_ARGS__); \
fprintf(stderr, "\n"); \
} \
-} while (0);
+} while(0)
#define VRUN(...) \
do { \
if (0 != VERBOSITY_LEVEL) { \
__VA_ARGS__; \
} \
-} while (0);
+} while(0)
@@ -282,7 +332,13 @@ do { \
do{ \
fprintf(stderr, "\n[LOG](%s)\t", __FUNCTION__); \
fprintf(stderr, __VA_ARGS__); \
-} while (0);
+} while(0)
+
+#define LOGADD(...) \
+do{ \
+ fprintf(stderr, "\n[LOG]\t"); \
+ fprintf(stderr, __VA_ARGS__); \
+} while(0)
/*
@@ -295,19 +351,20 @@ do{ \
do{ \
fprintf(stderr, "\n________________________________________\n"); \
fprintf(stderr, "[LOG]\t-> Program is now at <%s/%s>\n", __FILE__, __FUNCTION__); \
-}while (0);
+}while(0)
#define END_LOGSECTION \
do{ \
fprintf(stderr, "\n________________________________________\n"); \
-}while (0);
+}while(0)
#define BEGIN_LOGSECTION_MSG(msg) \
do{ \
fprintf(stderr, "\n________________________________________\n"); \
fprintf(stderr, "[LOG]\t-> Program is now at <%s/%s>\n", __FILE__, __FUNCTION__); \
fprintf(stderr, "[LOG]\t-> %s\n", msg); \
-}while (0);
+}while(0)
+
/*
@@ -322,7 +379,7 @@ do{ \
fprintf(stderr, "\n%s!!\n", #msg); \
exit(1); \
} \
-} while (0);
+} while(0)
/*
@@ -362,7 +419,7 @@ do{ \
} \
(p) = (type) tmp; \
tmp = NULL; \
- } while (0);
+ } while(0)
#define MALLOC(type, p, size) \
@@ -580,16 +637,6 @@ enum OUTFC {
BBGZ, // bgzip (binary) [2]
};
-// input file types for main data input
-#define IN_VCF (1 << 0) // 1
-#define IN_DM (1 << 1) // 2
-#define IN_DXY (1 << 2) // 4
-
-#define A_BASE_IDX 0
-#define C_BASE_IDX 1
-#define G_BASE_IDX 2
-#define T_BASE_IDX 3
-
/* ========================================================================== */
/// @brief strIsNumeric - check if string is numeric
diff --git a/test/data/data0to5_metadata_first3ind.txt b/test/data/data0to5_metadata_first3ind.txt
new file mode 100644
index 0000000..f573396
--- /dev/null
+++ b/test/data/data0to5_metadata_first3ind.txt
@@ -0,0 +1,4 @@
+Individual Population
+popA_ind1 popA
+popA_ind2 popA
+popB_ind1 popB
diff --git a/test/data/data7.vcf b/test/data/data7.vcf
new file mode 100644
index 0000000..c42e907
--- /dev/null
+++ b/test/data/data7.vcf
@@ -0,0 +1,20 @@
+##fileformat=VCFv4.2
+##FILTER=<ID=PASS,Description="All filters passed">
+##source=tskit 0.4.1
+##contig=<ID=c1,length=10000000>
+##fileDate=Mon Jan 1 15:53:36 2024
+##source=vcfgl [version: v0.4.1-a4eb562] [build: Dec 27 2023 09:45:11] [htslib: 1.15.1-20-g46c56fcc]
+##source=Command: vcfgl --verbose 0 --threads 1 --seed 42 --input test/data/data0_msprime.vcf --output test/data/data4 --output-mode v --depth 100.000000 --error-rate 0.000000 --error-qs 0 --beta-variance -1.000000e+00 --gl-model 2 --gl1-theta 0.830000 --platform 0 --precise-gl 0 --i16-mapq 20 -explode 0 --rm-invar-sites 0 --rm-empty-sites 0 -doUnobserved 4 -doGVCF 0 -printPileup 0 -printTruth 1 -addGL 1 -addGP 0 -addPL 0 -addI16 0 -addQS 1 -addFormatDP 1 -addInfoDP 1 -addFormatAD 1 -addInfoAD 0 -addFormatADF 0 -addInfoADF 0 -addFormatADR 0 -addInfoADR 0
+##ALT=<ID=*,Description="Symbolic alternate allele representing any possible alternative allele at this location">
+##FORMAT=<ID=GL,Number=G,Type=Float,Description="Genotype likelihood in log10 likelihood ratio format">
+##FORMAT=<ID=AD,Number=R,Type=Integer,Description="Allelic depths for the REF and ALT alleles">
+##bcftools_annotateVersion=1.19-2-g91514451+htslib-1.18-72-g9e1ffd85
+##bcftools_annotateCommand=annotate -x INFO/DP,FORMAT/DP,INFO/QS data6.vcf; Date=Sun Mar 10 17:19:16 2024
+#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT popA_ind1 popA_ind2 popB_ind1
+c1 70 . G T,A,C,<*> . PASS . GL:AD 0,-25.5875,-576.055,-25.5875,-576.055,-576.055,-25.5875,-576.055,-576.055,-576.055,-25.5875,-576.055,-576.055,-576.055,-576.055:85,0,0,0,0 0,-31.6081,-711.597,-31.6081,-711.597,-711.597,-31.6081,-711.597,-711.597,-711.597,-31.6081,-711.597,-711.597,-711.597,-711.597:105,0,0,0,0 .,.,.,.,.,.,.,.,.,.:0,0,0,0
+c1 188 . C A,G,T,<*> . PASS . GL:AD .,.,.,.,.,.,.,.,.,.:0,0,0,0 0,-33.7153,-759.037,-33.7153,-759.037,-759.037,-33.7153,-759.037,-759.037,-759.037,-33.7153,-759.037,-759.037,-759.037,-759.037:112,0,0,0,0 0,-30.705,-691.266,-30.705,-691.266,-691.266,-30.705,-691.266,-691.266,-691.266,-30.705,-691.266,-691.266,-691.266,-691.266:102,0,0,0,0
+c1 286 . C A,G,T,<*> . PASS . GL:AD -254.536,0,-362.97,-271.996,-375.613,-647.609,-271.996,-375.613,-647.609,-647.609,-271.996,-375.613,-647.609,-647.609,-647.609:58,42,0,0,0 -315.831,0,-295.5,-330.281,-310.852,-641.133,-330.281,-310.852,-641.133,-641.133,-330.281,-310.852,-641.133,-641.133,-641.133:48,51,0,0,0 .,.,.,.,.,.,.,.,.,.:0,0,0,0
+c1 287 . G T,A,C,<*> . PASS . GL:AD 0,-27.9958,-630.272,-27.9958,-630.272,-630.272,-27.9958,-630.272,-630.272,-630.272,-27.9958,-630.272,-630.272,-630.272,-630.272:93,0,0,0,0 .,.,.,.,.,.,.,.,.,.:0,0,0,0 .,.,.,.,.,.,.,.,.,.:0,0,0,0
+c1 832 . G T,A,C,<*> . PASS . GL:AD .,.,.,.,.,.,.,.,.,.:0,0,0,0 0,-29.1999,-657.381,-29.1999,-657.381,-657.381,-29.1999,-657.381,-657.381,-657.381,-29.1999,-657.381,-657.381,-657.381,-657.381:97,0,0,0,0 .,.,.,.,.,.,.,.,.,.:0,0,0,0
+c1 835 . G T,A,C,<*> . PASS . GL:AD 0,-29.5009,-664.158,-29.5009,-664.158,-664.158,-29.5009,-664.158,-664.158,-664.158,-29.5009,-664.158,-664.158,-664.158,-664.158:98,0,0,0,0 0,-26.1896,-589.61,-26.1896,-589.61,-589.61,-26.1896,-589.61,-589.61,-589.61,-26.1896,-589.61,-589.61,-589.61,-589.61:87,0,0,0,0 -562.501,-24.9855,0,-562.501,-24.9855,-562.501,-562.501,-24.9855,-562.501,-562.501,-562.501,-24.9855,-562.501,-562.501,-562.501:0,83,0,0,0
+c1 836 . G T,A,C,<*> . PASS . GL:AD 0,-29.5009,-664.158,-29.5009,-664.158,-664.158,-29.5009,-664.158,-664.158,-664.158,-29.5009,-664.158,-664.158,-664.158,-664.158:98,0,0,0,0 .,.,.,.,.,.,.,.,.,.:0,0,0,0 .,.,.,.,.,.,.,.,.,.:0,0,0,0
diff --git a/test/reference/test1/test1.amova.csv b/test/reference/test1/test1.amova.csv
index 000ccfc..453aa31 100644
--- a/test/reference/test1/test1.amova.csv
+++ b/test/reference/test1/test1.amova.csv
@@ -1,17 +1,17 @@
df,Total,8
-SSD,Total,0.029773
-MSD,Total,0.003722
+SSD,Total,0.028734
+MSD,Total,0.003592
df,Among_Population_within_Total,2
-SSD,Among_Population_within_Total,0.009583
-MSD,Among_Population_within_Total,0.004791
+SSD,Among_Population_within_Total,0.009405
+MSD,Among_Population_within_Total,0.004703
df,Among_Individual_within_Population,6
-SSD,Among_Individual_within_Population,0.020190
-MSD,Among_Individual_within_Population,0.003365
-Phi,Population_in_Total,0.123789
+SSD,Among_Individual_within_Population,0.019329
+MSD,Among_Individual_within_Population,0.003221
+Phi,Population_in_Total,0.132891
Variance_coefficient,c_0_0,3.000000
Variance_coefficient,c_0_1,1.000000
Variance_coefficient,c_1_1,1.000000
-Variance_component,Population,0.000475
-Percentage_variance,Population,12.378908
-Variance_component,Individual,0.003365
-Percentage_variance,Individual,87.621092
+Variance_component,Population,0.000494
+Percentage_variance,Population,13.289141
+Variance_component,Individual,0.003221
+Percentage_variance,Individual,86.710859
diff --git a/test/reference/test1/test1.distance_matrix.csv b/test/reference/test1/test1.distance_matrix.csv
index 2304c1b..5d9ae05 100644
--- a/test/reference/test1/test1.distance_matrix.csv
+++ b/test/reference/test1/test1.distance_matrix.csv
@@ -1,6 +1,6 @@
1
LTED
-1
+0
0
9
36
@@ -13,39 +13,39 @@ pop2_ind3
pop3_ind1
pop3_ind2
pop3_ind3
-0.010164
-0.009636
-0.005723
-0.005265
-0.007881
-0.006320
-0.007046
-0.008354
-0.006991
-0.003571
-0.008278
-0.004509
-0.010059
-0.005541
-0.004524
-0.007259
-0.006881
-0.007311
-0.003687
-0.008434
-0.004238
-0.012404
-0.011252
-0.012119
-0.007589
-0.009054
-0.004053
-0.008441
-0.010237
-0.008753
-0.012565
-0.005459
-0.006724
-0.004664
-0.006325
-0.006645
+0.099941876464224796
+0.095928932519746588
+0.074660087151404667
+0.06988508306555366
+0.086496181719053947
+0.075635796461499746
+0.084242255586261444
+0.090241760274019894
+0.083620725852630598
+0.057715408482645079
+0.091270317689165728
+0.065973961092173003
+0.099600316460724245
+0.072870937444969391
+0.065332554796685968
+0.083480967842828671
+0.080908719470539875
+0.083298734371724248
+0.059267276095000987
+0.088681920513218521
+0.063917145675082734
+0.11055160045412823
+0.10501214425273186
+0.10880269760987357
+0.084970426798190973
+0.093733459221089221
+0.061528175018082665
+0.088490431553562107
+0.099248349010176609
+0.091536067699058379
+0.11064248101093817
+0.072865827284072787
+0.082177413817313916
+0.06611328188874957
+0.078064401456360191
+0.079924397515776013
diff --git a/test/reference/test10/test10.distance_matrix.csv b/test/reference/test10/test10.distance_matrix.csv
index f1ee300..c222ebe 100644
--- a/test/reference/test10/test10.distance_matrix.csv
+++ b/test/reference/test10/test10.distance_matrix.csv
@@ -1,6 +1,6 @@
1
LTED
-1
+0
0
8
28
@@ -12,31 +12,31 @@ popC_ind1
popC_ind2
popD_ind1
popD_ind2
-0.132231
-0.101240
-0.250000
-0.051653
-0.132231
-0.250000
-0.002066
-0.132231
-0.101240
-0.051653
-0.033058
-0.250000
-0.033058
-0.132231
-0.033058
-0.002066
-0.132231
-0.101240
-0.051653
-0.002066
-0.033058
-0.250000
-0.206612
-0.349174
-0.464876
-0.250000
-0.404959
-0.206612
+0.36363636363636365
+0.31818181818181818
+0.5
+0.22727272727272729
+0.36363636363636365
+0.5
+0.045454545454545456
+0.36363636363636365
+0.31818181818181818
+0.22727272727272729
+0.18181818181818182
+0.5
+0.18181818181818182
+0.36363636363636365
+0.18181818181818182
+0.045454545454545456
+0.36363636363636365
+0.31818181818181818
+0.22727272727272729
+0.045454545454545456
+0.18181818181818182
+0.5
+0.45454545454545453
+0.59090909090909094
+0.68181818181818177
+0.5
+0.63636363636363646
+0.45454545454545459
diff --git a/test/reference/test2/test2.amova.csv b/test/reference/test2/test2.amova.csv
index b89b44d..453aa31 100644
--- a/test/reference/test2/test2.amova.csv
+++ b/test/reference/test2/test2.amova.csv
@@ -1,17 +1,17 @@
df,Total,8
-SSD,Total,0.029773
-MSD,Total,0.003722
+SSD,Total,0.028734
+MSD,Total,0.003592
df,Among_Population_within_Total,2
-SSD,Among_Population_within_Total,0.009583
-MSD,Among_Population_within_Total,0.004791
+SSD,Among_Population_within_Total,0.009405
+MSD,Among_Population_within_Total,0.004703
df,Among_Individual_within_Population,6
-SSD,Among_Individual_within_Population,0.020190
-MSD,Among_Individual_within_Population,0.003365
-Phi,Population_in_Total,0.123808
+SSD,Among_Individual_within_Population,0.019329
+MSD,Among_Individual_within_Population,0.003221
+Phi,Population_in_Total,0.132891
Variance_coefficient,c_0_0,3.000000
Variance_coefficient,c_0_1,1.000000
Variance_coefficient,c_1_1,1.000000
-Variance_component,Population,0.000475
-Percentage_variance,Population,12.380778
-Variance_component,Individual,0.003365
-Percentage_variance,Individual,87.619222
+Variance_component,Population,0.000494
+Percentage_variance,Population,13.289141
+Variance_component,Individual,0.003221
+Percentage_variance,Individual,86.710859
diff --git a/test/reference/test3/test3.amova.csv b/test/reference/test3/test3.amova.csv
index fd9cff3..2a47603 100644
--- a/test/reference/test3/test3.amova.csv
+++ b/test/reference/test3/test3.amova.csv
@@ -1,27 +1,27 @@
df,Total,8
-SSD,Total,0.029773
-MSD,Total,0.003722
+SSD,Total,0.028734
+MSD,Total,0.003592
df,Among_Region_within_Total,1
-SSD,Among_Region_within_Total,0.005325
-MSD,Among_Region_within_Total,0.005325
+SSD,Among_Region_within_Total,0.005197
+MSD,Among_Region_within_Total,0.005197
df,Among_Population_within_Region,1
-SSD,Among_Population_within_Region,0.004257
-MSD,Among_Population_within_Region,0.004257
+SSD,Among_Population_within_Region,0.004208
+MSD,Among_Population_within_Region,0.004208
df,Among_Individual_within_Population,6
-SSD,Among_Individual_within_Population,0.020190
-MSD,Among_Individual_within_Population,0.003365
-Phi,Population_in_Region,0.081201
-Phi,Region_in_Total,0.067952
-Phi,Population_in_Total,0.143636
+SSD,Among_Individual_within_Population,0.019329
+MSD,Among_Individual_within_Population,0.003221
+Phi,Population_in_Region,0.092616
+Phi,Region_in_Total,0.065134
+Phi,Population_in_Total,0.151718
Variance_coefficient,c_0_0,4.000000
Variance_coefficient,c_0_1,3.000000
Variance_coefficient,c_0_2,1.000000
Variance_coefficient,c_1_1,3.000000
Variance_coefficient,c_1_2,1.000000
Variance_coefficient,c_2_2,1.000000
-Variance_component,Region,0.000267
-Percentage_variance,Region,6.795231
-Variance_component,Population,0.000297
-Percentage_variance,Population,7.568363
-Variance_component,Individual,0.003365
-Percentage_variance,Individual,85.636407
+Variance_component,Region,0.000247
+Percentage_variance,Region,6.513442
+Variance_component,Population,0.000329
+Percentage_variance,Population,8.658320
+Variance_component,Individual,0.003221
+Percentage_variance,Individual,84.828238
diff --git a/test/reference/test7/test7.amova.csv b/test/reference/test7/test7.amova.csv
index 6b4d06b..6df874b 100644
--- a/test/reference/test7/test7.amova.csv
+++ b/test/reference/test7/test7.amova.csv
@@ -1,27 +1,27 @@
df,Total,39
-SSD,Total,1.124258
-MSD,Total,0.028827
+SSD,Total,1.123449
+MSD,Total,0.028806
df,Among_Region_within_Total,1
-SSD,Among_Region_within_Total,0.790646
-MSD,Among_Region_within_Total,0.790646
+SSD,Among_Region_within_Total,0.787029
+MSD,Among_Region_within_Total,0.787029
df,Among_Population_within_Region,2
-SSD,Among_Population_within_Region,0.031475
-MSD,Among_Population_within_Region,0.015737
+SSD,Among_Population_within_Region,0.031009
+MSD,Among_Population_within_Region,0.015504
df,Among_Individual_within_Population,36
-SSD,Among_Individual_within_Population,0.302138
-MSD,Among_Individual_within_Population,0.008393
-Phi,Population_in_Region,0.080470
-Phi,Region_in_Total,0.809344
-Phi,Population_in_Total,0.824687
+SSD,Among_Individual_within_Population,0.305411
+MSD,Among_Individual_within_Population,0.008484
+Phi,Population_in_Region,0.076431
+Phi,Region_in_Total,0.807677
+Phi,Population_in_Total,0.822376
Variance_coefficient,c_0_0,20.000000
Variance_coefficient,c_0_1,10.000000
Variance_coefficient,c_0_2,1.000000
Variance_coefficient,c_1_1,10.000000
Variance_coefficient,c_1_2,1.000000
Variance_coefficient,c_2_2,1.000000
-Variance_component,Region,0.038745
-Percentage_variance,Region,80.934441
-Variance_component,Population,0.000734
-Percentage_variance,Population,1.534215
-Variance_component,Individual,0.008393
-Percentage_variance,Individual,17.531344
+Variance_component,Region,0.038576
+Percentage_variance,Region,80.767709
+Variance_component,Population,0.000702
+Percentage_variance,Population,1.469935
+Variance_component,Individual,0.008484
+Percentage_variance,Individual,17.762356
diff --git a/test/reference/test8/test8.newick b/test/reference/test8/test8.newick
index bd421c7..76f1e7e 100644
--- a/test/reference/test8/test8.newick
+++ b/test/reference/test8/test8.newick
@@ -1 +1 @@
-((pop3_ind1:0.004862,(pop3_ind2:0.004862,pop3_ind3:0.004618):0.004618):0.004862,(pop2_ind2:0.004862,pop2_ind3:0.004618):0.004618,(pop2_ind1:0.004862,(pop1_ind1:0.004862,(pop1_ind2:0.004862,pop1_ind3:0.004618):0.004618):0.004618):0.003867);
+((pop3_ind1:0.049370,(pop3_ind2:0.049370,pop3_ind3:0.047993):0.047993):0.049370,(pop2_ind2:0.049370,pop2_ind3:0.047993):0.047993,(pop2_ind1:0.049370,(pop1_ind1:0.049370,(pop1_ind2:0.049370,pop1_ind3:0.047993):0.047993):0.047993):0.043843);
diff --git a/test/reference/test9/test9.amova.csv b/test/reference/test9/test9.amova.csv
index 000ccfc..453aa31 100644
--- a/test/reference/test9/test9.amova.csv
+++ b/test/reference/test9/test9.amova.csv
@@ -1,17 +1,17 @@
df,Total,8
-SSD,Total,0.029773
-MSD,Total,0.003722
+SSD,Total,0.028734
+MSD,Total,0.003592
df,Among_Population_within_Total,2
-SSD,Among_Population_within_Total,0.009583
-MSD,Among_Population_within_Total,0.004791
+SSD,Among_Population_within_Total,0.009405
+MSD,Among_Population_within_Total,0.004703
df,Among_Individual_within_Population,6
-SSD,Among_Individual_within_Population,0.020190
-MSD,Among_Individual_within_Population,0.003365
-Phi,Population_in_Total,0.123789
+SSD,Among_Individual_within_Population,0.019329
+MSD,Among_Individual_within_Population,0.003221
+Phi,Population_in_Total,0.132891
Variance_coefficient,c_0_0,3.000000
Variance_coefficient,c_0_1,1.000000
Variance_coefficient,c_1_1,1.000000
-Variance_component,Population,0.000475
-Percentage_variance,Population,12.378908
-Variance_component,Individual,0.003365
-Percentage_variance,Individual,87.621092
+Variance_component,Population,0.000494
+Percentage_variance,Population,13.289141
+Variance_component,Individual,0.003221
+Percentage_variance,Individual,86.710859
diff --git a/test/runTests.sh b/test/runTests.sh
index 5736ecc..cce5cc9 100644
--- a/test/runTests.sh
+++ b/test/runTests.sh
@@ -87,7 +87,8 @@ runTestDiff(){
local logFile=${outPref}.log
local diffFile=${outPref}.diff
- local diffcmd="diff -s ${outFile} ${refFile} > ${diffFile} 2>&1"
+ local diffcmd="diff -s ${outFile} ${refFile}
+ > ${diffFile} 2>&1"
printf "# ${id} -> Running diff\n"
printf "# Command:\n${diffcmd}\n"
@@ -220,7 +221,7 @@ doAmova=1
printDistanceMatrix=1
minInd=2
doDist=1
-maxEmIter=100
+maxEmIter=10
emTole="1e-10"
metadataFile=${DATADIR}/metadata_Individual_Population.tsv
formula="Individual ~Population "
@@ -293,7 +294,7 @@ doEm=1
doAmova=1
minInd=2
doDist=1
-maxEmIter=100
+maxEmIter=10
emTole="1e-10"
metadataFile=${DATADIR}/metadata_Individual_Region_Population.tsv
formula="Individual~Region /Population"
@@ -330,7 +331,6 @@ doEm=0
doAmova=1
minInd=2
doDist=1
-nThreads=0
metadataFile=${DATADIR}/metadata_Individual_Region_Population_Subpopulation_groupNotUniq.tsv
formula="Individual~Population"
@@ -341,7 +341,6 @@ ARGS=" \
-doAMOVA ${doAmova} \
--minInd ${minInd} \
-doDist ${doDist} \
---nThreads ${nThreads} \
-m ${metadataFile} \
-f '${formula}'
"
@@ -426,9 +425,9 @@ doJGTM=1
doEm=1
doAmova=1
doDist=1
-maxEmIter=50
+maxEmIter=5
emTole="1e-5"
-nThreads=4
+nThreads=2
metadataFile=${DATADIR}/sim_demes_v2-model1_metadata_Individual_Region_Population.tsv
formula="Individual~Region/Population"
@@ -489,7 +488,7 @@ doAmova=1
printDistanceMatrix=1
minInd=2
doDist=1
-maxEmIter=100
+maxEmIter=10
emTole="1e-10"
metadataFile=${DATADIR}/metadata_Individual_Population.tsv
formula="Individual~Population"
@@ -527,7 +526,7 @@ doAmova=1
printDistanceMatrix=1
minInd=2
doDist=1
-maxEmIter=100
+maxEmIter=10
emTole="1e-10"
rmInvarSites=1
metadataFile=${DATADIR}/data0to5_metadata.txt
@@ -570,7 +569,7 @@ doAmova=1
printDistanceMatrix=1
minInd=2
doDist=1
-maxEmIter=100
+maxEmIter=10
emTole="1e-10"
rmInvarSites=1
metadataFile=${DATADIR}/data0to5_metadata.txt
@@ -611,7 +610,7 @@ doAmova=1
printDistanceMatrix=1
minInd=2
doDist=1
-maxEmIter=100
+maxEmIter=10
emTole="1e-10"
rmInvarSites=1
metadataFile=${DATADIR}/data0to5_metadata.txt
@@ -654,7 +653,7 @@ doAmova=1
printDistanceMatrix=1
minInd=2
doDist=1
-maxEmIter=100
+maxEmIter=10
emTole="1e-10"
rmInvarSites=1
metadataFile=${DATADIR}/data0to5_metadata.txt
@@ -695,7 +694,7 @@ doAmova=1
printDistanceMatrix=1
minInd=2
doDist=1
-maxEmIter=100
+maxEmIter=10
emTole="1e-10"
rmInvarSites=1
metadataFile=${DATADIR}/data0to5_metadata.txt
@@ -737,7 +736,7 @@ printDistanceMatrix=1
printJointGenotypeCountMatrix=1
minInd=2
doDist=1
-maxEmIter=100
+maxEmIter=10
emTole="1e-10"
rmInvarSites=1
metadataFile=${DATADIR}/data0to5_metadata.txt
@@ -818,7 +817,7 @@ doAmova=1
printDistanceMatrix=1
minInd=2
doDist=1
-maxEmIter=100
+maxEmIter=10
emTole="1e-10"
rmInvarSites=1
metadataFile=${DATADIR}/data0to5_metadata.txt
diff --git a/vcfReader.cpp b/vcfReader.cpp
index b998ec4..df5e0c5 100644
--- a/vcfReader.cpp
+++ b/vcfReader.cpp
@@ -2,14 +2,126 @@
#include "bootstrap.h"
+void bblocks_match_contig(bblocks_t* bblocks, const char* this_contigName, const size_t vcf_contig_i) {
+ size_t bb_contig_i;
+ if (bblocks->contig_names->find(this_contigName, &bb_contig_i)) {
+ if (bb_contig_i != vcf_contig_i) {
+ if (PROGRAM_HAS_INPUT_BLOCKS) {
+ ERROR("Block contigs do not have the same order as the VCF contigs. Please make sure the contigs in the block file are in the same order as the contigs in the VCF file.");
+ } else {
+ NEVER; // we generated the blocks; something went wrong
+ }
+ } // else OK
+ } else {
+ if (PROGRAM_HAS_INPUT_BLOCKS) {
+ ERROR("Contig %s not found in block file", this_contigName);
+ } else {
+ NEVER; // we generated the blocks; something went wrong
+ }
+ }
+ return;
+}
+
+gldata_t* gldata_init(void) {
+ gldata_t* gldata = (gldata_t*)malloc(sizeof(gldata_t));
+ ASSERT(gldata != NULL);
+
+ gldata->n_gls = 0;
+ gldata->n_ind = 0;
+ gldata->size = 0;
+ gldata->step_size = 0;
+ gldata->type = ARG_GLDATA_TYPE_UNMOD;
+
+ gldata->d = NULL;
+ gldata->mis = NULL;
+
+ return(gldata);
+}
+
+static gldata_t* gldata_alloc(const uint8_t n_gls, const size_t n_ind, const size_t init_n_sites, const uint8_t type) {
+ gldata_t* gldata = gldata_init();
+ gldata->n_gls = n_gls;
+ gldata->n_ind = n_ind;
+ gldata->size = init_n_sites;
+ gldata->step_size = init_n_sites;
+ gldata->type = type;
+
+ if (n_gls != 3) {
+ ERROR("Not implemented yet.");
+ }
+
+ gldata->d = (double***)malloc(n_ind * sizeof(double**));
+ ASSERT(gldata->d != NULL);
+
+ gldata->mis = (bool**)malloc(n_ind * sizeof(bool*));
+ ASSERT(gldata->mis != NULL);
+
+ for (size_t i = 0;i < n_ind;++i) {
+
+ gldata->d[i] = (double**)malloc(init_n_sites * sizeof(double*));
+ ASSERT(gldata->d[i] != NULL);
+ for (size_t s = 0;s < init_n_sites;++s) {
+ gldata->d[i][s] = NULL;
+ // alloc in-place if !mis[i][s]
+ }
+
+ gldata->mis[i] = NULL;
+ gldata->mis[i] = (bool*)malloc(init_n_sites * sizeof(bool));
+ ASSERT(gldata->mis[i] != NULL);
+
+ for (size_t j = 0;j < init_n_sites;++j) {
+ gldata->mis[i][j] = true; // init to true to exclude the unused sites at the end if array size is larger than the actual number of sites
+ }
+
+ }
+
+ return(gldata);
+}
+
+static void gldata_realloc(gldata_t* gldata) {
+
+ const size_t new_size = gldata->size + gldata->step_size;
+
+ for (size_t i = 0; i < gldata->n_ind; ++i) {
+ gldata->d[i] = (double**)realloc(gldata->d[i], new_size * sizeof(double*));
+ ASSERT(gldata->d[i] != NULL);
+ for (size_t j = gldata->size; j < new_size; ++j) {
+ gldata->d[i][j] = NULL;
+ }
+
+ gldata->mis[i] = (bool*)realloc(gldata->mis[i], new_size * sizeof(bool));
+ ASSERT(gldata->mis[i] != NULL);
+ for (size_t j = gldata->size; j < new_size; ++j) {
+ gldata->mis[i][j] = true;
+ }
+ }
+
+ gldata->size = new_size;
+ return;
+}
-//TODO
-// - check if required source tags (defined in args->bcfSrc) exist in the header
-inline void vcfdata_set_pars(paramStruct* pars, vcfData* vcfd) {
- pars->nInd = bcf_hdr_nsamples(vcfd->hdr);
+/// @param indends - array of pointers to individual ends (ptr to the last allocated site)
+void gldata_destroy(gldata_t* gldata) {
+ for (size_t i = 0; i < gldata->n_ind; ++i) {
+ for (size_t j = 0; j < gldata->size; ++j) {
+ if (NULL != gldata->d[i][j]) {
+ // isnull if mis[i][j]
+ FREE(gldata->d[i][j]);
+ }
+ }
+ FREE(gldata->d[i]);
+ FREE(gldata->mis[i]);
+ }
+ FREE(gldata->d);
+ FREE(gldata->mis);
+ FREE(gldata);
+ return;
}
+
+
+
const char* vcfData::get_contig_name(void) {
const char* ret = bcf_hdr_id2name(this->hdr, this->rec ? this->rec->rid : -1);
if (NULL == ret) {
@@ -50,12 +162,12 @@ int require_unpack() {
}
int require_index(paramStruct* pars) {
- if (INPUT_IS_VCF) {
+ if (PROGRAM_HAS_INPUT_VCF) {
if (NULL != args->in_region) {
return (IDX_CSI);
}
}
- if (pars->in_ft & IN_DM) {
+ if (PROGRAM_HAS_INPUT_DM) {
return (IDX_NONE);
}
return (IDX_NONE);
@@ -84,20 +196,20 @@ extern const int acgt_charToInt[256] = {
// return 1: skip site (for all individuals)
// assume ref=allele1 (major/anc) and alt=allele2 (minor/der)
-static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
+int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars, const bool contig_changed) {
- const int64_t curr_pos = vcfd->rec->pos;
+ const int64_t this_pos = vcfd->rec->pos;
const int nAlleles = vcfd->rec->n_allele;
bool allow_invar_site = false; //TODO determine based on analysistype and args
if (0 == nAlleles) {
- IO::vprint(2, "Skipping site at %s:%ld. Reason: No data found at site.\n", vcfd->get_contig_name(), curr_pos + 1);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: No data found at site.\n", vcfd->get_contig_name(), this_pos + 1);
return(1);
}
if (!allow_invar_site) {
if (1 == nAlleles) {
- IO::vprint(2, "Skipping site at %s:%ld. Reason: Site is not variable.\n", vcfd->get_contig_name(), curr_pos + 1);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: Site is not variable.\n", vcfd->get_contig_name(), this_pos + 1);
return(1);
}
}
@@ -111,13 +223,13 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
if ('*' == vcfd->rec->d.allele[i][0]) {
// '*' symbol = allele missing due to overlapping deletion
// source: VCF specs v4.3
- IO::vprint(2, "Skipping site at %s:%ld. Reason: REF allele is set to '*' (allele missing due to overlapping deletion).\n", vcfd->get_contig_name(), curr_pos + 1);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: REF allele is set to '*' (allele missing due to overlapping deletion).\n", vcfd->get_contig_name(), this_pos + 1);
return(1);
}
if ('.' == vcfd->rec->d.allele[i][0]) {
// '.' symbol = MISSING value (no variant)
// source: VCF specs v4.3
- IO::vprint(2, "Skipping site at %s:%ld. Reason: REF allele is set to '*' (allele missing due to overlapping deletion).\n", vcfd->get_contig_name(), curr_pos + 1);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: REF allele is set to '*' (allele missing due to overlapping deletion).\n", vcfd->get_contig_name(), this_pos + 1);
return(1);
}
@@ -137,16 +249,16 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
int j = 0;
while (vcfd->rec->d.allele[i][j] != '\0') {
if (vcfd->rec->d.allele[i][j] != 'A' && vcfd->rec->d.allele[i][j] != 'C' && vcfd->rec->d.allele[i][j] != 'G' && vcfd->rec->d.allele[i][j] != 'T') {
- ERROR("Found bad allele '%s' at %s:%ld. If you believe this is a valid allele, please contact the developers.\n", vcfd->rec->d.allele[i], vcfd->get_contig_name(), curr_pos + 1);
+ ERROR("Found bad allele '%s' at %s:%ld. If you believe this is a valid allele, please contact the developers.\n", vcfd->rec->d.allele[i], vcfd->get_contig_name(), this_pos + 1);
break;
}
j++;
}
- IO::vprint(2, "Skipping site at %s:%ld. Reason: REF allele is an INDEL.\n", vcfd->get_contig_name(), curr_pos + 1);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: REF allele is an INDEL.\n", vcfd->get_contig_name(), this_pos + 1);
return(1);
}
- ERROR("Found bad allele '%s' at %s:%ld. If you believe this is a valid allele, please contact the developers.\n", vcfd->rec->d.allele[i], vcfd->get_contig_name(), curr_pos + 1);
+ ERROR("Found bad allele '%s' at %s:%ld. If you believe this is a valid allele, please contact the developers.\n", vcfd->rec->d.allele[i], vcfd->get_contig_name(), this_pos + 1);
}
// -> allele starts with '<'
@@ -157,7 +269,7 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
vcfd->allele_unobserved = i;
continue;
} else {
- ERROR("Found bad allele '%s' at %s:%ld. If you believe this is a valid allele, please contact the developers.\n", vcfd->rec->d.allele[i], vcfd->get_contig_name(), curr_pos + 1);
+ ERROR("Found bad allele '%s' at %s:%ld. If you believe this is a valid allele, please contact the developers.\n", vcfd->rec->d.allele[i], vcfd->get_contig_name(), this_pos + 1);
}
@@ -167,7 +279,7 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
vcfd->allele_unobserved = i;
continue;
} else {
- ERROR("Found bad allele '%s' at %s:%ld. If you believe this is a valid allele, please contact the developers.\n", vcfd->rec->d.allele[i], vcfd->get_contig_name(), curr_pos + 1);
+ ERROR("Found bad allele '%s' at %s:%ld. If you believe this is a valid allele, please contact the developers.\n", vcfd->rec->d.allele[i], vcfd->get_contig_name(), this_pos + 1);
}
}
NEVER;
@@ -178,7 +290,7 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
if (0 == vcfd->allele_unobserved) {
if (PROGRAM_WILL_USE_ALLELES_REF_ALT1) {
- IO::vprint(2, "Skipping site at %s:%ld. Reason: Program will use REF allele as major allele, but REF allele is set to <NON_REF> or <*>.\n", vcfd->get_contig_name(), curr_pos + 1);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: Program will use REF allele as major allele, but REF allele is set to <NON_REF> or <*>.\n", vcfd->get_contig_name(), this_pos + 1);
return(1);
}
@@ -195,7 +307,7 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
if (2 == nAlleles) {
// ALT is set but REF is not, so probably an invariable site with no REF allele specified
if (args->rmInvarSites) {
- IO::vprint(2, "Skipping site at %s:%ld. Reason: Site is not variable.\n", vcfd->get_contig_name(), curr_pos + 1);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: Site is not variable.\n", vcfd->get_contig_name(), this_pos + 1);
return(1);
}
}
@@ -206,7 +318,7 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
} else if (1 == vcfd->allele_unobserved) {
if (PROGRAM_WILL_USE_ALLELES_REF_ALT1) {
- IO::vprint(2, "Skipping site at %s:%ld. Reason: Program will use REF allele as major allele, but ALT allele is set to <NON_REF> or <*>.\n", vcfd->get_contig_name(), curr_pos + 1);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: Program will use REF allele as major allele, but ALT allele is set to <NON_REF> or <*>.\n", vcfd->get_contig_name(), this_pos + 1);
return(1);
}
@@ -215,7 +327,7 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
if (2 == nAlleles) {
// invariable site, only REF and UNOBSERVED alleles are found
// if (args->rmInvarSites) {
- IO::vprint(2, "Skipping site at %s:%ld. Reason: Site is not variable.\n", vcfd->get_contig_name(), curr_pos + 1);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: Site is not variable.\n", vcfd->get_contig_name(), this_pos + 1);
return(1);
// }
}
@@ -228,7 +340,7 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
if (1 == nAlleles) {
if (args->rmInvarSites) {
- IO::vprint(2, "Skipping site at %s:%ld. Reason: Site is not variable.\n", vcfd->get_contig_name(), curr_pos + 1);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: Site is not variable.\n", vcfd->get_contig_name(), this_pos + 1);
return(1);
}
} else {
@@ -254,7 +366,7 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
int32_t n = 0;
n = bcf_get_format_int32(vcfd->hdr, vcfd->rec, "AD", &ads, &n_ads);
if (n <= 0) {
- ERROR("Could not read AD tag from the VCF file. AD tag is required for removing invariable sites via --rm-invar-sites 1.");
+ ERROR("Could not read AD tag from the VCF file.");
}
// make sure that AD is non-0 for at least 2 alleles
@@ -262,7 +374,7 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
for (int i = 0; i < nAlleles; ++i) {
n_non0[i] = 0;
}
- for (int i = 0; i < pars->nInd; ++i) {
+ for (int i = 0; i < pars->names->len; ++i) {
for (int j = 0; j < nAlleles; ++j) {
if (ads[i * nAlleles + j] > 0) {
n_non0[j]++;
@@ -277,7 +389,7 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
}
FREE(ads);
if (tot_non0 < 2) {
- IO::vprint(2, "Skipping site at %s:%ld. Reason: Site is not variable.\n", vcfd->get_contig_name(), curr_pos + 1);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: Site is not variable.\n", vcfd->get_contig_name(), this_pos + 1);
return(1);
}
}
@@ -305,31 +417,30 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
size_t cnamesidx;
if (alleles->cnames->find(contigname, &cnamesidx)) {
pars->alleles_contigidx = cnamesidx;
- pars->contig_changed = false; // initial state
pars->alleles_posidx = alleles->cposidx->d[cnamesidx];
pos_search_start = pars->alleles_posidx; // start from the first pos of the contig instead
} else {
- IO::vprint(2, "Skipping site at %s:%ld. Reason: Contig %s not found in alleles file.\n", contigname, curr_pos + 1, contigname);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: Contig %s not found in alleles file.\n", contigname, this_pos + 1, contigname);
return(1);
}
- }
+ } else {
- if (pars->contig_changed) {
- const char* contigname = bcf_hdr_id2name(vcfd->hdr, vcfd->rec ? vcfd->rec->rid : -1);
- if (NULL == contigname) {
- ERROR("Could not retrieve contig name for record %d.", vcfd->rec->rid);
- }
- // find the new contig, start with the last contig idx+1
- size_t search_start = pars->alleles_contigidx + 1;
- size_t cnamesidx;
- if (alleles->cnames->find_from(contigname, &cnamesidx, pars->alleles_contigidx)) {
- pars->alleles_contigidx = cnamesidx;
- pars->contig_changed = false; // reset
- pars->alleles_posidx = alleles->cposidx->d[cnamesidx];
- pos_search_start = pars->alleles_posidx; // start from the first pos of the contig instead
- } else {
- IO::vprint(2, "Skipping site at %s:%ld. Reason: Contig %s not found in alleles file.\n", contigname, curr_pos + 1, contigname);
- return(1);
+ if (contig_changed) {
+ const char* contigname = bcf_hdr_id2name(vcfd->hdr, vcfd->rec ? vcfd->rec->rid : -1);
+ if (NULL == contigname) {
+ ERROR("Could not retrieve contig name for record %d.", vcfd->rec->rid);
+ }
+ // find the new contig, start with the last contig idx+1
+ size_t search_start = pars->alleles_contigidx + 1;
+ size_t cnamesidx;
+ if (alleles->cnames->find_from(contigname, &cnamesidx, pars->alleles_contigidx)) {
+ pars->alleles_contigidx = cnamesidx;
+ pars->alleles_posidx = alleles->cposidx->d[cnamesidx];
+ pos_search_start = pars->alleles_posidx; // start from the first pos of the contig instead
+ } else {
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: Contig %s not found in alleles file.\n", contigname, this_pos + 1, contigname);
+ return(1);
+ }
}
}
@@ -342,25 +453,20 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
foundSite = false;
for (size_t posidx = pos_search_start; posidx < pos_search_end; posidx++) {
- if (alleles->pos->d[posidx] == (size_t)curr_pos) {
+ if (alleles->pos->d[posidx] == (size_t)this_pos) {
pars->alleles_posidx = posidx;
foundSite = true;
break;
}
}
if (!foundSite) {
- IO::vprint(2, "Skipping site at %s:%ld. Reason: Site not found in alleles file.\n", vcfd->get_contig_name(), curr_pos + 1);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: Site not found in alleles file.\n", vcfd->get_contig_name(), this_pos + 1);
return(1);
}
- DEVPRINT("Found VCF record at %s:%ld in the alleles file.", vcfd->get_contig_name(), curr_pos + 1);
-
- // char* allele1 = (char*)malloc(2);
- // char* allele2 = (char*)malloc(2);
char allele1[2];
char allele2[2];
-
// the 64bit block containing the alleles for the position
uint64_t tmp = alleles->d[(pars->alleles_posidx / 16)];
// the index of the position in the 64bit block
@@ -381,22 +487,22 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
if ('\0' == vcfd->rec->d.allele[vcfal][1]) {
if (vcfd->rec->d.allele[vcfal][0] == *allele1) {
pars->a1a2[0] = vcfal;
- DEVPRINT("The allele1 (%c)'s position in vcf rec->d.allele (REF:%s ALT:%s) is %d", *allele1, vcfd->rec->d.allele[0], vcfd->rec->d.allele[1], pars->a1a2[0]);
+ // DEVPRINT("The allele1 (%c)'s position in vcf rec->d.allele (REF:%s ALT:%s) is %d", *allele1, vcfd->rec->d.allele[0], vcfd->rec->d.allele[1], pars->a1a2[0]);
}
if (vcfd->rec->d.allele[vcfal][0] == *allele2) {
pars->a1a2[1] = vcfal;
- DEVPRINT("The allele2 (%c)'s position in vcf rec->d.allele (REF:%s ALT:%s) is %d", *allele2, vcfd->rec->d.allele[0], vcfd->rec->d.allele[1], pars->a1a2[1]);
+ // DEVPRINT("The allele2 (%c)'s position in vcf rec->d.allele (REF:%s ALT:%s) is %d", *allele2, vcfd->rec->d.allele[0], vcfd->rec->d.allele[1], pars->a1a2[1]);
}
}
}
// -> make sure allele1 allele2 exists in vcfd->rec->d.allele
if (pars->a1a2[0] == -1) {
- IO::vprint(2, "Skipping site at %s:%ld. Reason: Allele1 (%c) not found in vcf rec->d.allele (REF:%s ALT:%s).\n", vcfd->get_contig_name(), curr_pos + 1, *allele1, vcfd->rec->d.allele[0], vcfd->rec->d.allele[1]);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: Allele1 (%c) not found in vcf rec->d.allele (REF:%s ALT:%s).\n", vcfd->get_contig_name(), this_pos + 1, *allele1, vcfd->rec->d.allele[0], vcfd->rec->d.allele[1]);
return(1);
}
if (pars->a1a2[1] == -1) {
- IO::vprint(2, "Skipping site at %s:%ld. Reason: Allele2 (%c) not found in vcf rec->d.allele (REF:%s ALT:%s).\n", vcfd->get_contig_name(), curr_pos + 1, *allele2, vcfd->rec->d.allele[0], vcfd->rec->d.allele[1]);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: Allele2 (%c) not found in vcf rec->d.allele (REF:%s ALT:%s).\n", vcfd->get_contig_name(), this_pos + 1, *allele2, vcfd->rec->d.allele[0], vcfd->rec->d.allele[1]);
return(1);
}
@@ -406,20 +512,25 @@ static int read_site_with_alleles_ref_alt(vcfData* vcfd, paramStruct* pars) {
}
// return 1: skip site for all individuals
-int site_read_GL(vcfData* vcfd, paramStruct* pars) {
+static int site_read_3GL(vcfData* vcfd, paramStruct* pars, const size_t block_i, const bool contig_changed) {
+
+
+
int ret;
- if (0 != (ret = read_site_with_alleles_ref_alt(vcfd, pars))) {
+ if (0 != (ret = read_site_with_alleles_ref_alt(vcfd, pars, contig_changed))) {
return(ret);
}
+ const size_t site_idx = pars->nSites;
+ const size_t gldata_n_gls = vcfd->gldata->n_gls;
+
+
const int ngt_to_use = vcfd->nGT;
// DEVPRINT("n_gls: %d nGT:%d", lgls.n_gls, vcfd->nGT);
// ngls is observed number of genotypes as they appear in gl data
// ngt_to_use is what we need to construct the pairwise genotype counts matrix (i.e. 3 or 10)
- const int nInd = pars->nInd;
-
float* lgls = NULL;
int size_e = 0;
int n_gls = 0;
@@ -428,7 +539,7 @@ int site_read_GL(vcfData* vcfd, paramStruct* pars) {
if (n_values <= 0) {
ERROR("Could not read GL tag from the VCF file.");
}
- n_gls = n_values / pars->nInd;
+ n_gls = n_values / pars->names->len;
if (n_gls < ngt_to_use) {
@@ -437,49 +548,50 @@ int site_read_GL(vcfData* vcfd, paramStruct* pars) {
}
float* sample_lgls = NULL;
- double* sample_lngls = NULL;
- bool isMissing;
- bool includeInds[nInd];
+
int skipSite = 0;
- const size_t site_idx = pars->nSites;
+ bool** isMissing = vcfd->gldata->mis;
- // ASSERT(ngt_to_use == 3);//TODO add 10gls
uint64_t ordered_gt_fetch[3];
ordered_gt_fetch[0] = bcf_alleles2gt(pars->a1a2[0], pars->a1a2[0]);
ordered_gt_fetch[1] = bcf_alleles2gt(pars->a1a2[0], pars->a1a2[1]);
ordered_gt_fetch[2] = bcf_alleles2gt(pars->a1a2[1], pars->a1a2[1]);
- for (int indi = 0; indi < nInd; indi++) {
+ double* iptr;
+
+ double max;
+ const size_t nInd = pars->names->len;
+
+ for (size_t i = 0; i < nInd; i++) {
- includeInds[indi] = false;
- isMissing = false;
+ isMissing[i][site_idx] = false;
- sample_lgls = lgls + indi * n_gls;
+ sample_lgls = lgls + i * n_gls;
// ----------------------------------------------------------------- //
// -> check for missing data
if (bcf_float_is_missing(sample_lgls[0])) {
// missing data check 1
- isMissing = true;
+ isMissing[i][site_idx] = true;
} else {
// missing data check 2
int z = 1;
- for (int i = 1; i < n_gls; ++i) {
- if (sample_lgls[0] == sample_lgls[i]) {
+ for (int j = 1; j < n_gls; ++j) {
+ if (sample_lgls[0] == sample_lgls[j]) {
++z;
}
}
if (z == n_gls) {
// missing data (all gl values are the same)
- isMissing = true;
+ isMissing[i][site_idx] = true;
}
}
- if (isMissing) {
- // if minInd 0 (the program will only use sites shared across all individuals)
+ if (isMissing[i][site_idx]) {
+ // if minInd 0 (the program will only use sites shared across all ividuals)
// skip site when you encounter any missing data
if (0 == args->minInd) {
skipSite = 1;
@@ -494,8 +606,8 @@ int site_read_GL(vcfData* vcfd, paramStruct* pars) {
n_missing_ind++;
- if (indi == nInd - 1 && nInd == n_missing_ind) {
- // check if all individuals are missing
+ if (i == nInd - 1 && nInd == n_missing_ind) {
+ // check if all ividuals are missing
skipSite = 1;
break;
}
@@ -503,7 +615,7 @@ int site_read_GL(vcfData* vcfd, paramStruct* pars) {
// skip site if minInd is defined and #non-missing inds=<nInd
if (args->minInd != 2) {
if ((nInd - n_missing_ind) < args->minInd) {
- IO::vprint(2, "Skipping site at %s:%ld. Reason: Number of non-missing individuals is less than the minimum number of individuals -minInd.\n", vcfd->get_contig_name(), vcfd->rec->pos + 1);
+ IO::vprint(2, "Skipping site at %s:%ld. Reason: Number of non-missing ividuals is less than the minimum number of ividuals -minInd.\n", vcfd->get_contig_name(), vcfd->rec->pos + 1);
skipSite = 1;
break;
}
@@ -511,44 +623,65 @@ int site_read_GL(vcfData* vcfd, paramStruct* pars) {
continue;
}
+
// -> not missing
- includeInds[indi] = true;
- sample_lgls = lgls + (indi * n_gls);
- sample_lngls = vcfd->lngl->d[site_idx] + indi * vcfd->nGT;
- if (includeInds[indi]) {
- sample_lngls[0] = (double)LOG2LN(sample_lgls[ordered_gt_fetch[0]]);
- sample_lngls[1] = (double)LOG2LN(sample_lgls[ordered_gt_fetch[1]]);
- sample_lngls[2] = (double)LOG2LN(sample_lgls[ordered_gt_fetch[2]]);
- }
- }
+ DEVASSERT(!isMissing[i][site_idx]);
+
+ sample_lgls = lgls + (i * n_gls);
+
+
+ vcfd->gldata->d[i][site_idx] = (double*)malloc(n_gls * sizeof(double));
+ ASSERT(vcfd->gldata->d[i][site_idx] != NULL);
+
+ iptr = vcfd->gldata->d[i][site_idx];
+
+ if (vcfd->gldata->type == ARG_GLDATA_TYPE_UNMOD) {
+ iptr[0] = (double)sample_lgls[ordered_gt_fetch[0]];
+ iptr[1] = (double)sample_lgls[ordered_gt_fetch[1]];
+ iptr[2] = (double)sample_lgls[ordered_gt_fetch[2]];
+ } else {
+
+ if (vcfd->gldata->type == ARG_GLDATA_TYPE_LOG10GL) {
+ iptr[0] = (double)sample_lgls[ordered_gt_fetch[0]];
+ iptr[1] = (double)sample_lgls[ordered_gt_fetch[1]];
+ iptr[2] = (double)sample_lgls[ordered_gt_fetch[2]];
+
+ } else if (vcfd->gldata->type == ARG_GLDATA_TYPE_LNGL) {
+ iptr[0] = (double)LOG2LN(sample_lgls[ordered_gt_fetch[0]]);
+ iptr[1] = (double)LOG2LN(sample_lgls[ordered_gt_fetch[1]]);
+ iptr[2] = (double)LOG2LN(sample_lgls[ordered_gt_fetch[2]]);
+
+ } else {
+ NEVER;
+ }
- if (skipSite) {
- // clear all data we wrote so far in sample_lngls
- for (int i = 0; i < nInd; i++) {
- if (includeInds[i]) {
- sample_lngls = vcfd->lngl->d[site_idx] + i * vcfd->nGT;
- sample_lngls[0] = NEG_INF;
- sample_lngls[1] = NEG_INF;
- sample_lngls[2] = NEG_INF;
+ max = iptr[0];
+ for (size_t i = 1; i < 3; i++) {
+ max = (max < iptr[i]) ? iptr[i] : max;
+ }
+ for (size_t i = 0; i < 3; i++) {
+ iptr[i] -= max;
}
}
+
+
}
+
FREE(lgls);
return(skipSite);
}
// return 1: skip site for all individuals
-int site_read_GT(jgtmat_t* jgtm, vcfData* vcfd, paramStruct* pars) {
+int site_read_GT(jgtmat_t* jgtm, vcfData* vcfd, paramStruct* pars, const bool contig_changed) {
int ret;
- if (0 != (ret = read_site_with_alleles_ref_alt(vcfd, pars))) {
+ if (0 != (ret = read_site_with_alleles_ref_alt(vcfd, pars, contig_changed))) {
return(ret);
}
- const int nInd = pars->nInd;
- bool isMissing;
+ const int nInd = pars->names->len;
int n_missing_ind = 0;
int32_t* gts = NULL;
int size_e = 0;
@@ -557,12 +690,12 @@ int site_read_GT(jgtmat_t* jgtm, vcfData* vcfd, paramStruct* pars) {
if (n_values <= 0) {
ERROR("Could not read GT tag from the VCF file.");
}
- if ((n_values / pars->nInd) != PROGRAM_PLOIDY) {
- ERROR("Ploidy %d not supported.", n_values / pars->nInd);
+ if ((n_values / pars->names->len) != PROGRAM_PLOIDY) {
+ ERROR("Ploidy %d not supported.", n_values / pars->names->len);
}
int32_t* sample_gts = NULL;
- bool includeInds[nInd];
+ bool hasData[nInd];
int skipSite = 0;
@@ -575,16 +708,11 @@ int site_read_GT(jgtmat_t* jgtm, vcfData* vcfd, paramStruct* pars) {
for (int indi = 0; indi < nInd; ++indi) {
- includeInds[indi] = false;
- isMissing = false;
+ hasData[indi] = false;
sample_gts = gts + indi * PROGRAM_PLOIDY;
if (1 == bcf_gt_is_missing(sample_gts[0]) || 1 == bcf_gt_is_missing(sample_gts[1])) {
- isMissing = true;
- }
-
- if (isMissing) {
// minInd 0
// only use sites shared across all individuals
// so skip site when you first encounter nan
@@ -616,20 +744,20 @@ int site_read_GT(jgtmat_t* jgtm, vcfData* vcfd, paramStruct* pars) {
}
}
continue;
+ } else {
+ // -> not missing
+ hasData[indi] = true;
}
- // -> not missing
- includeInds[indi] = true;
-
}
bool skipInd;
- // -> rm invar sites based on GT
- // @dragon
- if (args->rmInvarSites) {
+ if (!skipSite && args->rmInvarSites != 0) {
+ // -> rm invar sites based on GT
+ // @dragon
int tot_nder = 0;
for (int i = 0; i < nInd; ++i) {
- if (!includeInds[i]) {
+ if (!hasData[i]) {
continue;
}
skipInd = false;
@@ -661,7 +789,6 @@ int site_read_GT(jgtmat_t* jgtm, vcfData* vcfd, paramStruct* pars) {
}
-
do {
if (skipSite) {
break;
@@ -672,7 +799,7 @@ int site_read_GT(jgtmat_t* jgtm, vcfData* vcfd, paramStruct* pars) {
int pidx = 0;
for (int i1 = 1; i1 < nInd; ++i1) {
- if (!includeInds[i1]) {
+ if (!hasData[i1]) {
++pidx;
continue;
}
@@ -702,14 +829,14 @@ int site_read_GT(jgtmat_t* jgtm, vcfData* vcfd, paramStruct* pars) {
if (skipInd) {
IO::vprint(2, "Skipping individual %d (%s) at site %s:%ld. Reason: Allele in genotype (%c) is not one of alleles to use (%c %c).", i1, vcfd->hdr->samples[i1], vcfd->get_contig_name(), vcfd->rec->pos + 1, vcfd->rec->d.allele[gt1][0], vcfd->rec->d.allele[0][0], vcfd->rec->d.allele[1][0]);
- includeInds[i1] = false;
+ hasData[i1] = false;
++pidx;
continue;
}
for (int i2 = 0;i2 < i1; ++i2) {
- if (!includeInds[i2]) {
+ if (!hasData[i2]) {
++pidx;
continue;
}
@@ -738,7 +865,7 @@ int site_read_GT(jgtmat_t* jgtm, vcfData* vcfd, paramStruct* pars) {
if (skipInd) {
IO::vprint(2, "Skipping individual %d (%s) at site %s:%ld. Reason: Allele in genotype (%c) is not one of alleles to use (%c %c).", i2, vcfd->hdr->samples[i2], vcfd->get_contig_name(), vcfd->rec->pos + 1, vcfd->rec->d.allele[gt1][0], vcfd->rec->d.allele[0][0], vcfd->rec->d.allele[1][0]);
- includeInds[i2] = false;
+ hasData[i2] = false;
++pidx;
continue;
}
@@ -755,7 +882,7 @@ int site_read_GT(jgtmat_t* jgtm, vcfData* vcfd, paramStruct* pars) {
return(skipSite);
}
-vcfData* vcfData_init(paramStruct* pars) {
+vcfData* vcfData_init(paramStruct* pars, metadataStruct* metadata) {
BEGIN_LOGSECTION;
@@ -797,8 +924,6 @@ vcfData* vcfData_init(paramStruct* pars) {
}
- vcfdata_set_pars(pars, vcfd);
-
if (1 == args->doDist) {
// EM using 3 GL values
if (args->doEM == 1) {
@@ -828,25 +953,111 @@ vcfData* vcfData_init(paramStruct* pars) {
}
- if (args->bcfSrc & ARG_INTPLUS_BCFSRC_FMT_GL) {
+ const size_t in_vcf_nind = bcf_hdr_nsamples(vcfd->hdr);
- vcfd->lngl = new lnglStruct(vcfd->nGT, pars->nInd);
- }
- //////
+ if (metadata == NULL) {
+ pars->names = strArray_alloc(in_vcf_nind);
+ for (size_t i = 0;i < in_vcf_nind;++i) {
+ pars->names->add(vcfd->hdr->samples[i]);
+ }
+ DEVASSERT(pars->names->len == in_vcf_nind);
+ LOGADD("Found %d individuals in the VCF file", pars->names->len);
+ } else {
+ // -> set pars with vcfdata
+ const size_t mtd_nind = pars->metadata->indNames->len;
- LOG("Found %d individuals in the VCF file", pars->nInd);
- END_LOGSECTION;
+ size_t midx;
+ bool badorder = false;
+ uint64_t samples_metad2vcf[mtd_nind];
+
+ size_t newvidx = 0;
+ for (size_t vidx = 0;vidx < in_vcf_nind;++vidx) {
+ if (pars->metadata->indNames->find(vcfd->hdr->samples[vidx], &midx)) {
+ samples_metad2vcf[midx] = vidx;
+ if (newvidx != midx) {
+ badorder = true;
+ break;
+ }
+ ++newvidx;
+ } else {
+ LOGADD("Skipping individual %s in the VCF file. Reason: Individual not present in metadata file.", vcfd->hdr->samples[vidx]);
+ }
+ }
+ if (newvidx != mtd_nind) {
+ ERROR("All individuals listed in metadata file must be present in the VCF file. Program could only find %ld out of %ld individuals.", newvidx, mtd_nind);
+ }
+
+ if (badorder) {
+ ERROR("The order of individuals in the VCF file does not match the order of individuals in the metadata file. Please sort the individuals in the metadata file to match the order of individuals in the VCF file.");
+ }
+
+ pars->names = pars->metadata->indNames;
+
+ kstring_t tmp = KS_INITIALIZE;
+ for (size_t i = 0;i < mtd_nind;++i) {
+ ksprintf(&tmp, "%s", vcfd->hdr->samples[samples_metad2vcf[i]]);
+ if (i != mtd_nind - 1) {
+ kputc(',', &tmp);
+ }
+ }
+ if (bcf_hdr_set_samples(vcfd->hdr, tmp.s, 0)) {
+ ERROR("Could not set samples in bcf header.");
+ }
+ FREE(tmp.s);
+ int nsamples = bcf_hdr_nsamples(vcfd->hdr); // sanity check
+ ASSERT(nsamples == pars->metadata->indNames->len);
+
+ if (nsamples != in_vcf_nind) {
+ LOGADD("Will use %d individuals (out of %d) found in the VCF file.", nsamples, in_vcf_nind);
+ } else {
+ LOGADD("Found %d individuals in the VCF file", nsamples);
+ }
- pars->names = strArray_alloc(pars->nInd);
- for (int i = 0;i < pars->nInd;++i) {
- pars->names->add(vcfd->hdr->samples[i]);
}
+
+
+
+
vcfd->nContigs = vcfd->hdr->n[BCF_DT_CTG];
ASSERT(vcfd->nContigs > 0);
- check_consistency_args_pars(pars);
+ if (args->minInd == pars->names->len) {
+ DEVPRINT("-minInd %d is equal to the number of individuals found in file: %d. Setting -minInd to 0 (all).\n", args->minInd, pars->names->len);
+ args->minInd = 0;
+ }
+
+ if (pars->names->len == 1) {
+ ERROR("Only one individual found in the VCF file. At least two individuals are required to perform the analysis.");
+ }
+
+ if (pars->names->len < args->minInd) {
+ ERROR("Number of individuals in the VCF file is less than the minimum number of individuals provided by -minInd (%d).", args->minInd);
+ }
+
+
+ if (PROGRAM_WILL_USE_BCF_FMT_GL) {
+ int tag_id = bcf_hdr_id2int(vcfd->hdr, BCF_DT_ID, "GL");
+ if (!bcf_hdr_idinfo_exists(vcfd->hdr, BCF_HL_FMT, tag_id)) {
+ ERROR("GL tag not found in the VCF file.");
+ }
+ }
+
+ if (PROGRAM_WILL_USE_BCF_FMT_GT) {
+ int tag_id = bcf_hdr_id2int(vcfd->hdr, BCF_DT_ID, "GT");
+ if (!bcf_hdr_idinfo_exists(vcfd->hdr, BCF_HL_FMT, tag_id)) {
+ ERROR("GT tag not found in the VCF file.");
+ }
+ }
+ if (args->rmInvarSites && !(PROGRAM_WILL_USE_BCF_FMT_GT)) {
+ int tag_id = bcf_hdr_id2int(vcfd->hdr, BCF_DT_ID, "AD");
+ if (!bcf_hdr_idinfo_exists(vcfd->hdr, BCF_HL_FMT, tag_id)) {
+ ERROR("AD tag not found in the VCF file.");
+ }
+ }
+
+ END_LOGSECTION;
return(vcfd);
}
@@ -862,8 +1073,8 @@ void vcfData_destroy(vcfData* v) {
exit(BCF_CLOSE);
}
- if (v->lngl != NULL) {
- delete v->lngl;
+ if (v->gldata != NULL) {
+ gldata_destroy(v->gldata);
}
if (NULL != v->itr) {
@@ -881,111 +1092,236 @@ void vcfData_destroy(vcfData* v) {
delete v;
}
-void readSites(jgtmat_t* jgtm, vcfData* vcfd, paramStruct* pars, blobStruct* blob) {
+
+int vcfData::records_next() {
+ int ret = -42;
+
+ if (NULL == itr) {
+ // no region specified
+ ret = bcf_read(in_fp, hdr, rec);
+ } else {
+ // region reading using iterator
+ ret = bcf_itr_next(in_fp, itr, rec);
+
+ if (-1 > ret) {
+ ERROR("An error occurred while reading the VCF file.");
+ }
+ }
+
+ bcf_unpack(rec, DO_BCF_UNPACK);
+
+ ASSERT(0 == rec->errcode);
+
+ if (-1 == ret) {
+ // no more data
+ return 0;
+ }
+
+ return 1;
+}
+
+
+
+
+void readSites(jgtmat_t* jgtm, bblocks_t* bblocks, vcfData* vcfd, paramStruct* pars) {
BEGIN_LOGSECTION;
int skip_site = 0;
- int contig_i = -1;
char prev_contig[1024];
- int nBlocks = 1;
- if (NULL != blob) {
- ASSERT(blob->bootstraps->nBlocks > 1);
- nBlocks = blob->bootstraps->nBlocks;
+ const size_t nInd = pars->names->len;
- } else {
- // if block reading is disabled, read everything into one single block
- nBlocks = 1;
+
+
+ // while we read the data into memory, all we do for bootstrap is to keep track of the number of sites in each block
+
+
+ const bool doBlocks = (bblocks != NULL);
+ size_t nBlocks = (doBlocks) ? bblocks->n_blocks : 1;
+ uint64_t* nsites_per_block = NULL;
+ if (doBlocks) {
+ nsites_per_block = bblocks->nsites_per_block;
}
+ if (args->bcfSrc & ARG_INTPLUS_BCFSRC_FMT_GL) {
+ vcfd->gldata = gldata_alloc(vcfd->nGT, pars->names->len, 4096, ARG_GLDATA_TYPE_LNGL);
+ }
+
+
+
+ const char* this_contigName = NULL;
+
+ int contig_i = -1;
+ size_t block_i = 0;
+
+ bool isFirstRecord = true;
+ bool newBlock;
+
+ bool contig_changed;
+
+ size_t block_start_pos;
+ size_t block_end_pos;
+ size_t this_pos;
- // read vcf records until the end of the block (or the end of the vcf file)
- // if region is specified, records will not contain any sites outside the region
- // but the number of contigs in the header still includes all contigs in the vcf file
while (1 == (vcfd->records_next())) {
- pars->contig_changed = false;
-
- if (pars->nSites >= (size_t)pars->nSites_arrays_size) {
- // expand gl arrays
- if (args->bcfSrc & ARG_INTPLUS_BCFSRC_FMT_GL) {
- //TODO
- // NB> in full genome arrays use pars->nSites (for total)
- // in contig arrays use (per contig)
- lnglStruct* lngl = vcfd->lngl;
- const size_t oldSize = (size_t)pars->nSites_arrays_size;
- ASSERT((size_t)pars->nSites_arrays_size == lngl->size1);
- const size_t newSize = oldSize + 4096;
- pars->nSites_arrays_size = newSize;
- lngl->size1 = newSize;
-
- if (lngl != NULL) {
- lngl->size1 = newSize;
- REALLOC(double**, lngl->d, lngl->size1);
-
- for (size_t i = oldSize; i < lngl->size1;++i) {
- lngl->d[i] = (double*)malloc(lngl->size2 * sizeof(double));
- ASSERT(lngl->d[i] != NULL);
- for (size_t j = 0;j < lngl->size2;++j) {
- lngl->d[i][j] = NEG_INF;
- }
- }
- }
- }
+ if (vcfd->rec->pos >= 0) {
+ this_pos = vcfd->rec->pos;
+ } else {
+ NEVER;
+ }
+
+ if (vcfd->gldata != NULL && pars->nSites == vcfd->gldata->size) {
+ gldata_realloc(vcfd->gldata);
+ } else if (vcfd->gldata != NULL && pars->nSites > vcfd->gldata->size) {
+ NEVER;
}
- // if at the very beginning
- if (-1 == contig_i) {
+
+ newBlock = false;
+ contig_changed = false;
+
+ ASSERT(NULL != (this_contigName = bcf_hdr_id2name(vcfd->hdr, vcfd->rec ? vcfd->rec->rid : -1)));
+
+ if (!isFirstRecord) {
+ if (0 != strcmp(prev_contig, this_contigName)) {
+ // ** contig change **
+ contig_changed = true;
+ ++contig_i;
+ strncpy(prev_contig, this_contigName, sizeof(prev_contig));
+ if (prev_contig[sizeof(prev_contig) - 1] != '\0') {
+ ERROR("Contig name is too long");
+ }
+
+ if (doBlocks) {
+ bblocks_match_contig(bblocks, this_contigName, contig_i);
+
+ newBlock = true;
+ }
+ }
+
+ } else {
+ newBlock = true;
+ // if at the very beginning; first ever pos
+ contig_changed = false;
contig_i = 0;
- strncpy(prev_contig, vcfd->get_contig_name(), sizeof(prev_contig));
- if (prev_contig[sizeof(prev_contig) - 1] != '\0') {
- ERROR("Contig name is too long");
+ strncpy(prev_contig, this_contigName, sizeof(prev_contig));
+ ASSERT(prev_contig[sizeof(prev_contig) - 1] == '\0');
+ isFirstRecord = false;
+
+
+ if (doBlocks) {
+ bblocks_match_contig(bblocks, this_contigName, contig_i);
}
+
}
- // ** contig change **
- if (0 != strcmp(vcfd->get_contig_name(), prev_contig)) {
- if (contig_i >= vcfd->nContigs) {
- ERROR("Number of contigs in the VCF file header is larger than the number of contigs in the VCF file");
+ // ------------------------------------------------- //
+ // -> read data into memory
+
+
+ // -> first, if doBlocks, find which block we are in
+ if (doBlocks) {
+
+
+
+
+ while (1) {
+ block_start_pos = bblocks->block_start_pos[block_i];
+ block_end_pos = bblocks->block_end_pos[block_i];
+
+ if (block_i < nBlocks) {
+ if (this_pos > block_start_pos) {
+ if (this_pos >= block_end_pos) {
+ // inc '=' since end is exclusive
+ ++block_i;
+ newBlock = true;
+ continue;
+ } else if (this_pos < block_end_pos) {
+ break;
+ }
+
+ } else if (this_pos == block_start_pos) {
+ break;
+ } else {
+ NEVER;
+ }
+ } else if (block_i == nBlocks) {
+ // out of blocks but still not found
+ ERROR("Could not find the block for the current position %ld in contig %s", this_pos + 1, this_contigName);
+ } else {
+ NEVER;
+ }
+
}
- pars->contig_changed = true;
- ++contig_i;
+ ASSERT(bblocks->block_contig[block_i] == contig_i);
+
+ // DEVPRINT("Position %ld is located at block %ld (%s:%ld-%ld)", this_pos + 1, block_i, this_contigName, block_start_pos, block_end_pos);
- strncpy(prev_contig, vcfd->get_contig_name(), sizeof(prev_contig));
- if (prev_contig[sizeof(prev_contig) - 1] != '\0') {
- ERROR("Contig name is too long");
+ if (newBlock) {
+ bblocks->block_start_siteidx[block_i] = pars->nSites;
+ bblocks->block_contig[block_i] = contig_i;
}
+
}
+ // <- block bootstrapping prep
+
+ if (doBlocks && !newBlock && 0 == nsites_per_block[block_i]) {
+ //TODO add testcase forthis
+ // a previous site that belongs to this block was found before but was skipped for all indvs
+ // so treat it as a new block
+ newBlock = true;
+ }
- // read the needed bcf format/data fields
+ // -> then, read the needed bcf format/data fields
if (PROGRAM_WILL_USE_BCF_FMT_GL) {
- skip_site = site_read_GL(vcfd, pars);
+ skip_site = site_read_3GL(vcfd, pars, block_i, contig_changed);
} else if (PROGRAM_WILL_USE_BCF_FMT_GT) {
if (args->doJGTM) {
- skip_site = site_read_GT(jgtm, vcfd, pars);
+ skip_site = site_read_GT(jgtm, vcfd, pars, contig_changed);
} else {
- ERROR("Nothing to do.");
+ NEVER;
}
} else {
NEVER;
}
+ // -> skip site if needed
if (skip_site != 0) {
- IO::vprint(1, "Skipped site at %s:%ld", vcfd->get_contig_name(), vcfd->rec->pos + 1);
+ IO::vprint(1, "Skipped site at %s:%ld", vcfd->get_contig_name(), this_pos + 1);
pars->totSites++;
-
continue;
}
+ // -> if site is NOT skipped
+ if (doBlocks) {
+ nsites_per_block[block_i]++;
+ }
pars->nSites++;
pars->totSites++;
+
+ } // end sites loop
+
+
+
+
+ if (doBlocks) {
+ for (size_t i = 0;i < nBlocks;++i) {
+ if (nsites_per_block[i] == 0) {
+ ERROR("Block %ld has no sites", i);
+ }
+ if (PROGRAM_VERBOSITY_LEVEL >= 1) {
+ LOG("Block %ld has %ld sites", i, nsites_per_block[i]);
+
+ }
+ }
}
@@ -1000,14 +1336,9 @@ void readSites(jgtmat_t* jgtm, vcfData* vcfd, paramStruct* pars, blobStruct* blo
if (0 == pair_shared_nSites) {
ERROR("Pair %ld has no shared sites", pairidx);
}
- for (size_t i = 0;i < jgtm->size;++i) {
- // prob = count / total count
- jgtm->pm[pairidx][i] = (double)jgtm->m[pairidx][i] / pair_shared_nSites;
- }
}
}
-
LOG("Finished reading sites.");
LOG("Read %d (out of %d) contigs from the VCF file.", contig_i + 1, vcfd->nContigs);
LOG("Number of contigs retained: %d", contig_i + 1);
@@ -1016,33 +1347,4 @@ void readSites(jgtmat_t* jgtm, vcfData* vcfd, paramStruct* pars, blobStruct* blo
LOG("Total number of sites skipped for all individual pairs: %lu", pars->totSites - pars->nSites);
LOG("Total number of sites retained: %lu", pars->nSites);
END_LOGSECTION;
- vcfd->nSites = pars->nSites;
}
-
-int vcfData::records_next() {
- int ret = -42;
-
- if (NULL == itr) {
- // no region specified
- ret = bcf_read(in_fp, hdr, rec);
- } else {
- // region reading using iterator
- ret = bcf_itr_next(in_fp, itr, rec);
-
- if (-1 > ret) {
- ERROR("An error occurred while reading the VCF file.");
- }
- }
-
- bcf_unpack(rec, DO_BCF_UNPACK);
-
- ASSERT(0 == rec->errcode);
-
- if (-1 == ret) {
- // no more data
- return 0;
- }
-
- return 1;
-}
-
diff --git a/vcfReader.h b/vcfReader.h
index c04c601..dc39253 100644
--- a/vcfReader.h
+++ b/vcfReader.h
@@ -9,7 +9,11 @@
#include "io.h"
#include "paramStruct.h"
-
+// UNMOD: unmodified, do not perform any transform or normalization
+#define ARG_GLDATA_TYPE_UNMOD 0
+#define ARG_GLDATA_TYPE_GL 1
+#define ARG_GLDATA_TYPE_LNGL 2
+#define ARG_GLDATA_TYPE_LOG10GL 3
// BASES are: { A, C, G, T, BASE_UNOBSERVED }
@@ -22,10 +26,9 @@
typedef struct vcfData vcfData;
typedef struct gtData gtData;
-// typedef struct glData glData;
struct blobStruct;
-struct lnglStruct;
+typedef struct bblocks_t bblocks_t;
/* -------------------------------------------------------------------------- */
@@ -72,6 +75,48 @@ int require_index(paramStruct* pars);
/// @return 0 if no unpacking is required, otherwise return a BCF_UN_* value
int require_unpack();
+
+typedef struct gldata_t gldata_t;
+struct gldata_t {
+
+ /// @details known a priori (args->doEM)
+ uint8_t n_gls;
+
+ /// @details known a priori (pars->nInd; from vcfd->hdr->n[BCF_DT_SAMPLE])
+ size_t n_ind;
+
+ /// @details at runtime, worst case scenario: total number of sites in the vcf file header for all contigs+step_size
+ /// at the end of read_sites, equal to pars->nSites
+ size_t size;
+
+ /// @var step_size - step size for reallocating/expanding the number of gl arrays
+ /// step_size*n_gls is the added size when reallocating
+ size_t step_size;
+
+ /// @var type - type of genotype likelihoods
+ /// @details values: ARG_GLDATA_TYPE_*
+ uint8_t type;
+
+ /// @var d[n_ind][n_sites][n_gls]
+ /// @details
+ /// d[i] - (double**) array of all gls for individual i
+ /// d[i][j] - (double*) array of gls for individual i at site j
+ /// allocated iff !mis[i][j]; size=n_gls
+ /// otherwise NULL
+ /// d[i][j][k] - (double) gl for individual i at site j for genotype k
+ double*** d;
+
+ /// @var mis[n_ind][n_sites] - indicator for whether if an individual has missing data at a site
+ /// @details
+ /// mis[i] - array of bool indicators for missing data for individual i
+ /// mis[i][j] - bool indicator for missing data for individual i at site j
+ /// @note
+ /// mis[i][j] == true if individual i does not have data at site j,
+ /// OR, if it did have data but due to some filter we decided to ignore it
+ bool** mis;
+
+};
+
struct vcfData {
vcfFile* in_fp = NULL;
@@ -98,24 +143,15 @@ struct vcfData {
int DO_BCF_UNPACK;
int nContigs = 0;
- int nSites;
int* skipContigs = NULL;
- lnglStruct* lngl = NULL;
+ gldata_t* gldata = NULL;
int nGT = 0;
-
- /*
- * [nIndCmb][9+1]
- * last element contains total number of sites shared
- * //TODO is this still the case? is this being used?
- */
-
-
- // index of the unobserved allele in vcfd->rec->d.allele (if any)
- // set to -1 if no unobserved allele is found in d.alleles
+ // index of the unobserved allele in vcfd->rec->d.allele (if any)
+ // set to -1 if no unobserved allele is found in d.alleles
int allele_unobserved = -1;
int nJointClasses = 0;
@@ -141,9 +177,10 @@ struct vcfData {
};
-vcfData* vcfData_init(paramStruct* pars);
+vcfData* vcfData_init(paramStruct* pars, metadataStruct* metadata);
void vcfData_destroy(vcfData* v);
+
/*
* @template struct get_data
*
@@ -161,7 +198,6 @@ void vcfData_destroy(vcfData* v);
*/
template <typename T>
struct get_data {
- //TODO
T* data = NULL;
int size_e = 0;
@@ -184,7 +220,7 @@ struct get_data {
}
};
-void readSites(jgtmat_t* jgtm, vcfData* vcfd, paramStruct* pars, blobStruct* blob);
+void readSites(jgtmat_t* jgtm, bblocks_t* bblocks, vcfData* vcfd, paramStruct* pars);