From 876755bd128f4192c962e3e3ed7d749d85b6d59e Mon Sep 17 00:00:00 2001 From: yh202109 Date: Mon, 29 Jul 2024 21:33:02 -0400 Subject: [PATCH] v0.2.21 --- README.md | 10 +- docs/index.md | 4 - docs/statlab_corr_spearman_rho.rst | 164 --------- docs/statlab_corr_tau.rst | 211 ------------ docs/statlab_kappa.rst | 511 ----------------------------- docs/statlab_kappa2.rst | 345 ------------------- mtbp3/statlab/__init__.py | 1 - mtbp3/statlab/corr.py | 107 ------ mtbp3/statlab/kappa.py | 330 ------------------- tests/test_statlab_corr.py | 30 -- tests/test_statlab_kappa.py | 40 --- 11 files changed, 4 insertions(+), 1749 deletions(-) delete mode 100644 docs/statlab_corr_spearman_rho.rst delete mode 100644 docs/statlab_corr_tau.rst delete mode 100644 docs/statlab_kappa.rst delete mode 100644 docs/statlab_kappa2.rst delete mode 100644 mtbp3/statlab/__init__.py delete mode 100644 mtbp3/statlab/corr.py delete mode 100644 mtbp3/statlab/kappa.py delete mode 100644 tests/test_statlab_corr.py delete mode 100644 tests/test_statlab_kappa.py diff --git a/README.md b/README.md index 20f23611..e69ee4d0 100644 --- a/README.md +++ b/README.md @@ -23,13 +23,11 @@ Functions are grouped into: - ICH/MedDRA: `health.emt` - WHODrug: - Standard: - - CDISC*: `stdcdisc.lib` + - CDISC[*]: `stdcdisc.lib` - ISO: `stdiso.pdfsummary` -- StatLab*: - - Correlation: `statlab.corr` - - Reliability: `statlab.kappa` +- StatLab has been moved to a new package `mtbp3Lab` after v0.2.21 and will be extend to provide a broarder range of practicing topics. Enjoy! -* Documents are not executed while building. +[*] Documents are not executed while building. ## Table of Contents @@ -87,4 +85,4 @@ Hsu, Y. (2024). mtbp3: My tool box in Python [Software]. Retrieved from https:// url = {https://yh202109.github.io/mtbp3/index.html}, note = {Software} } -``` \ No newline at end of file +``` diff --git a/docs/index.md b/docs/index.md index 5a31424f..bdb98d8a 100755 --- a/docs/index.md +++ b/docs/index.md @@ -17,10 +17,6 @@ example_emt4.ipynb std_iso_pdf.ipynb std_iso_idmp.ipynb std_cdisc.ipynb -statlab_kappa.rst -statlab_kappa2.rst -statlab_corr_tau.rst -statlab_corr_spearman_rho.rst changelog.md contributing.md diff --git a/docs/statlab_corr_spearman_rho.rst b/docs/statlab_corr_spearman_rho.rst deleted file mode 100644 index fc48916c..00000000 --- a/docs/statlab_corr_spearman_rho.rst +++ /dev/null @@ -1,164 +0,0 @@ -.. - # Copyright (C) 2023-2024 Y Hsu - # - # This program is free software: you can redistribute it and/or modify - # it under the terms of the GNU General Public license as published by - # the Free software Foundation, either version 3 of the License, or - # any later version. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of - # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - # GNU General Public License for more details - # - # You should have received a copy of the GNU General Public license - # along with this program. If not, see - -.. role:: red-b - -.. role:: red - -############# -StatLab/Corr/NP/Spearman's Rho -############# - -:red-b:`Disclaimer:` -:red:`This page is provided only for studying and practicing. The author does not intend to promote or advocate any particular analysis method or software.` - -************* -Background -************* - -Spearman's rho (:math:`\rho`) is a statistic used for measuring rank correlation [1]_ . - -************* -Notation -************* - -Let :math:`(Y_{i1}, Y_{i2})` be a pair of random variables corresponding to the :math:`i` th sample where :math:`i = 1, \ldots, n`. - -.. list-table:: Observed Value - :widths: 10 10 10 - :header-rows: 1 - :name: tbl_count1 - - * - - - :math:`Y_{i1}` - - :math:`Y_{i2}` - * - **Sample:** 1 - - :math:`Y_{11}` - - :math:`Y_{12}` - * - **Sample:** 2 - - :math:`Y_{21}` - - :math:`Y_{22}` - * - :math:`\vdots` - - :math:`\vdots` - - :math:`\vdots` - * - **Sample:** :math:`n` - - :math:`Y_{n1}` - - :math:`Y_{n2}` - -Let :math:`(R_{i1}, R_{i2})` be the rank of :math:`Y_{i1}` and the rank of :math:`Y_{i2}`. -In the case of ties, one method is to assign the tied group with the average of unique ranks corresponding the tied group. -For the :math:`i` th sample, let -:math:`S_{i1,1}` be the number of observed values less than :math:`Y_{i1}`, -:math:`S_{i1,2}` be the number of observed values equal to :math:`Y_{i1}`, -and :math:`S_{i1,3}` be the number of observed values greater to :math:`Y_{i1}`. -We can calculate the rank of a single sample as - -.. math:: - :label: eq_rank - - R_{i1} = S_{i1,1} + \frac{S_{i1,2}+1}{2} = n - S_{i1,3} - \frac{S_{i1,2}-1}{2}. - -For a vector, ``pandas.DataFrame`` has the ``rank`` function with ``method='average'`` option to calculate rank as defined in :eq:`eq_rank`. -In ``R``, that can be calculated using the ``rank`` function with ``ties.method='average'`` option. -See reference [2]_ for ranking in ``Julia``. - -The Spearman's :math:`\rho` can be calculated as: - -.. math:: - :label: eq_rho - - \rho = \frac{\frac{1}{n}\sum_i R_{i1}R_{i2} - \frac{1}{4}(n+1)^2}{s_1 s_2}, - -where :math:`s_1^2 = \sum_i R_{i1}^2 - \frac{1}{4}(n+1)^2`, -and :math:`s_2^2 = \sum_i R_{i2}^2 - \frac{1}{4}(n+1)^2`. - -************* -Example - Group-1 -************* - -.. list-table:: Spearman's :math:`\rho = 1.0` - :widths: 10 10 10 - :header-rows: 1 - :name: tbl_ex1 - - * - - - :math:`Y_{i1}` - - :math:`Y_{i2}` - * - **Sample:** 1 - - 1 - - 4 - * - **Sample:** 2 - - 3 - - 6 - * - **Sample:** 3 - - 2 - - 5 - -.. list-table:: Spearman's :math:`\rho = -1.0` - :widths: 10 10 10 - :header-rows: 1 - :name: tbl_ex1 - - * - - - :math:`Y_{i1}` - - :math:`Y_{i2}` - * - **Sample:** 1 - - 1 - - 6 - * - **Sample:** 2 - - 3 - - 4 - * - **Sample:** 3 - - 2 - - 5 - -************* -How-to -************* - -To use ``scipy.stats`` [3]_: - -.. code:: python - - from scipy.stats import spearmanr - - y1 = [1, 3, 2] - y2 = [4, 6, 5] - - rho, p_value = spearmanr(y1, y2) - print("Spearman's rho:", rho) - -************* -More Details -************* - -Assume that :math:`Y_{i1} \sim \mathcal{D}`. -For continuous :math:`Y_{i1}`, if we can assume -that :math:`P(S_{i1,2}=1)=1` for all :math:`i`, -then :eq:`eq_rank` can be simplified as :math:`R_{i1} = S_{i1,1}+1`. -For a given sample size :math:`n`, and :math:`r \in \{1, \ldots, n\}`, the pmf of :math:`R_{i1}` is -:math:`P(R_{i1} = r) = \frac{1}{n}`, which does not depend on :math:`r` or :math:`\mathcal{D}` [4]_. - - -************* -Reference -************* - -.. [1] Wikipedia. (year). Spearman's rank correlation coefficient. https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient -.. [2] julialang.org. (2022). Ranking of elements of a vector. https://discourse.julialang.org/t/ranking-of-elements-of-a-vector/88293/4 -.. [3] scipy.org. (year). spearmanr. https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.spearmanr.html -.. [4] John Borkowski. (2014). Introduction to the Theory of Order Statistics and Rank Statistics. https://math.montana.edu/jobo/thainp/rankstat.pdf - diff --git a/docs/statlab_corr_tau.rst b/docs/statlab_corr_tau.rst deleted file mode 100644 index 86b664ce..00000000 --- a/docs/statlab_corr_tau.rst +++ /dev/null @@ -1,211 +0,0 @@ -.. - # Copyright (C) 2023-2024 Y Hsu - # - # This program is free software: you can redistribute it and/or modify - # it under the terms of the GNU General Public license as published by - # the Free software Foundation, either version 3 of the License, or - # any later version. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of - # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - # GNU General Public License for more details - # - # You should have received a copy of the GNU General Public license - # along with this program. If not, see - -.. role:: red-b - -.. role:: red - -############# -StatLab/Corr/NP/Kendall's Tau -############# - -:red-b:`Disclaimer:` -:red:`This page is provided only for studying and practicing. The author does not intend to promote or advocate any particular analysis method or software.` - -************* -Background -************* - -Kendall's tau (:math:`\tau`) is a statistic used for measuring rank correlation [1]_ [2]_. - -************* -Notation -************* - -Let :math:`(Y_{i1}, Y_{i2})` be a pair of random variables corresponding to the :math:`i` th sample where :math:`i = 1, \ldots, n`. - -.. list-table:: Observed Value - :widths: 10 10 10 - :header-rows: 1 - :name: tbl_count1 - - * - - - :math:`Y_{i1}` - - :math:`Y_{i2}` - * - **Sample:** 1 - - :math:`Y_{11}` - - :math:`Y_{12}` - * - **Sample:** 2 - - :math:`Y_{21}` - - :math:`Y_{22}` - * - :math:`\vdots` - - :math:`\vdots` - - :math:`\vdots` - * - **Sample:** :math:`n` - - :math:`Y_{n1}` - - :math:`Y_{n2}` - -Let :math:`Z_{ij1} \equiv sign(Y_{i1}-Y_{j1})`, :math:`Z_{ij2} \equiv sign(Y_{i2}-Y_{j2})`, -:math:`c = \sum_{i=1}^n \sum_{j < i} I(Z_{ij1}Z_{ij2}=1)`, -:math:`d = \sum_{i=1}^n \sum_{j < i} I(Z_{ij1}Z_{ij2}=-1)` -and :math:`t = \frac{n(n-1)}{2}`. -The coefficient :math:`\tau` (tau-a) can be calculated as - -.. math:: - :label: eq_tau1 - - \tau = \frac{ c - d }{t}. - -If there are no ties, the maximum value of :eq:`eq_tau1` is 1 at :math:`c=t`, -and the minimum is -1 at :math:`d=t`. - -:eq:`eq_tau1` can also be expressed as - -.. math:: - :label: eq_tau2 - - \tau =& \frac{2}{n(n-1)} \left( \sum_{i=1}^n \sum_{j < i} Z_{ij1}Z_{ij2} \right) \\ - =& \frac{1}{n(n-1)} \left( \sum_{i=1}^n \sum_{j=1}^n Z_{ij1}Z_{ij2} \right). - -Under independent sample assumption, for a fixed :math:`n`, we know that -:math:`E(Z_{ij1})=E(Z_{ij2})=0` and -:math:`Var(Z_{ij1})=Var(Z_{ij2})=1-\frac{1}{n}`. -From :eq:`eq_tau2`, we can see that :math:`\tau` is a type of correlation coefficient. - -If there are ties, the maximum value of :eq:`eq_tau1` becomes less then 1. -Consider the scenario that there are :math:`n_{t1}` groups of ties in :math:`\{Y_{i1}\}`, -and there are :math:`n_{t2}` groups of ties in :math:`\{Y_{i2}\}`. -Let :math:`n_{t1,k}` be the number of ties within the :math:`k` th group of ties in :math:`\{Y_{i1}\}`, -and :math:`n_{t2,k}` be the number of ties within the :math:`k` th group of ties in :math:`\{Y_{i2}\}` -The adjusted :math:`\tau` (tau-b) is calculated by replacing :math:`t` in :eq:`eq_tau1` with -:math:`t^* = \sqrt{\frac{1}{2}n(n-1)-\sum_{k=1}^{n_{t1}} \frac{1}{2}n_{t1,k}(n_{t1,k}-1)}\sqrt{\frac{1}{2}n(n-1)-\sum_{k=1}^{n_{t2}} \frac{1}{2}n_{t2,k}(n_{t2,k}-1)}` - -************* -Example - Group-1 -************* - -.. list-table:: Kendall's :math:`\tau = 1.0` - :widths: 10 10 10 - :header-rows: 1 - :name: tbl_ex1 - - * - - - :math:`Y_{i1}` - - :math:`Y_{i2}` - * - **Sample:** 1 - - 1 - - 4 - * - **Sample:** 2 - - 3 - - 6 - * - **Sample:** 3 - - 2 - - 5 - -.. list-table:: Kendall's :math:`\tau = -1.0` - :widths: 10 10 10 - :header-rows: 1 - :name: tbl_ex1 - - * - - - :math:`Y_{i1}` - - :math:`Y_{i2}` - * - **Sample:** 1 - - 1 - - 6 - * - **Sample:** 2 - - 3 - - 4 - * - **Sample:** 3 - - 2 - - 5 - -************* -How-to -************* - -To use ``scipy.stats`` [3]_: - -.. code:: python - - from scipy.stats import kendalltau - y1 = [1,3,2] - y2 = [4,6,5] - - tau, p_value = kendalltau(y1, y2) - print("Kendall's tau:", tau) - -************* -Lab Exercise -************* - -1. Show :math:`E(Z_{ij})=0`. - -************* -Algorithm - 1 -************* - -**WARNING: FOR SMALL SAMPLE SIZES ONLY** - -Note that the algorithm in this section is implement in ``mtbp3.stalab`` for illustration purpose. -Although the matrix form is closely representing :eq:`eq_tau2`, -the calculation time increases greatly when the sample size increases. -Other algorithms can be found in references. - -Let :math:`Y_{1} = (Y_{11}, \ldots, Y_{n1})` and :math:`Y_{2} = (Y_{12}, \ldots, Y_{n2})`. -Let :math:`\times` represent the matrix product, -:math:`\times_{car}` represent the Cartesian product, -:math:`\times_{ele}` represent the element-wise product, -:math:`g([(a,b)]) = [sign(a-b)]`. -and :math:`h(X_n) = 1_n \times X_n \times 1_n^T` -where :math:`X_n` is a size :math:`n` by :math:`n` matrix, and :math:`1_n` is a length :math:`n` one vector. -Both tau-a and tau-b can be calculated using the following steps: - -1. calculate components :math:`\tau_1 = g(Y_{1} \times_{car} Y_{1})` and :math:`\tau_2 = g(Y_{2} \times_{car} Y_{2})` -2. calculate :math:`\tau` as :math:`\tau = \frac{h(\tau_1 \times_{ele} \tau_2) }{ \sqrt{h(abs(\tau_1))}\sqrt{h(abs(\tau_2))} }` - -============= -How-to -============= - -To use ``mtbp3.corr``: - -.. code:: python - - import numpy as np - from mtbp3.corr import CorrCalculator - - size = 100 - y1 = np.random.randint(1, size+1, size=size).tolist() - y2 = np.subtract(np.random.randint(1, size+1, size=size),y1).tolist() - t = CorrCalculator([y1,y2]) - print("Kendall's tau (mtbp3.corr):", t.calculate_kendall_tau()) - -To create a scatter plot of ``y1`` and ``y2``: - -.. code:: python - - t.plot_y_list(axis_label=['y1','y2']) - - -************* -Reference -************* - -.. [1] Wikipedia. (year). Kendall rank correlation coefficient. https://en.wikipedia.org/wiki/Kendall_rank_correlation_coefficient -.. [2] Encyclopedia of Mathematics. (yeawr). Kendall tau metric. https://encyclopediaofmath.org/index.php?title=Kendall_tau_metric -.. [3] Scipy. (year). kendalltau. https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.kendalltau.html - diff --git a/docs/statlab_kappa.rst b/docs/statlab_kappa.rst deleted file mode 100644 index a2823b92..00000000 --- a/docs/statlab_kappa.rst +++ /dev/null @@ -1,511 +0,0 @@ -.. - # Copyright (C) 2023-2024 Y Hsu - # - # This program is free software: you can redistribute it and/or modify - # it under the terms of the GNU General Public license as published by - # the Free software Foundation, either version 3 of the License, or - # any later version. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of - # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - # GNU General Public License for more details - # - # You should have received a copy of the GNU General Public license - # along with this program. If not, see - - -.. role:: red-b - -.. role:: red - -.. role:: bg-ltsteelblue - - - -############# -StatLab/Reli/Cohen's Kappa -############# - -:red-b:`Disclaimer:` -:red:`This page is provided only for studying and practicing. The author does not intend to promote or advocate any particular analysis method or software.` - -************* -Background -************* - -Cohen's kappa (:math:`\kappa`) is a statistic used for describing inter-ratter reliability of two ratters (or intra-rater) with categorical rating outcomes [1]_. -Please note that there are also additional considerations for the use of :math:`\kappa` for quantifying agreement [2]_ [3]_ . - -************* -Notation -************* - -For two ratters and two categories rating, let :math:`Y_{r,i} \in \{v_j; j=1,2\}` represent rating -from rater :math:`r=1,2` for sample :math:`i = 1, \ldots, n`. -Let :math:`N_{j_1,j_2}` represent the total number of sample received ratings :math:`(v_{j_1}, v_{j_2})` from two raters, where :math:`j_1,j_2 \in \{1,2\}`. - -.. list-table:: Counts for 2 categories - :widths: 10 10 10 10 - :header-rows: 1 - - * - - - Ratter 2: :math:`v_1` - - Ratter 2: :math:`v_2` - - Row Total - * - **Ratter 1:** :math:`v_1` - - :math:`N_{11}` - - :math:`N_{12}` - - :math:`N_{1\bullet}` - * - **Ratter 1:** :math:`v_2` - - :math:`N_{21}` - - :math:`N_{22}` - - :math:`N_{2\bullet}` - * - **Column Total** - - :math:`N_{\bullet 1}` - - :math:`N_{\bullet 2}` - - :math:`n` - -For two ratters and three or more categories rating, let :math:`Y_{r,i} \in \{v_1,v_2,v_3, \ldots, v_J \}` represent rating -from rater :math:`r=1,2` for sample :math:`i = 1, \ldots, n`. -Let :math:`N_{j_1,j_2}` represent the total number of sample received ratings :math:`(v_{j_1}, v_{j_2})` from two raters, where :math:`j_1,j_2 \in \{1,\ldots,J\}`. - -.. list-table:: Counts for 3 or more categories - :widths: 10 10 10 10 10 10 - :header-rows: 1 - - * - - - Ratter 2: :math:`v_1` - - Ratter 2: :math:`v_2` - - Ratter 2: :math:`v_3` - - :math:`\ldots` - - Row Total - * - **Ratter 1:** :math:`v_1` - - :math:`N_{11}` - - :math:`N_{12}` - - :math:`N_{13}` - - :math:`\ldots` - - :math:`N_{1\bullet}` - * - **Ratter 1:** :math:`v_2` - - :math:`N_{21}` - - :math:`N_{22}` - - :math:`N_{23}` - - :math:`\ldots` - - :math:`N_{2\bullet}` - * - **Ratter 1:** :math:`v_3` - - :math:`N_{31}` - - :math:`N_{32}` - - :math:`N_{33}` - - :math:`\ldots` - - :math:`N_{3\bullet}` - * - :math:`\vdots` - - :math:`\vdots` - - :math:`\vdots` - - :math:`\vdots` - - :math:`\ddots` - - :math:`\vdots` - * - **Column Total** - - :math:`N_{\bullet 1}` - - :math:`N_{\bullet 2}` - - :math:`N_{\bullet 3}` - - :math:`\ldots` - - :math:`n` - -The observed raw percentage of agreement is defined as - -.. math:: - - p_O = \sum_{j=1}^J N_{jj} / n - -where :math:`J \geq 2` is the size of value set. - -Assume that - -.. math:: - (N_{1\bullet}, \ldots N_{J\bullet}) \sim multi(n, (p_{r=1,1}, \ldots, p_{r=1,J})), - -and - -.. math:: - (N_{\bullet 1}, \ldots N_{\bullet J}) \sim multi(n, (p_{r=2,1}, \ldots, p_{r=2,J})), - -with :math:`\sum_j N_{j \bullet} = \sum_j N_{\bullet j} = n` -and :math:`\sum_j p_{r=1,j} = \sum_j p_{r=2, j} = 1`. - -Under independence assumption, the expected number of agreement is estimated by -:math:`\sum_{j=1}^J\hat{E}_{j} = \frac{1}{n}\sum_{j=1}^J N_{\bullet j} N_{j\bullet} \equiv n p_E`. - -The Cohen's :math:`\kappa` statistic is calculated as - -.. math:: - \kappa = \frac{p_O - p_E}{1-p_E}. - -The SE of :math:`\kappa` is calculated as - -.. math:: - \sqrt{\frac{p_O(1-p_O)}{n(1-p_E)^2}}. - -************* -Interpretation of Cohen's Kappa Suggested in Literature -************* - -There are several groups of interpretation. Some roughly (not-strictly) defined types are listed below: - -1. Table based interpretation: a shared interpretation simplifies application process and provides a easy to compare values. -2. Interpretation based on Approximated model based confidence interval or Bootstrap confidence intervals with a preselected criterion -3. Bayesian inference based interpretation [8]_ - -Cohen (1960) [4]_ suggested the Kappa result be interpreted as follows: - -.. list-table:: Cohen's Kappa Interpretation (Cohen, 1960) - :widths: 10 10 - :header-rows: 1 - - * - Value of :math:`\kappa` - - Interpretation - * - :math:`-1 \leq \kappa \leq 0` - - indicating no agreement - * - :math:`0 < \kappa \leq 0.2` - - none to slight - * - :math:`0.2 < \kappa \leq 0.4` - - fair - * - :math:`0.4 < \kappa \leq 0.6` - - moderate - * - :math:`0.6 < \kappa \leq 0.8` - - substantial - * - :math:`0.8 < \kappa \leq 1` - - almost perfect agreement - -Interpretation suggested by McHugh (2012) [5]_: - -.. list-table:: Cohen's Kappa Interpretation (McHugh, 2012) - :widths: 10 10 10 - :header-rows: 1 - - * - Value of :math:`\kappa` - - Level of Agreement - - % of Data That Are Reliable - * - :math:`-1 \leq \kappa \leq 0` - - Disagreement - - NA - * - :math:`0-.20` - - None - - :math:`0-4%` - * - :math:`.21-.39` - - Minimal - - :math:`4-15%` - * - :math:`.40-.59` - - Weak - - :math:`15-35%` - * - :math:`.60-.79` - - Moderate - - :math:`35-63%` - * - :math:`.80-.90` - - Strong - - :math:`64-81%` - * - Above.90 - - Almost Perfect - - :math:`82-100%` - -As discussed by Sim and Wright [6]_ , biases and other factors could have impact on the interpretation. - -************* -Example - Group-1 -************* - -.. list-table:: Cohen's :math:`\kappa = 0` - :widths: 10 10 10 10 - :header-rows: 1 - - * - - - Ratter 2: :math:`v_1` - - Ratter 2: :math:`v_2` - - Row Total - * - **Ratter 1:** :math:`v_1` - - 9 - - 21 - - 30 - * - **Ratter 1:** :math:`v_2` - - 21 - - 49 - - 70 - * - **Column Total** - - 30 - - 70 - - 100 - -.. list-table:: Cohen's :math:`\kappa = 0` - :widths: 10 10 10 10 - :header-rows: 1 - - * - - - Ratter 2: :math:`v_1` - - Ratter 2: :math:`v_2` - - Row Total - * - **Ratter 1:** :math:`v_1` - - 49 - - 21 - - 70 - * - **Ratter 1:** :math:`v_2` - - 21 - - 9 - - 30 - * - **Column Total** - - 70 - - 30 - - 100 - -.. list-table:: Cohen's :math:`\kappa = 1` - :widths: 10 10 10 10 - :header-rows: 1 - - * - - - Ratter 2: :math:`v_1` - - Ratter 2: :math:`v_2` - - Row Total - * - **Ratter 1:** :math:`v_1` - - 30 - - 0 - - 30 - * - **Ratter 1:** :math:`v_2` - - 0 - - 70 - - 70 - * - **Column Total** - - 30 - - 70 - - 100 - -.. list-table:: Cohen's :math:`\kappa = 1` - :widths: 10 10 10 10 - :header-rows: 1 - - * - - - Ratter 2: :math:`v_1` - - Ratter 2: :math:`v_2` - - Row Total - * - **Ratter 1** :math:`v_1` - - 50 - - 0 - - 50 - * - **Ratter 1:** :math:`v_2` - - 0 - - 50 - - 50 - * - **Column Total** - - 50 - - 50 - - 100 - -.. list-table:: Cohen's :math:`\kappa = -1` - :widths: 10 10 10 10 - :header-rows: 1 - - * - - - Ratter 2: :math:`v_1` - - Ratter 2: :math:`v_2` - - Row Total - * - **Ratter 1:** :math:`v_1` - - 0 - - 50 - - 50 - * - **Ratter 1:** :math:`v_2` - - 50 - - 0 - - 50 - * - **Column Total** - - 50 - - 50 - - 100 - -.. list-table:: Cohen's :math:`\kappa = -0.7241379310344827` - :widths: 10 10 10 10 - :header-rows: 1 - - * - - - Ratter 2: :math:`v_1` - - Ratter 2: :math:`v_2` - - Row Total - * - **Ratter 1:** :math:`v_1` - - 0 - - 30 - - 30 - * - **Ratter 1:** :math:`v_2` - - 70 - - 0 - - 70 - * - **Column Total** - - 70 - - 30 - - 100 - - -************* -How-to -************* - -To use ``sklearn.metrics`` (stable): - -.. code:: python - - from sklearn.metrics import cohen_kappa_score - y1 = ['v2'] * 70 + ['v1'] * 30 - y2 = ['v1'] * 70 + ['v2'] * 30 - print("Cohen's kappa:", cohen_kappa_score(y1, y2)) - -To use ``mtbp3.statlab`` (testing): - -.. code:: python - - from mtbp3.statlab import kappa - y1 = ['v2'] * 70 + ['v1'] * 30 - y2 = ['v1'] * 70 + ['v2'] * 30 - kappa = kappa.KappaCalculator([y1,y2]) - print("Cohen's kappa:", kappa.cohen_kappa) - -============= -Bootstrap CI -============= - -To use ``mtbp3.statlab``: - -.. testsetup:: * - - from mtbp3.statlab import kappa - y1 = ['v2'] * 70 + ['v1'] * 30 - y2 = ['v1'] * 70 + ['v2'] * 30 - kappa = kappa.KappaCalculator(y1,y2) - -.. testcode:: - - print( kappa.bootstrap_cohen_ci(n_iterations=1000, confidence_level=0.95, out_digits=6) ) - -Output: - -.. testoutput:: - - Cohen's kappa: -0.724138 - Confidence Interval (95.0%): [-0.907669, -0.496558] - - -Note that examples of using ``SAS/PROC FREQ`` and ``R`` package ``vcd`` for calculating :math:`\kappa` can be found in reference [7]_ . - -============= -Bubble Plot -============= - -To create a bubble plot using ``mtbp3.statlab``: - -.. code:: python - - from mtbp3.statlab import kappa - - fruits = ['Apple', 'Orange', 'Pear'] - np.random.seed(100) - r1 = np.random.choice(fruits, size=100).tolist() - r2 = np.random.choice(fruits, size=100).tolist() - - kappa = KappaCalculator([r1,r2], stringna='NA') - print("Cohen's kappa (mtbp3.statlab): "+str(kappa.cohen_kappa)) - print("Number of raters per sample: "+str(kappa.n_rater)) - print("Number of rating categories: "+str(kappa.n_category)) - print("Number of sample: "+str(kappa.y_count.shape[0])) - - kappa.create_bubble_plot() - -Output: - -.. testoutput:: - - Cohen's kappa (mtbp3.statlab): 0.06513872135102527 - Number of raters per sample: 2.0 - Number of rating categories: 3 - Number of sample: 100 - -.. figure:: /_static/fig/statlab_kappa_fig1.svg - :align: center - :alt: bubble plot - -Sometimes monitoring individual raters rates might be needed for the interpretation of :math:`\kappa`. -To create a bubble plot with individual raters summary using ``mtbp3.statlab``: - -.. code:: python - - kappa.create_bubble_plot(hist=True) - -.. figure:: /_static/fig/statlab_kappa_fig2.svg - :align: center - :alt: bubble plot with hist - -Note that the agreed counts are on the 45 degree line. -To put agreed counts on the -45 degree line: - -.. code:: python - - kappa.create_bubble_plot(hist=True, reverse_y=True) - -.. figure:: /_static/fig/statlab_kappa_fig3.svg - :align: center - :alt: bubble plot with hist - reverse - -************* -Lab Exercise -************* - -Assume that there are two raters responsible for rating 2 studies with a sample size of 100 for each study. -Assume that the you are tasked with studying the characteristics of :math:`\kappa`. - -For the first study, the first rater completed the rating with marginal rates -following a multinomial distribution (100, (1/3, 1/3, 1/3)). -Afterwards, assume that you filled -a portion (:math:`0 < r < 1`) of the sample's ratings as a second rater with exactly the same rating as the first rater, -and filled out the rest with random ratings following the same distribution as the first rater. - -For the second study, the first rater completed the rating with marginal rates -following a multinomial distribution (100, (0.9, 0.05, 0.05)). -Afterwards, assume that you filled -a portion (:math:`0 < r < 1`) of the sample's ratings as a second rater with exactly the same rating as the first rater, -and filled out the rest with random ratings following the same distribution as the first rater. - -1. Find the relationship between :math:`r` and :math:`\kappa` for these two studies. - -************* -Extensions -************* - -Some scenarios discussed by Hallgren (2012) [9]_ include: - -- the **prevalence** problem: one category has much higher percentage than other categories and causes :math:`\kappa` to be low. -- the **bias** problem: there are substantial differences in marginal distributions and causes :math:`\kappa` tend to be high. -- unequal importance - -(Please note that this is not an exhaustive list.) - -************* -Weighted :math:`\kappa` -************* - -Let :math:`w_{j_1,j_2}` represent the weight given to total number of sample received ratings :math:`(v_{j_1}, v_{j_2})` from two raters, where :math:`j_1,j_2 \in \{1,\ldots,J\}`. -The weighted :math:`\kappa` is calculated as - -.. math:: - \kappa = 1- \frac{\sum_{j_1=1}^J\sum_{j_2=1}^J w_{j_1,j_2}N_{j_1,j_2}}{\sum_{j_1=1}^J\sum_{j_2=1}^J w_{j_1,j_2}\hat{E}_{j_1, j_2}}. - -(There shall be another page discussing weighted methods and variations) - - - -************* -Reference -************* - -.. [1] Wikipedia. (year). Cohen's kappa. https://en.wikipedia.org/wiki/Cohen%27s_kappa. -.. [2] Uebersax, J. (year). Kappa Coefficients: A Critical Appraisal. https://www.john-uebersax.com/stat/kappa.htm#procon. -.. [3] Brennan, R. L., & Prediger, D. J. (1981). Coefficient Kappa: Some Uses, Misuses, and Alternatives. Educational and Psychological Measurement, 41(3), 687-699. https://doi.org/10.1177/0013164481041003070 -.. [4] Cohen, J. (1960). A Coefficient of Agreement for Nominal Scales. Educational and Psychological Measurement, 20(1), 37-46. https://doi.org/10.1177/001316446002000104 -.. [5] McHugh M. L. (2012). Interrater reliability: the kappa statistic. Biochemia medica, 22(3), 276-282. https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3900052/ -.. [6] Sim, J., Wright, C. C. (2005). The Kappa Statistic in Reliability Studies: Use, Interpretation, and Sample Size Requirements, Physical Therapy, Volume 85, Issue 3, Pages 257-268, https://doi.org/10.1093/ptj/85.3.257 -.. [7] PSU. STAT504: Measure of Agreement: Kappa. https://online.stat.psu.edu/stat504/lesson/11/11.2/11.2.4 -.. [8] Basu, S., Banerjee, M., & Sen, A. (2000). Bayesian inference for kappa from single and multiple studies. Biometrics, 56(2), 577–582. https://doi.org/10.1111/j.0006-341x.2000.00577.x -.. [9] Hallgren K. A. (2012). Computing Inter-Rater Reliability for Observational Data: An Overview and Tutorial. Tutorials in quantitative methods for psychology, 8(1), 23–34. https://doi.org/10.20982/tqmp.08.1.p023 -.. [10] Landis, J. R., & Koch, G. G. (1977). The measurement of observer agreement for categorical data. Biometrics, 33(1), 159–174. \ No newline at end of file diff --git a/docs/statlab_kappa2.rst b/docs/statlab_kappa2.rst deleted file mode 100644 index 92f5292f..00000000 --- a/docs/statlab_kappa2.rst +++ /dev/null @@ -1,345 +0,0 @@ -.. - # Copyright (C) 2023-2024 Y Hsu - # - # This program is free software: you can redistribute it and/or modify - # it under the terms of the GNU General Public license as published by - # the Free software Foundation, either version 3 of the License, or - # any later version. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of - # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - # GNU General Public License for more details - # - # You should have received a copy of the GNU General Public license - # along with this program. If not, see - -.. role:: red-b - -.. role:: red - -############# -StatLab/Reli/Fleiss's Kappa -############# - -:red-b:`Disclaimer:` -:red:`This page is provided only for studying and practicing. The author does not intend to promote or advocate any particular analysis method or software.` - -************* -Background -************* - -Fleiss's kappa (:math:`\kappa`) is a statistic used for describing inter-rater reliability of multiple independent raters -with categorical rating outcomes [1]_ [2]_. - -************* -Notation -************* - -Assume there are the same :math:`R+N_0` (:math:`\geq 2+N_0`) raters and each of :math:`n` samples were rated by :math:`R` randomly selected raters and were not rated by the rest of :math:`N_0` raters. -For :math:`J` categories rating, let :math:`Y_{r,i} \in \{v_0, v_1,v_2,\ldots, v_J \}` represent rating -from rater :math:`r=1,2,\ldots,R+N_0` for sample :math:`i = 1, \ldots, n`. -Let :math:`N_{ij}` represent the total number of raters gave rating :math:`(v_j)` to sample :math:`i`, where :math:`j \in \{0, 1,\ldots,J\}`. -The value :math:`v_0` represent raters did not rate the sample :math:`i` and :math:`N_{i0}=N_0` is a fixed number for all :math:`i`. -Therefore, :math:`v_0` will not be included in the discussion below. - -.. list-table:: Count of Ratings - :widths: 10 10 10 10 10 10 - :header-rows: 1 - :name: tbl_count1 - - * - - - :math:`v_1` - - :math:`v_2` - - :math:`\ldots` - - :math:`v_J` - - Row Total - * - **Sample:** 1 - - :math:`N_{11}` - - :math:`N_{12}` - - :math:`\ldots` - - :math:`N_{1J}` - - :math:`R` - * - **Sample:** 2 - - :math:`N_{21}` - - :math:`N_{22}` - - :math:`\ldots` - - :math:`N_{2J}` - - :math:`R` - * - **Sample:** 3 - - :math:`N_{31}` - - :math:`N_{32}` - - :math:`\ldots` - - :math:`N_{3J}` - - :math:`R` - * - :math:`\vdots` - - :math:`\vdots` - - :math:`\vdots` - - :math:`\ddots` - - :math:`\vdots` - - :math:`\vdots` - * - **Sample:** :math:`n` - - :math:`N_{n1}` - - :math:`N_{n2}` - - :math:`\ldots` - - :math:`N_{nJ}` - - :math:`R` - * - **Column total** - - :math:`N_{\bullet 1}` - - :math:`N_{\bullet 2}` - - :math:`\ldots` - - :math:`N_{\bullet J}` - - :math:`nR` - -The observed averaged agreement is calculated as - -.. math:: - :label: eq_obs1 - - \bar{p}_O = \frac{1}{n} \sum_{i=1}^n p_{O,i}, - -where :math:`p_{O,i} = \frac{1}{R(R-1)} \left(\sum_{j=1}^J N_{ij}(N_{ij}-1)\right)= \frac{1}{R(R-1)} \left(\sum_{j=1}^J N_{ij}^2 - R\right)`. - -The expected agreement is calculated as - -.. math:: - :label: eq_exp1 - - \bar{p}_E = \sum_{j=1}^J p_{E,j}^2, - -where :math:`p_{E,j} = \frac{N_{\bullet j}}{nR}`. - -The Fleiss's :math:`\kappa` statistic is calculated from :eq:`eq_obs1` and :eq:`eq_exp1` as - -.. math:: - :label: eq_kappa1 - - \kappa = \frac{\bar{p}_O - \bar{p}_E}{1-\bar{p}_E}. - -************* -Example - Group-1 -************* - -.. list-table:: Fleiss's :math:`\kappa = 1.0` - :widths: 10 10 10 10 10 - :header-rows: 1 - - * - - - :math:`v_1` - - :math:`v_2` - - :math:`v_3` - - :math:`v_4` - * - **Sample 1** - - 12 - - 0 - - 0 - - 0 - * - **Sample 2** - - 0 - - 12 - - 0 - - 0 - * - **Sample 3** - - 0 - - 0 - - 12 - - 0 - * - **Sample 4** - - 0 - - 0 - - 12 - - 0 - * - **Sample 5** - - 0 - - 0 - - 0 - - 12 - * - **Column Total** - - 12 - - 12 - - 24 - - 12 - - -.. list-table:: Fleiss's :math:`\kappa` = -0.0909090909090909 - :widths: 10 10 10 10 10 - :header-rows: 1 - - * - - - :math:`v_1` - - :math:`v_2` - - :math:`v_3` - - :math:`v_4` - * - **Sample 1** - - 3 - - 3 - - 3 - - 3 - * - **Sample 2** - - 3 - - 3 - - 3 - - 3 - * - **Sample 3** - - 3 - - 3 - - 3 - - 3 - * - **Sample 4** - - 3 - - 3 - - 3 - - 3 - * - **Sample 5** - - 3 - - 3 - - 3 - - 3 - * - **Column Total** - - 15 - - 15 - - 15 - - 15 - -************* -How-to -************* - -To use both ``statsmodels.stats.inter_rater`` and ``mtbp3.statlab``: - -.. testcode:: - - import statsmodels.stats.inter_rater as ir - from mtbp3.statlab import kappa - - r1 = ['NA'] * 20 + ['B'] * 50 + ['A'] * 30 - r2 = ['A'] * 20 + ['NA'] * 20 + ['B'] * 60 - r3 = ['A'] * 40 + ['NA'] * 20 + ['B'] * 30 + ['C'] * 10 - r4 = ['B'] * 60 + ['NA'] * 20 + ['C'] * 10 + ['A'] * 10 - r5 = ['C'] * 60 + ['A'] * 10 + ['B'] * 10 + ['NA'] * 20 - data = [r1, r2, r3, r4, r5] - kappa = KappaCalculator(data, stringna='NA') - - print("Fleiss's kappa (stasmodels.stats.inter_rater): "+str(ir.fleiss_kappa(kappa.y_count))) - print("Fleiss's kappa (mtbp3.statlab): "+str(kappa.fleiss_kappa)) - print("Number of raters per sample: "+str(kappa.n_rater)) - print("Number of rating categories: "+str(kappa.n_category)) - print("Number of sample: "+str(kappa.y_count.shape[0])) - -Output: - -.. testoutput:: - - Fleiss's kappa (stasmodels.stats.inter_rater): -0.14989733059548255 - Fleiss's kappa (mtbp3.statlab): -0.14989733059548255 - Number of raters per sample: 4.0 - Number of rating categories: 3 - Number of sample: 100 - -************* -Lab Exercise -************* - -1. Find Bootstrap CI of Fleiss's kappa. (see the function of Cohen's kappa CI) - -************* -More Details -************* - -:eq:`eq_obs1` corresponds to the observed -probability of having agreement for a sample from two randomly selected raters estimated from :numref:`Tabel %s `. -:eq:`eq_exp1` corresponds to the expected -probability of having agreement for a sample from two randomly selected raters under the assumption of no agreement, -which corresponds to the assumption of :math:`(N_{i1},\ldots, N_{iJ}) \sim multi(R, (p_1,\ldots, p_J))` where :math:`R>4`. - - -Let :math:`S_{p2} = \sum_j p_j^2`, :math:`S_{p3} = \sum_j p_j^3`, and :math:`S_{p4} = \sum_j p_j^4`. -The equation :eq:`eq_kappa1` can be expressed as [2]_ :sup:`(Eq. 9)`, - -.. math:: - - \kappa = \frac{\sum_{i=1}^{n}\sum_{j=1}^J N_{ij}^2 - nR\left(1+(R-1) S_{p2} \right)}{nR(R-1)(1- S_{p2} )} - - -Note that Fleiss (1971) assumed large :math:`n` and fixed :math:`p_j` while deriving variance of kappa. -Please see the Fleiss (1971) for more discussions. -The variance of :math:`\kappa` under the assumption of no agreement beyond chance can be approximated as: - -.. math:: - :label: eq_kappa2_vk - - var(\kappa) = c(n,R,\{p_j\}) var\left(\sum_{j=1}^J N_{1j}^2 \right), - -where - -.. math:: - - c(n,R,\{p_j\}) = n^{-1}\left(R(R-1)\left(1-S_{p2}\right)\right)^{-2}, - -and - -.. math:: - :label: eq_kappa2_vn2 - - var\left(\sum_{j} N_{ij}^2 \right) - =& E\left( \left(\sum_{j} N_{ij}^2\right)^2\right) - \left(E\left(\sum_{j} N_{ij}^2\right)\right)^2 \\ - =& E\left(\sum_{j} N_{ij}^4\right) + E\left(\sum_j\sum_{k \neq j} N_{ij}^2 N_{ik}^2 \right) - \left(E\left(\sum_{j} N_{ij}^2\right)\right)^2. - -To calculate :eq:`eq_kappa2_vn2`, -we can use the MGF, :math:`\left(\sum_{j}p_je^{t_j}\right)^R`, to derive -:math:`E\left(N_{ij}^2\right) = Rp_j + R(R-1)p_j^2`, and -:math:`E\left(N_{ij}^3\right) = Rp_j + 3R(R-1)p_j^2 + R(R-1)(R-2)p_j^3`. - -The first element of :eq:`eq_kappa2_vn2` can be calculated as [2]_ :sup:`(Eq. 12)` - -.. math:: - :label: eq_kappa2_vn3 - - E\left(\sum_{j} N_{ij}^4\right) - = R + 7R(R-1)S_{p2} + 6R(R-1)(R-2)S_{p3} + R(R-1)(R-2)(R-3)S_{p4} - -The third element of :eq:`eq_kappa2_vn2` can be calculated as [2]_ :sup:`(Eq. 14)` - -.. math:: - :label: eq_kappa2_vn4 - - \left(E\left(\sum_{j} N_{ij}^2\right)\right)^2 - =& R^2\left(1 + (R-1)S_{p2} \right)^2 \\ - =& R^2 + R^2(R-1)\left(2 S_{p2} + (R-1)S_{p2}^2\right) - -The second element of :eq:`eq_kappa2_vn2` can be calculated, using -:math:`E\left( N_{ij}^2 N_{ik}^2 \right) = R(R-1)p_j(p_k+(R-2)p_k^2) + R(R-1)(R-2)p_j^2(p_k+(R-3)p_k^2)`, as - -.. math:: - :label: eq_kappa2_vn5 - - E\left( \sum_j\sum_{k \neq j} N_{ij}^2 N_{ik}^2 \right) - =& R(R-1) + R(R-1)(2R-5)S_{p2} - - 2R(R-1)(R-2)S_{p3} \\ - &- R(R-1)(R-2)(R-3)S_{p4} + R(R-1)(R-2)(R-3) S_{p2}^2 - -Combining :eq:`eq_kappa2_vn3`, :eq:`eq_kappa2_vn4`, and :eq:`eq_kappa2_vn5`, -:eq:`eq_kappa2_vn2` can be calculated as [2]_ :sup:`(Eq. 15)` - -.. math:: - - var\left(\sum_{j} N_{ij}^2 \right) - = 2R(R-1)\left(S_{p2} - (2R-3)S_{p2}^2 + 2(R-2)S_{p3}\right). - -Let :math:`s^2` be the estimated variance of :math:`\kappa` using :eq:`eq_kappa2_vk`. -Under the hypothesis of no agreement beyond chances, the limit distribution :math:`\kappa/s` would be a standard normal distribution. -The value of :math:`\kappa/s` then could be used to describe if the overall agreement is greater then by chance alone [2]_. - -************* -Lab Exercise -************* - -2. Find :math:`Cov(N_{i1},N_{i2})` under no agreement assumption. - -************* -Reference -************* - -.. [1] Wikipedia. (year). Fleiss's kappa. https://en.wikipedia.org/wiki/Fleiss%27_kappa -.. [2] Fleiss, J. L. (1971). Measuring nominal scale agreement among many raters. Psychological Bulletin, 76(5), 378-382. https://doi.org/10.1037/h0031619 - diff --git a/mtbp3/statlab/__init__.py b/mtbp3/statlab/__init__.py deleted file mode 100644 index ebf38adf..00000000 --- a/mtbp3/statlab/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .kappa import * diff --git a/mtbp3/statlab/corr.py b/mtbp3/statlab/corr.py deleted file mode 100644 index f3131fb7..00000000 --- a/mtbp3/statlab/corr.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (C) 2023-2024 Y Hsu -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public license as published by -# the Free software Foundation, either version 3 of the License, or -# any later version. -#j -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details -# -# You should have received a copy of the GNU General Public license -# along with this program. If not, see - -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt - -class CorrCalculator: - """ - A class for calculating correlation and plotting scatter plots. - - Parameters: - - y: A pandas DataFrame or a list - - If y is a pandas DataFrame, it must be a 2-dimensional DataFrame with at least 2 columns and 2 rows. - - If y is a list, it must be a list with at least 2 elements, where each element is a list of strings or numbers. - - remove_na: bool, optional (default=True) - - If True, remove rows with missing values (NA) from the input data. - - If False, raise a ValueError if the input data contains missing values (NA). - - Methods: - - calculate_kendall_tau(): - - Calculates the Kendall's tau correlation coefficient between the first two columns of the input data. - - Returns the Kendall's tau coefficient as a float. - - - plot_y_list(loc=[0,1], axis_label=['x','y']): - - Plots a scatter plot of the input data. - - loc: list, optional (default=[0,1]) - - The indices of the columns to be plotted on the x and y axes. - - axis_label: list, optional (default=['x','y']) - - The labels for the x and y axes of the scatter plot. - """ - - def __init__(self, y, remove_na=True): - - assert isinstance(y, (pd.DataFrame, list)), "y must be either a pandas DataFrame or a list" - if isinstance(y, pd.DataFrame): - assert y.ndim == 2, "y must be a 2-dimensional DataFrame" - assert y.shape[0] >= 2 and y.shape[1] >= 2, "y must be a pd.DataFrame with at least 2 columns and 2 rows" - if remove_na == True: - self.y_df = y.dropna() - else: - if y.isna().any().any(): - raise ValueError("Input data contains missing values (NA)") - self.y_shape0, self.y_shape1 = self.y_df.shape - self.y_list = self.y_df.values.tolist() - else: - assert isinstance(y, list) and len(y) >= 2, "y must be a list with at least 2 elements" - assert all(isinstance(x, list) for x in y), "all elements of y must be lists" - assert all(isinstance(x, (str, int)) for sublist in y for x in sublist if x is not None), "all elements of y must be strings or numbers" - assert all(len(x) == len(y[0]) for x in y), "all sublists in y must have the same length" - self.y_shape0 = len(y[0]) - self.y_shape1 = len(y) - self.y_list = y - self.y_df = pd.DataFrame(self.y_list).T.dropna() - - return - - @staticmethod - def __g(x): - result = [np.sign(sublist[0] - sublist[1]) for sublist in x] - return result - - def calculate_kendall_tau(self): - """ - Calculates the Kendall's tau correlation coefficient between the first two columns of the input data. - - Returns: - - tau: float - - The Kendall's tau correlation coefficient. - """ - tau1 = np.sign(np.repeat(self.y_list[0], self.y_shape0)-np.tile(self.y_list[0], self.y_shape0)) - tau2 = np.sign(np.repeat(self.y_list[1], self.y_shape0)-np.tile(self.y_list[1], self.y_shape0)) - tau = np.sum(np.multiply(tau1,tau2))/np.sqrt(np.multiply(np.sum(np.abs(tau1)),np.sum(np.abs(tau2)))) - return tau - - - def plot_y_list(self, loc=[0,1], axis_label=['x','y']): - """ - Plots a scatter plot of the input data. - - Parameters: - - loc: list, optional (default=[0,1]) - - The indices of the columns to be plotted on the x and y axes. - - axis_label: list, optional (default=['x','y']) - - The labels for the x and y axes of the scatter plot. - """ - plt.scatter(self.y_list[loc[0]], self.y_list[loc[1]]) - plt.xlabel(axis_label[0]) - plt.ylabel(axis_label[1]) - plt.title('Scatter Plot') - plt.show() - -if __name__ == "__main__": - - pass diff --git a/mtbp3/statlab/kappa.py b/mtbp3/statlab/kappa.py deleted file mode 100644 index a156b6ea..00000000 --- a/mtbp3/statlab/kappa.py +++ /dev/null @@ -1,330 +0,0 @@ -# Copyright (C) 2023-2024 Y Hsu -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public license as published by -# the Free software Foundation, either version 3 of the License, or -# any later version. -#j -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details -# -# You should have received a copy of the GNU General Public license -# along with this program. If not, see - -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -from sklearn.utils import resample -import os -import seaborn as sns -import random - - -class KappaCalculator: - """ - A class for calculating Cohen's kappa and Fleiss' kappa. - - Parameters: - - y: The input data. It can be either a pandas DataFrame or a list. - - infmt: The format of the input data. Allowed values are 'sample_list', 'sample_df', 'count_sq_df', and 'count_df'. - - stringna: The string representation of missing values. - - Methods: - - bootstrap_cohen_ci(n_iterations, confidence_level, outfmt, out_digits): Calculates the bootstrap confidence interval for Cohen's kappa. - - """ - - def __init__(self, y, infmt='sample_list', stringna="stringna"): - """ - Initializes the KappaCalculator object. - - Parameters: - - y: The input data. It can be either a pandas DataFrame or a list. - - infmt: The format of the input data. Allowed values are 'sample_list', 'sample_df', 'count_sq_df', and 'count_df'. - - stringna: The string representation of missing values. - - Raises: - - ValueError: If the value of infmt is invalid. - - AssertionError: If the input data does not meet the required conditions. - - """ - - assert isinstance(y, (pd.DataFrame, list)), "y must be either a pandas DataFrame or a list" - if isinstance(y, pd.DataFrame): - assert y.ndim == 2, "y must be a 2-dimensional DataFrame" - assert y.shape[0] >= 2 and y.shape[1] >= 2, "y must be a pd.DataFrame with at least 2 columns and 2 rows" - else: - assert isinstance(y, list) and len(y) >= 2, "y must be a list with at least 2 elements" - assert all(isinstance(x, list) for x in y), "all elements of y must be lists" - assert all(isinstance(x, (str, int)) for sublist in y for x in sublist if x is not None), "all elements of y must be strings or numbers" - assert all(len(x) == len(y[0]) for x in y), "all sublists in y must have the same length" - - if infmt not in ['sample_list', 'sample_df', 'count_sq_df', 'count_df']: - raise ValueError("Invalid value for infmt. Allowed values are 'sample_list', 'sample_df', 'count_sq_df' and 'count_df'.") - - if infmt == 'sample_list' or infmt == 'sample_df': - if infmt == 'sample_list': - self.y_list = self.__convert_2dlist_to_string(y, stringna=stringna) - self.y_df = pd.DataFrame(self.y_list).T - else: - self.y_df = y.replace({np.nan: stringna, None: stringna}) - self.y_df = self.y_df.applymap(lambda x: str(x) if isinstance(x, (int, float)) else x) - self.y_list = self.y_df.values.tolist() - - self.y_count = self.y_df.apply(pd.Series.value_counts, axis=1).fillna(0) - if stringna in self.y_count.columns: - column_values = self.y_count[stringna].unique() - assert len(column_values) == 1, f"Total number in value '{stringna}' must be the same for all sample" - self.y_count.drop(columns=stringna, inplace=True) - tmp_row_sum = self.y_count.sum(axis=1) - assert tmp_row_sum.eq(tmp_row_sum[0]).all(), "Total number of raters per sample must be equal" - self.category = self.y_count.columns - self.n_category = len(self.category) - self.n_rater = tmp_row_sum[0] - - if self.n_rater == 2: - self.y_count_sq = pd.crosstab(self.y_list[0], self.y_list[1], margins = False, dropna=False) - i = self.y_count_sq.index.union(self.y_count_sq.columns, sort=True) - self.y_count_sq.reindex(index=i, columns=i, fill_value=0) - else: - self.y_count_sq = None - - elif infmt == 'count_sq_df': - assert y.shape[0] == y.shape[1], "y must be a square DataFrame" - self.y_count_sq = y - self.category = y.columns - self.n_category = len(self.category) - self.n_rater = 2 - tmp_count = self.y_count_sq.unstack().reset_index(name='count') - self.y_df = tmp_count.loc[np.repeat(tmp_count.index.values, tmp_count['count'])] - self.y_df.drop(columns='count', inplace=True) - self.y_list = self.y_df.values.tolist() - self.y_count = self.y_df.apply(pd.Series.value_counts, axis=1).fillna(0) - - elif infmt == 'count_df': - self.y_count = y - if stringna in self.y_count.columns: - column_values = self.y_count[stringna].unique() - assert len(column_values) == 1, f"All values in column '{stringna}' must be the same" - self.y_count.drop(columns=stringna, inplace=True) - tmp_row_sum = self.y_count.sum(axis=1) - assert tmp_row_sum.eq(tmp_row_sum[0]).all(), "Row sums of y must be equal" - self.category = self.y_count.columns - self.n_category = len(self.category) - self.n_rater = tmp_row_sum[0] - self.y_count_sq= None - self.y_list = None - self.y_df = None - - else: - self.y_list = None - self.y_df = None - self.y_count= None - self.y_count_sq= None - self.category = None - self.n_rater = None - self.n_category = None - return - - - if self.n_rater == 2: - if self.y_list is not None: - self.cohen_kappa = self.__calculate_cohen_kappa(self.y_list[0],self.y_list[1]) - else: - self.cohen_kappa = None - else: - self.cohen_kappa = None - - if self.n_rater >= 2: - if self.y_count is not None: - self.fleiss_kappa = self.__calculate_fleiss_kappa(self.y_count) - else: - self.fleiss_kappa = None - else: - self.fleiss_kappa = None - - return - - @staticmethod - def __convert_2dlist_to_string(y=[], stringna=""): - """ - Converts a 2-dimensional list to a string representation. - - Parameters: - - y: The input list. - - stringna: The string representation of missing values. - - Returns: - - The converted list. - - """ - for i in range(len(y)): - if any(isinstance(x, (int)) for x in y[i]): - y[i] = [str(x) if x is not None else stringna for x in y[i]] - return y - - @staticmethod - def __calculate_cohen_kappa(y1, y2): - """ - Calculates Cohen's kappa. - - Parameters: - - y1: The first rater's ratings. - - y2: The second rater's ratings. - - Returns: - - The calculated Cohen's kappa value. - - """ - total_pairs = len(y1) - observed_agreement = sum(1 for i in range(total_pairs) if y1[i] == y2[i]) / total_pairs - unique_labels = set(y1 + y2) - expected_agreement = sum((y1.count(label) / total_pairs) * (y2.count(label) / total_pairs) for label in unique_labels) - return (observed_agreement - expected_agreement) / (1 - expected_agreement) - - @staticmethod - def __calculate_fleiss_kappa(y): - """ - Calculates Fleiss' kappa. - - Parameters: - - y: The count matrix. - - Returns: - - The calculated Fleiss' kappa value. - - """ - nR = y.values.sum() - p = y.values.sum(axis = 0)/nR - Pbar_E = (p ** 2).sum() - R = y.values.sum(axis = 1)[0] - Pbar_O = (((y ** 2).sum(axis=1) - R) / (R * (R - 1))).mean() - - return (Pbar_O - Pbar_E) / (1 - Pbar_E) - - def bootstrap_cohen_ci(self, n_iterations=1000, confidence_level=0.95, outfmt='string', out_digits=6): - """ - Calculates the bootstrap confidence interval for Cohen's kappa. - - Parameters: - - n_iterations: The number of bootstrap iterations. - - confidence_level: The desired confidence level. - - outfmt: The output format. Allowed values are 'string' and 'list'. - - out_digits: The number of digits to round the output values. - - Returns: - - If outfmt is 'string', returns a string representation of the result. - - If outfmt is 'list', returns a list containing the result values. - - """ - assert isinstance(n_iterations, int) and n_iterations > 1, "n_iterations must be an integer greater than 1" - assert isinstance(confidence_level, (float)) and 0 < confidence_level < 1, "confidence_level must be a number between 0 and 1" - - if self.n_rater != 2: - return [] - - y1 = self.y_list[0] - y2 = self.y_list[1] - kappa_values = [] - idx = range(len(y1)) - for _ in range(n_iterations): - idxr = resample(idx) - y1r = [y1[i] for i in idxr] - y2r = [y2[i] for i in idxr] - - kappa = self.__calculate_cohen_kappa(y1r, y2r) - kappa_values.append(kappa) - - lower_percentile = (1 - confidence_level) / 2 - upper_percentile = 1 - lower_percentile - lower_bound = np.percentile(kappa_values, lower_percentile * 100) - upper_bound = np.percentile(kappa_values, upper_percentile * 100) - if outfmt=='string': - return "Cohen's kappa: {:.{}f}".format(self.cohen_kappa, out_digits) + "\nConfidence Interval ({}%): [{:.{}f}, {:.{}f}]".format(confidence_level * 100, lower_bound, out_digits, upper_bound, out_digits) - else: - return [self.cohen_kappa, n_iterations, confidence_level, lower_bound, upper_bound] - - def create_bubble_plot(self, out_path="", axis_label=[], max_size_ratio=0, hist=False, reverse_y=False): - """ - Creates a bubble plot based on the y_count_sq matrix. - - Parameters: - - out_path (str): The output path to save the plot. If not provided, the plot will be displayed. - - title (str): The title of the plot. If not provided, the default title is 'Bubble Plot'. - - axis_label (list): A list of two strings representing the labels for the x-axis and y-axis. - If not provided, the default labels are ['Rater 1', 'Rater 2']. - - max_size_ratio (int): The maximum size ratio for the bubbles. The size of the bubbles is determined by the values in the y_count_sq matrix. - The maximum size of the bubbles will be max_size_ratio times the maximum value in the matrix. - If not provided, the default value is 100. - - Returns: - - None - - Raises: - - None - - """ - if self.n_rater == 2 and self.y_count_sq is not None and self.y_count_sq.shape[0] == self.y_count_sq.shape[1] and self.y_count_sq.shape[0] > 0: - categories = self.y_count_sq.columns - n_categories = len(categories) - - r1 = [] - r2 = [] - sizes = [] - for i1, c1 in enumerate(categories): - for i2, c2 in enumerate(categories): - r1.append(c1) - r2.append(c2) - sizes.append(self.y_count_sq.iloc[i1, i2]) - df0 = pd.DataFrame({'r1': r1, 'r2': r2, 'sizes': sizes}) - df0['r1'] = pd.Categorical(df0['r1']) - df0['r2'] = pd.Categorical(df0['r2']) - df0['agree'] = np.where(df0['r1'] == df0['r2'], 'agree', 'disagree') - max_size_ratio = max_size_ratio if max_size_ratio >= 1 else max(1,int((6000/max(sizes)) / n_categories)) - if hist: - sns.jointplot( - data=df0, x="r1", y="r2", kind="scatter", - height=5, ratio=3, marginal_ticks=True, - marginal_kws={"hue": df0['agree'], "multiple": "stack", "weights": sizes, "shrink":.5, "legend": False}, - joint_kws={"hue": df0['agree'], "size": sizes, "legend": False, "sizes":(min(sizes)*max_size_ratio, max(sizes)*max_size_ratio)} - ) - if not reverse_y: - tmp1 = plt.ylim() - plt.ylim(tmp1[1], tmp1[0]) - else: - sns.scatterplot(data=df0, x="r1", y="r2", size="sizes", hue="agree", sizes=(min(sizes)*max_size_ratio, max(sizes)*max_size_ratio), legend=False) - tmp1 = plt.xlim() - tmp1d = ((tmp1[1] - tmp1[0])/n_categories) - plt.xlim(tmp1[0] - tmp1d, tmp1[1] + tmp1d) - plt.ylim(tmp1[0] - tmp1d, tmp1[1] + tmp1d) - if reverse_y: - tmp1 = plt.ylim() - plt.ylim(tmp1[1], tmp1[0]) - - for i in range(len(df0)): - plt.text(df0['r1'][i], df0['r2'][i], df0['sizes'][i], ha='center', va='center') - - if not axis_label: - axis_label = ['Rater 1', 'Rater 2'] - plt.xlabel(axis_label[0]) - plt.ylabel(axis_label[1]) - - plt.tight_layout() - if out_path: - try: - if os.path.isdir(out_path): - plt.savefig(os.path.join(out_path, "bubble_plot.svg")) - else: - plt.savefig(out_path) - except Exception as e: - print("Error saving the figure:", str(e)) - else: - plt.show() - else: - print("Cannot create bubble plot. y_count_sq is not a square non-empty matrix.") - -if __name__ == "__main__": - - pass diff --git a/tests/test_statlab_corr.py b/tests/test_statlab_corr.py deleted file mode 100644 index 0e4683b7..00000000 --- a/tests/test_statlab_corr.py +++ /dev/null @@ -1,30 +0,0 @@ -import unittest -import pandas as pd -import numpy as np -from mtbp3.statlab.corr import CorrCalculator -import matplotlib.pyplot as plt - -class TestCorrCalculator(unittest.TestCase): - - def setUp(self): - self.y_df = pd.DataFrame({'x': [1, 2, 3, 4, 5], 'y': [2, 4, 6, 8, 10]}) - self.y_list = [[1, 2, 3, 4, 5], [2, 4, 6, 8, 10]] - self.corr_calculator_df = CorrCalculator(self.y_df) - self.corr_calculator_list = CorrCalculator(self.y_list) - - def test_calculate_kendall_tau_with_dataframe(self): - expected_tau = 1.0 - tau = self.corr_calculator_df.calculate_kendall_tau() - self.assertEqual(tau, expected_tau) - - def test_calculate_kendall_tau_with_list(self): - expected_tau = 1.0 - tau = self.corr_calculator_list.calculate_kendall_tau() - self.assertEqual(tau, expected_tau) - - def test_plot_y_list(self): - self.corr_calculator_df.plot_y_list() - # No assertion, just checking if the plot is displayed correctly - -if __name__ == "__main__": - unittest.main() \ No newline at end of file diff --git a/tests/test_statlab_kappa.py b/tests/test_statlab_kappa.py deleted file mode 100644 index d808eda9..00000000 --- a/tests/test_statlab_kappa.py +++ /dev/null @@ -1,40 +0,0 @@ -import unittest -from mtbp3.statlab.kappa import KappaCalculator -import statsmodels.stats.inter_rater as ir - -class TestKappaCalculator(unittest.TestCase): - - def setUp(self): - y1 = ['B'] * 70 + ['A'] * 30 - y2 = ['A'] * 70 + ['B'] * 30 - y3 = ['A'] * 50 + ['B'] * 30 + ['C'] * 20 - y4 = ['B'] * 40 + ['C'] * 40 + ['A'] * 20 - y5 = ['C'] * 60 + ['A'] * 20 + ['B'] * 20 - data = [y1, y2, y3, y4, y5] - self.c1 = KappaCalculator([y1, y2]) - self.c2 = KappaCalculator(data) - - def test_cohen(self): - gt0 = ir.cohens_kappa(self.c1.y_count_sq) - self.assertAlmostEqual(gt0.kappa, self.c1.cohen_kappa, places=6) - self.assertIsInstance(self.c1.cohen_kappa, float) - self.assertGreaterEqual(self.c1.cohen_kappa, -1) - self.assertLessEqual(self.c1.cohen_kappa, 1) - - def test_bootstrap_cohen_ci(self): - result = self.c1.bootstrap_cohen_ci(n_iterations=1000, confidence_level=0.95, out_digits=6) - self.assertIsInstance(result, str) - self.assertIn("Cohen's kappa:", result) - self.assertIn("Confidence Interval", result) - - def test_fleiss_kappa(self): - gt1 = ir.fleiss_kappa(self.c1.y_count) - gt2 = ir.fleiss_kappa(self.c2.y_count) - self.assertAlmostEqual(gt1, self.c1.fleiss_kappa, places=6) - self.assertAlmostEqual(gt2, self.c2.fleiss_kappa, places=6) - self.assertIsInstance(gt2, float) - self.assertGreaterEqual(gt2, -1) - self.assertLessEqual(gt2, 1) - -if __name__ == "__main__": - unittest.main() \ No newline at end of file