diff --git a/notebooks/example.ipynb b/notebooks/example.ipynb index 22d90a9..f2a5cfc 100644 --- a/notebooks/example.ipynb +++ b/notebooks/example.ipynb @@ -15,7 +15,6 @@ } ], "source": [ - "\n", "from deepdiagnostics import models\n", "from deepdiagnostics import data\n", "from deepdiagnostics.utils.config import Config\n", @@ -89,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -97,7 +96,7 @@ "text/plain": [ "{'common': {'out_dir': './DeepDiagnosticsResources/results/',\n", " 'temp_config': './DeepDiagnosticsResources/temp/temp_config.yml',\n", - " 'sim_location': 'deepdiagnosticsResources/simulators',\n", + " 'sim_location': './DeepDiagnosticsResources/simulators',\n", " 'random_seed': 42},\n", " 'model': {'model_engine': 'SBIModel'},\n", " 'data': {'data_engine': 'H5Data',\n", @@ -128,7 +127,7 @@ " 'metrics': {'AllSBC': {}, 'CoverageFraction': {}, 'LC2ST': {}}}" ] }, - "execution_count": 6, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -149,7 +148,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -160,8 +159,8 @@ " [--model_engine {SBIModel}] [--data_path DATA_PATH]\n", " [--data_engine {H5Data,PickleData}] [--simulator SIMULATOR]\n", " [--out_dir OUT_DIR]\n", - " [--metrics [{,CoverageFraction,AllSBC,LC2ST}]]\n", - " [--plots [{,CDFRanks,CoverageFraction,Ranks,TARP,LC2ST,PPC,Parity,PriorPC}]]\n", + " [--metrics {,CoverageFraction,AllSBC,LC2ST} [{,CoverageFraction,AllSBC,LC2ST} ...]]\n", + " [--plots {,CDFRanks,CoverageFraction,Ranks,TARP,LC2ST,PPC,Parity,PriorPC} [{,CDFRanks,CoverageFraction,Ranks,TARP,LC2ST,PPC,Parity,PriorPC} ...]]\n", "\n", "options:\n", " -h, --help show this help message and exit\n", @@ -186,12 +185,10 @@ " `utils.register_simulator` method.\n", " --out_dir OUT_DIR Where the results will be saved. Path need not exist,\n", " it will be created.\n", - " --metrics [{,CoverageFraction,AllSBC,LC2ST}]\n", - " List of metrics to run. To not run any, supply\n", - " `--metrics `\n", - " --plots [{,CDFRanks,CoverageFraction,Ranks,TARP,LC2ST,PPC,Parity,PriorPC}]\n", - " List of plots to run. To not run any, supply `--plots\n", - " `\n" + " --metrics {,CoverageFraction,AllSBC,LC2ST} [{,CoverageFraction,AllSBC,LC2ST} ...]\n", + " List of metrics to run.\n", + " --plots {,CDFRanks,CoverageFraction,Ranks,TARP,LC2ST,PPC,Parity,PriorPC} [{,CDFRanks,CoverageFraction,Ranks,TARP,LC2ST,PPC,Parity,PriorPC} ...]\n", + " List of plots to run.\n" ] } ], @@ -332,15 +329,14 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "['CoverageFraction', 'TARP']\n", - "100%|████████████████████████████████████████| 100/100 [00:00<00:00, 755.86it/s]\n" + "100%|████████████████████████████████████████| 100/100 [00:00<00:00, 725.57it/s]\n" ] } ], @@ -359,6 +355,178 @@ "\"Coverage" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can do a similar thing with metrics, or with plot metrics or plots. " + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n", + "/Users/maggiev-local/Library/Caches/pypoetry/virtualenvs/deepdiagnostics-081AeCAa-py3.10/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "! diagnose --model_path ../resources/savedmodels/sbi/sbi_linear_from_data.pkl --data_path ../resources/saveddata/data_validation.h5 --simulator MySimulator --metrics LC2ST" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[K12656823, 0.025117976419460897, 0.3210162208447511, 0.07601853392691404, 0.06380:\u001b[K3287, 1.0, 1.0, 0.3491858283608714, 0.11148460600402221, 0.1705053499709257, 1.0], [0.18886563228152808, 0.999801565848368, 0.06420094428578094, 0.003829460887442848, 1.0, 0.0, 0.05521830659929361, 1.0, 1.0, 1.0], [0.3723571672317456, 0.04190096758672124, 0.05917814671785937, 1.0, 0.5279380279033324, 1.0, 0.3939235158158101, 1.0, 0.3331214449554769, 0.14278821453735335], [0.16794501054145394, 1.0, 0.2836227076540423, 1.0, 0.08725893205243684, 1.0, 1.0, 1.0, 0.2898180261624975, 1.0], [3.4497582568349117e-10, 0.9999999999997726, 2.2475195038396123e-09, 0.9999999860851057, 0.0, 0.01498136204169076, 0.9999999999999996, 1.0, 1.0, 5.1250789168122424e-08]], \"lc2st_null_hypothesis_probabilities\": [[[0.2790388229755212, 0.2735488578185722, 8.397499558887578e-05, 0.002544143029782342, 0.4337491005580647, 0.5281736584242632, 0.2941875894651199, 0.8471557950221957, 0.491588056016956, 0.14904058563692402], [0.9298861402948934, 0.5799854862294562, 0.3862690266799903, 0.3032391090498666, 0.7976109456527416, 0.581394460307604, 0.6422201235768321, 0.9667699549263491, 0.4300242827109272, 0.603196699074676], [0.7458949626530249, 0.4908091805562187, 0.3295099292259487, 0.9048431566532888, 0.4475631382010551, 0.37204375915039756, 0.37192176501806684, 0.2599461309841308, 0.12187398110432535, 0.28911814717222706], [0.9715750712017363, 0.9302578076591757, 0.280485962197362, 0.7789217374607884, 0.5768391118918303, 0.6121690665788355, 0.8770606717117648, 0.5942009041559981, 0.6127097617959039, 0.9487840622625356], [0.5326295893366368, 0.7700871663694474, 0.6654461398938305, 0.8528700301216272, 0.8800724382630881, 0.6637915760733999, 0.8461285354871109, 0.8827762176374074, 0.2243871033069973, 0.6636540545834573]], [[0.049262331605416376, 0.0071008394:\u001b[K\u0007\u001b[H\u001b[2J\u001b[H\u001b[H\u001b[2J\u001b[H{\"lc2st_probabilities\": [[0.1983312341145994, 0.19218918728498446, 1.0, 0.20351917821653287, 1.0, 1.0, 0.3491858283608714, 0.11148460600402221, 0.1705053499709257, 1.0], [0.18886563228152808, 0.999801565848368, 0.06420094428578094, 0.003829460887442848, 1.0, 0.0, 0.05521830659929361, 1.0, 1.0, 1.0], [0.3723571672317456, 0.04190096758672124, 0.05917814671785937, 1.0, 0.5279380279033324, 1.0, 0.3939235158158101, 1.0, 0.3331214449554769, 0.14278821453735335], [0.16794501054145394, 1.0, 0.2836227076540423, 1.0, 0.08725893205243684, 1.0, 1.0, 1.0, 0.2898180261624975, 1.0], [3.4497582568349117e-10, 0.9999999999997726, 2.2475195038396123e-09, 0.9999999860851057, 0.0, 0.01498136204169076, 0.9999999999999996, 1.0, 1.0, 5.1250789168122424e-08]], \"lc2st_null_hypothesis_probabilities\": [[[0.2790388229755212, 0.2735488578185722, 8.397499558887578e-05, 0.002544143029782342, 0.4337491005580647, 0.5281736584242632, 0.2941875894651199, 0.8471557950221957, 0.491588056016956, 0.14904058563692402], [0.9298861402948934, 0.5799854862294562, 0.3862690266799903, 0.3032391090498666, 0.7976109456527416, 0.581394460307604, 0.6422201235768321, 0.9667699549263491, 0.4300242827109272, 0.603196699074676], [0.7458949626530249, 0.4908091805562187, 0.3295099292259487, 0.9048431566532888, 0.4475631382010551, 0.37204375915039756, 0.37192176501806684, 0.2599461309841308, 0.12187398110432535, 0.28911814717222706], [0.9715750712017363, 0.9302578076591757, 0.280485962197362, 0.7789217374607884, 0.5768391118918303, 0.6121690665788355, 0.8770606717117648, 0.5942009041559981, 0.6127097617959039, 0.9487840622625356], [0.5326295893366368, 0.7700871663694474, 0.6654461398938305, 0.8528700301216272, 0.8800724382630881, 0.6637915760733999, 0.8461285354871109, 0.8827762176374074, 0.2243871033069973, 0.6636540545834573]], [[0.049262331605416376, 0.0071008394:\u001b[K" + ] + } + ], + "source": [ + "! cat DeepDiagnosticsResources/results/diagnostic_metrics.json | less" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/src/deepdiagnostics/metrics/__init__.py b/src/deepdiagnostics/metrics/__init__.py index 18dc3df..7f683ed 100644 --- a/src/deepdiagnostics/metrics/__init__.py +++ b/src/deepdiagnostics/metrics/__init__.py @@ -2,8 +2,14 @@ from deepdiagnostics.metrics.coverage_fraction import CoverageFraction from deepdiagnostics.metrics.local_two_sample import LocalTwoSampleTest as LC2ST +def void(*args, **kwargs): + def void2(*args, **kwargs): + return None + return void2 + + Metrics = { - "": lambda **kwargs: None, + "": void, CoverageFraction.__name__: CoverageFraction, AllSBC.__name__: AllSBC, "LC2ST": LC2ST diff --git a/src/deepdiagnostics/plots/__init__.py b/src/deepdiagnostics/plots/__init__.py index a63b51c..3e0bd1a 100644 --- a/src/deepdiagnostics/plots/__init__.py +++ b/src/deepdiagnostics/plots/__init__.py @@ -7,8 +7,14 @@ from deepdiagnostics.plots.parity import Parity from deepdiagnostics.plots.predictive_prior_check import PriorPC +def void(*args, **kwargs): + def void2(*args, **kwargs): + return None + return void2 + + Plots = { - "": lambda **kwargs: None, + "": void, CDFRanks.__name__: CDFRanks, CoverageFraction.__name__: CoverageFraction, Ranks.__name__: Ranks,