From 79b54f2ae54d3f57b35454703d0f67dbabce66ea Mon Sep 17 00:00:00 2001
From: LukasDrews97 <drews.lukas@yahoo.com>
Date: Thu, 11 Jan 2024 14:40:04 +0100
Subject: [PATCH 01/65] update pipfile

---
 Pipfile      |   12 +-
 Pipfile.lock | 1722 ++++++++++++++++++++++++++++----------------------
 2 files changed, 976 insertions(+), 758 deletions(-)

diff --git a/Pipfile b/Pipfile
index 25da2c20..401d8f4b 100644
--- a/Pipfile
+++ b/Pipfile
@@ -9,13 +9,17 @@ numpy = "*"
 scikit-learn = "*"
 
 [dev-packages]
-pytest = "*"
-pytest-flake8 = "*"
-pytest-pydocstyle = "*"
-pytest-cov = "*"
+pytest = "7.1.2"
+flake8 = "4.0.1"
+pytest-flake8 = "1.1.1"
+pydocstyle = "6.1.1"
+pytest-pydocstyle = "2.3.0"
+pytest-cov = "3.0.0"
 twine = "*"
 sphinx = "4.1.1"
 sphinx-rtd-theme = "0.5.2"
+black = "22.10.0"
+pre-commit = "2.20.0"
 
 [extras]
 ray = "*"
diff --git a/Pipfile.lock b/Pipfile.lock
index 1151d408..676474e5 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "4f36f76cfc52dc83b74cc00b92af24b8ff556ca22dc2f21401224c39659346eb"
+            "sha256": "1043424ee27a00eb7644d011194b69a29cd890785a075ed8817819a6c61f714a"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -24,103 +24,119 @@
         },
         "networkx": {
             "hashes": [
-                "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36",
-                "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"
+                "sha256:9f1bb5cf3409bf324e0a722c20bdb4c20ee39bf1c30ce8ae499c8502b0b5e0c6",
+                "sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
-            "version": "==3.1"
+            "markers": "python_version >= '3.9'",
+            "version": "==3.2.1"
         },
         "numpy": {
             "hashes": [
-                "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2",
-                "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55",
-                "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf",
-                "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01",
-                "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca",
-                "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901",
-                "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d",
-                "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4",
-                "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf",
-                "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380",
-                "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044",
-                "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545",
-                "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f",
-                "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f",
-                "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3",
-                "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364",
-                "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9",
-                "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418",
-                "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f",
-                "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295",
-                "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3",
-                "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187",
-                "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926",
-                "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357",
-                "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"
+                "sha256:02f98011ba4ab17f46f80f7f8f1c291ee7d855fcef0a5a98db80767a468c85cd",
+                "sha256:0b7e807d6888da0db6e7e75838444d62495e2b588b99e90dd80c3459594e857b",
+                "sha256:12c70ac274b32bc00c7f61b515126c9205323703abb99cd41836e8125ea0043e",
+                "sha256:1666f634cb3c80ccbd77ec97bc17337718f56d6658acf5d3b906ca03e90ce87f",
+                "sha256:18c3319a7d39b2c6a9e3bb75aab2304ab79a811ac0168a671a62e6346c29b03f",
+                "sha256:211ddd1e94817ed2d175b60b6374120244a4dd2287f4ece45d49228b4d529178",
+                "sha256:21a9484e75ad018974a2fdaa216524d64ed4212e418e0a551a2d83403b0531d3",
+                "sha256:39763aee6dfdd4878032361b30b2b12593fb445ddb66bbac802e2113eb8a6ac4",
+                "sha256:3c67423b3703f8fbd90f5adaa37f85b5794d3366948efe9a5190a5f3a83fc34e",
+                "sha256:46f47ee566d98849323f01b349d58f2557f02167ee301e5e28809a8c0e27a2d0",
+                "sha256:51c7f1b344f302067b02e0f5b5d2daa9ed4a721cf49f070280ac202738ea7f00",
+                "sha256:5f24750ef94d56ce6e33e4019a8a4d68cfdb1ef661a52cdaee628a56d2437419",
+                "sha256:697df43e2b6310ecc9d95f05d5ef20eacc09c7c4ecc9da3f235d39e71b7da1e4",
+                "sha256:6d45b3ec2faed4baca41c76617fcdcfa4f684ff7a151ce6fc78ad3b6e85af0a6",
+                "sha256:77810ef29e0fb1d289d225cabb9ee6cf4d11978a00bb99f7f8ec2132a84e0166",
+                "sha256:7ca4f24341df071877849eb2034948459ce3a07915c2734f1abb4018d9c49d7b",
+                "sha256:7f784e13e598e9594750b2ef6729bcd5a47f6cfe4a12cca13def35e06d8163e3",
+                "sha256:806dd64230dbbfaca8a27faa64e2f414bf1c6622ab78cc4264f7f5f028fee3bf",
+                "sha256:867e3644e208c8922a3be26fc6bbf112a035f50f0a86497f98f228c50c607bb2",
+                "sha256:8c66d6fec467e8c0f975818c1796d25c53521124b7cfb760114be0abad53a0a2",
+                "sha256:8ed07a90f5450d99dad60d3799f9c03c6566709bd53b497eb9ccad9a55867f36",
+                "sha256:9bc6d1a7f8cedd519c4b7b1156d98e051b726bf160715b769106661d567b3f03",
+                "sha256:9e1591f6ae98bcfac2a4bbf9221c0b92ab49762228f38287f6eeb5f3f55905ce",
+                "sha256:9e87562b91f68dd8b1c39149d0323b42e0082db7ddb8e934ab4c292094d575d6",
+                "sha256:a7081fd19a6d573e1a05e600c82a1c421011db7935ed0d5c483e9dd96b99cf13",
+                "sha256:a8474703bffc65ca15853d5fd4d06b18138ae90c17c8d12169968e998e448bb5",
+                "sha256:af36e0aa45e25c9f57bf684b1175e59ea05d9a7d3e8e87b7ae1a1da246f2767e",
+                "sha256:b1240f767f69d7c4c8a29adde2310b871153df9b26b5cb2b54a561ac85146485",
+                "sha256:b4d362e17bcb0011738c2d83e0a65ea8ce627057b2fdda37678f4374a382a137",
+                "sha256:b831295e5472954104ecb46cd98c08b98b49c69fdb7040483aff799a755a7374",
+                "sha256:b8c275f0ae90069496068c714387b4a0eba5d531aace269559ff2b43655edd58",
+                "sha256:bdd2b45bf079d9ad90377048e2747a0c82351989a2165821f0c96831b4a2a54b",
+                "sha256:cc0743f0302b94f397a4a65a660d4cd24267439eb16493fb3caad2e4389bccbb",
+                "sha256:da4b0c6c699a0ad73c810736303f7fbae483bcb012e38d7eb06a5e3b432c981b",
+                "sha256:f25e2811a9c932e43943a2615e65fc487a0b6b49218899e62e426e7f0a57eeda",
+                "sha256:f73497e8c38295aaa4741bdfa4fda1a5aedda5473074369eca10626835445511"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.9'",
-            "version": "==1.25.2"
+            "version": "==1.26.3"
         },
         "scikit-learn": {
             "hashes": [
-                "sha256:0e8102d5036e28d08ab47166b48c8d5e5810704daecf3a476a4282d562be9a28",
-                "sha256:151ac2bf65ccf363664a689b8beafc9e6aae36263db114b4ca06fbbbf827444a",
-                "sha256:1d54fb9e6038284548072df22fd34777e434153f7ffac72c8596f2d6987110dd",
-                "sha256:3a11936adbc379a6061ea32fa03338d4ca7248d86dd507c81e13af428a5bc1db",
-                "sha256:436aaaae2c916ad16631142488e4c82f4296af2404f480e031d866863425d2a2",
-                "sha256:552fd1b6ee22900cf1780d7386a554bb96949e9a359999177cf30211e6b20df6",
-                "sha256:6a885a9edc9c0a341cab27ec4f8a6c58b35f3d449c9d2503a6fd23e06bbd4f6a",
-                "sha256:7617164951c422747e7c32be4afa15d75ad8044f42e7d70d3e2e0429a50e6718",
-                "sha256:79970a6d759eb00a62266a31e2637d07d2d28446fca8079cf9afa7c07b0427f8",
-                "sha256:850a00b559e636b23901aabbe79b73dc604b4e4248ba9e2d6e72f95063765603",
-                "sha256:8be549886f5eda46436b6e555b0e4873b4f10aa21c07df45c4bc1735afbccd7a",
-                "sha256:981287869e576d42c682cf7ca96af0c6ac544ed9316328fd0d9292795c742cf5",
-                "sha256:9877af9c6d1b15486e18a94101b742e9d0d2f343d35a634e337411ddb57783f3",
-                "sha256:998d38fcec96584deee1e79cd127469b3ad6fefd1ea6c2dfc54e8db367eb396b",
-                "sha256:9d953531f5d9f00c90c34fa3b7d7cfb43ecff4c605dac9e4255a20b114a27369",
-                "sha256:ae80c08834a473d08a204d966982a62e11c976228d306a2648c575e3ead12111",
-                "sha256:c470f53cea065ff3d588050955c492793bb50c19a92923490d18fcb637f6383a",
-                "sha256:c7e28d8fa47a0b30ae1bd7a079519dd852764e31708a7804da6cb6f8b36e3630",
-                "sha256:ded35e810438a527e17623ac6deae3b360134345b7c598175ab7741720d7ffa7",
-                "sha256:ee04835fb016e8062ee9fe9074aef9b82e430504e420bff51e3e5fffe72750ca",
-                "sha256:fd6e2d7389542eae01077a1ee0318c4fec20c66c957f45c7aac0c6eb0fe3c612"
+                "sha256:0402638c9a7c219ee52c94cbebc8fcb5eb9fe9c773717965c1f4185588ad3107",
+                "sha256:0ee107923a623b9f517754ea2f69ea3b62fc898a3641766cb7deb2f2ce450161",
+                "sha256:1215e5e58e9880b554b01187b8c9390bf4dc4692eedeaf542d3273f4785e342c",
+                "sha256:15e1e94cc23d04d39da797ee34236ce2375ddea158b10bee3c343647d615581d",
+                "sha256:18424efee518a1cde7b0b53a422cde2f6625197de6af36da0b57ec502f126157",
+                "sha256:1d08ada33e955c54355d909b9c06a4789a729977f165b8bae6f225ff0a60ec4a",
+                "sha256:3271552a5eb16f208a6f7f617b8cc6d1f137b52c8a1ef8edf547db0259b2c9fb",
+                "sha256:35a22e8015048c628ad099da9df5ab3004cdbf81edc75b396fd0cff8699ac58c",
+                "sha256:535805c2a01ccb40ca4ab7d081d771aea67e535153e35a1fd99418fcedd1648a",
+                "sha256:5b2de18d86f630d68fe1f87af690d451388bb186480afc719e5f770590c2ef6c",
+                "sha256:61a6efd384258789aa89415a410dcdb39a50e19d3d8410bd29be365bcdd512d5",
+                "sha256:64381066f8aa63c2710e6b56edc9f0894cc7bf59bd71b8ce5613a4559b6145e0",
+                "sha256:67f37d708f042a9b8d59551cf94d30431e01374e00dc2645fa186059c6c5d78b",
+                "sha256:6c43290337f7a4b969d207e620658372ba3c1ffb611f8bc2b6f031dc5c6d1d03",
+                "sha256:6fb6bc98f234fda43163ddbe36df8bcde1d13ee176c6dc9b92bb7d3fc842eb66",
+                "sha256:763f0ae4b79b0ff9cca0bf3716bcc9915bdacff3cebea15ec79652d1cc4fa5c9",
+                "sha256:785a2213086b7b1abf037aeadbbd6d67159feb3e30263434139c98425e3dcfcf",
+                "sha256:8db94cd8a2e038b37a80a04df8783e09caac77cbe052146432e67800e430c028",
+                "sha256:a19f90f95ba93c1a7f7924906d0576a84da7f3b2282ac3bfb7a08a32801add93",
+                "sha256:a2f54c76accc15a34bfb9066e6c7a56c1e7235dda5762b990792330b52ccfb05",
+                "sha256:b8692e395a03a60cd927125eef3a8e3424d86dde9b2370d544f0ea35f78a8073",
+                "sha256:cb06f8dce3f5ddc5dee1715a9b9f19f20d295bed8e3cd4fa51e1d050347de525",
+                "sha256:dc9002fc200bed597d5d34e90c752b74df516d592db162f756cc52836b38fe0e",
+                "sha256:e326c0eb5cf4d6ba40f93776a20e9a7a69524c4db0757e7ce24ba222471ee8a1",
+                "sha256:ed932ea780517b00dae7431e031faae6b49b20eb6950918eb83bd043237950e0",
+                "sha256:fc4144a5004a676d5022b798d9e573b05139e77f271253a4703eed295bde0433"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==1.3.0"
+            "version": "==1.3.2"
         },
         "scipy": {
             "hashes": [
-                "sha256:0f3261f14b767b316d7137c66cc4f33a80ea05841b9c87ad83a726205b901423",
-                "sha256:10eb6af2f751aa3424762948e5352f707b0dece77288206f227864ddf675aca0",
-                "sha256:1342ca385c673208f32472830c10110a9dcd053cf0c4b7d4cd7026d0335a6c1d",
-                "sha256:214cdf04bbae7a54784f8431f976704ed607c4bc69ba0d5d5d6a9df84374df76",
-                "sha256:2b997a5369e2d30c97995dcb29d638701f8000d04df01b8e947f206e5d0ac788",
-                "sha256:2c91cf049ffb5575917f2a01da1da082fd24ed48120d08a6e7297dfcac771dcd",
-                "sha256:3aeb87661de987f8ec56fa6950863994cd427209158255a389fc5aea51fa7055",
-                "sha256:4447ad057d7597476f9862ecbd9285bbf13ba9d73ce25acfa4e4b11c6801b4c9",
-                "sha256:542a757e2a6ec409e71df3d8fd20127afbbacb1c07990cb23c5870c13953d899",
-                "sha256:8d9886f44ef8c9e776cb7527fb01455bf4f4a46c455c4682edc2c2cc8cd78562",
-                "sha256:90d3b1364e751d8214e325c371f0ee0dd38419268bf4888b2ae1040a6b266b2a",
-                "sha256:95763fbda1206bec41157582bea482f50eb3702c85fffcf6d24394b071c0e87a",
-                "sha256:ac74b1512d38718fb6a491c439aa7b3605b96b1ed3be6599c17d49d6c60fca18",
-                "sha256:afdb0d983f6135d50770dd979df50bf1c7f58b5b33e0eb8cf5c73c70600eae1d",
-                "sha256:b0620240ef445b5ddde52460e6bc3483b7c9c750275369379e5f609a1050911c",
-                "sha256:b133f237bd8ba73bad51bc12eb4f2d84cbec999753bf25ba58235e9fc2096d80",
-                "sha256:b29318a5e39bd200ca4381d80b065cdf3076c7d7281c5e36569e99273867f61d",
-                "sha256:b8425fa963a32936c9773ee3ce44a765d8ff67eed5f4ac81dc1e4a819a238ee9",
-                "sha256:d2b813bfbe8dec6a75164523de650bad41f4405d35b0fa24c2c28ae07fcefb20",
-                "sha256:d690e1ca993c8f7ede6d22e5637541217fc6a4d3f78b3672a6fe454dbb7eb9a7",
-                "sha256:e367904a0fec76433bf3fbf3e85bf60dae8e9e585ffd21898ab1085a29a04d16",
-                "sha256:ea932570b1c2a30edafca922345854ff2cd20d43cd9123b6dacfdecebfc1a80b",
-                "sha256:f28f1f6cfeb48339c192efc6275749b2a25a7e49c4d8369a28b6591da02fbc9a",
-                "sha256:f73102f769ee06041a3aa26b5841359b1a93cc364ce45609657751795e8f4a4a",
-                "sha256:fa4909c6c20c3d91480533cddbc0e7c6d849e7d9ded692918c76ce5964997898"
-            ],
-            "markers": "python_version < '3.13' and python_version >= '3.9'",
-            "version": "==1.11.2"
+                "sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c",
+                "sha256:028eccd22e654b3ea01ee63705681ee79933652b2d8f873e7949898dda6d11b6",
+                "sha256:1b7c3dca977f30a739e0409fb001056484661cb2541a01aba0bb0029f7b68db8",
+                "sha256:2c6ff6ef9cc27f9b3db93a6f8b38f97387e6e0591600369a297a50a8e96e835d",
+                "sha256:36750b7733d960d7994888f0d148d31ea3017ac15eef664194b4ef68d36a4a97",
+                "sha256:530f9ad26440e85766509dbf78edcfe13ffd0ab7fec2560ee5c36ff74d6269ff",
+                "sha256:5e347b14fe01003d3b78e196e84bd3f48ffe4c8a7b8a1afbcb8f5505cb710993",
+                "sha256:6550466fbeec7453d7465e74d4f4b19f905642c89a7525571ee91dd7adabb5a3",
+                "sha256:6df1468153a31cf55ed5ed39647279beb9cfb5d3f84369453b49e4b8502394fd",
+                "sha256:6e619aba2df228a9b34718efb023966da781e89dd3d21637b27f2e54db0410d7",
+                "sha256:8fce70f39076a5aa62e92e69a7f62349f9574d8405c0a5de6ed3ef72de07f446",
+                "sha256:90a2b78e7f5733b9de748f589f09225013685f9b218275257f8a8168ededaeaa",
+                "sha256:91af76a68eeae0064887a48e25c4e616fa519fa0d38602eda7e0f97d65d57937",
+                "sha256:933baf588daa8dc9a92c20a0be32f56d43faf3d1a60ab11b3f08c356430f6e56",
+                "sha256:acf8ed278cc03f5aff035e69cb511741e0418681d25fbbb86ca65429c4f4d9cd",
+                "sha256:ad669df80528aeca5f557712102538f4f37e503f0c5b9541655016dd0932ca79",
+                "sha256:b030c6674b9230d37c5c60ab456e2cf12f6784596d15ce8da9365e70896effc4",
+                "sha256:b9999c008ccf00e8fbcce1236f85ade5c569d13144f77a1946bef8863e8f6eb4",
+                "sha256:bc9a714581f561af0848e6b69947fda0614915f072dfd14142ed1bfe1b806710",
+                "sha256:ce7fff2e23ab2cc81ff452a9444c215c28e6305f396b2ba88343a567feec9660",
+                "sha256:cf00bd2b1b0211888d4dc75656c0412213a8b25e80d73898083f402b50f47e41",
+                "sha256:d10e45a6c50211fe256da61a11c34927c68f277e03138777bdebedd933712fea",
+                "sha256:ee410e6de8f88fd5cf6eadd73c135020bfbbbdfcd0f6162c36a7638a1ea8cc65",
+                "sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be",
+                "sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==1.11.4"
         },
         "threadpoolctl": {
             "hashes": [
@@ -134,177 +150,258 @@
     "develop": {
         "alabaster": {
             "hashes": [
-                "sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3",
-                "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2"
+                "sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65",
+                "sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==0.7.13"
+            "markers": "python_version >= '3.9'",
+            "version": "==0.7.16"
         },
-        "babel": {
+        "atomicwrites": {
             "hashes": [
-                "sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610",
-                "sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455"
+                "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"
+            ],
+            "markers": "sys_platform == 'win32'",
+            "version": "==1.4.1"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30",
+                "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.12.1"
+            "version": "==23.2.0"
         },
-        "bleach": {
+        "babel": {
             "hashes": [
-                "sha256:1a1a85c1595e07d8db14c5f09f09e6433502c51c595970edc090551f0db99414",
-                "sha256:33c16e3353dbd13028ab4799a0f89a83f113405c766e9c122df8a06f5b85b3f4"
+                "sha256:6919867db036398ba21eb5c7a0f6b28ab8cbc3ae7a73a44ebe34ae74a4e7d363",
+                "sha256:efb1a25b7118e67ce3a259bed20545c29cb68be8ad2c784c83689981b7a57287"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==6.0.0"
+            "version": "==2.14.0"
+        },
+        "black": {
+            "hashes": [
+                "sha256:14ff67aec0a47c424bc99b71005202045dc09270da44a27848d534600ac64fc7",
+                "sha256:197df8509263b0b8614e1df1756b1dd41be6738eed2ba9e9769f3880c2b9d7b6",
+                "sha256:1e464456d24e23d11fced2bc8c47ef66d471f845c7b7a42f3bd77bf3d1789650",
+                "sha256:2039230db3c6c639bd84efe3292ec7b06e9214a2992cd9beb293d639c6402edb",
+                "sha256:21199526696b8f09c3997e2b4db8d0b108d801a348414264d2eb8eb2532e540d",
+                "sha256:2644b5d63633702bc2c5f3754b1b475378fbbfb481f62319388235d0cd104c2d",
+                "sha256:432247333090c8c5366e69627ccb363bc58514ae3e63f7fc75c54b1ea80fa7de",
+                "sha256:444ebfb4e441254e87bad00c661fe32df9969b2bf224373a448d8aca2132b395",
+                "sha256:5b9b29da4f564ba8787c119f37d174f2b69cdfdf9015b7d8c5c16121ddc054ae",
+                "sha256:5cc42ca67989e9c3cf859e84c2bf014f6633db63d1cbdf8fdb666dcd9e77e3fa",
+                "sha256:5d8f74030e67087b219b032aa33a919fae8806d49c867846bfacde57f43972ef",
+                "sha256:72ef3925f30e12a184889aac03d77d031056860ccae8a1e519f6cbb742736383",
+                "sha256:819dc789f4498ecc91438a7de64427c73b45035e2e3680c92e18795a839ebb66",
+                "sha256:915ace4ff03fdfff953962fa672d44be269deb2eaf88499a0f8805221bc68c87",
+                "sha256:9311e99228ae10023300ecac05be5a296f60d2fd10fff31cf5c1fa4ca4b1988d",
+                "sha256:974308c58d057a651d182208a484ce80a26dac0caef2895836a92dd6ebd725e0",
+                "sha256:b8b49776299fece66bffaafe357d929ca9451450f5466e997a7285ab0fe28e3b",
+                "sha256:c957b2b4ea88587b46cf49d1dc17681c1e672864fd7af32fc1e9664d572b3458",
+                "sha256:e41a86c6c650bcecc6633ee3180d80a025db041a8e2398dcc059b3afa8382cd4",
+                "sha256:f513588da599943e0cde4e32cc9879e825d58720d6557062d1098c5ad80080e1",
+                "sha256:fba8a281e570adafb79f7755ac8721b6cf1bbf691186a287e990c7929c7692ff"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.7'",
+            "version": "==22.10.0"
         },
         "certifi": {
             "hashes": [
-                "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082",
-                "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
+                "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1",
+                "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2023.7.22"
+            "version": "==2023.11.17"
+        },
+        "cfgv": {
+            "hashes": [
+                "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9",
+                "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==3.4.0"
         },
         "charset-normalizer": {
             "hashes": [
-                "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96",
-                "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c",
-                "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710",
-                "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706",
-                "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020",
-                "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252",
-                "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad",
-                "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329",
-                "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a",
-                "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f",
-                "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6",
-                "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4",
-                "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a",
-                "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46",
-                "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2",
-                "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23",
-                "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace",
-                "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd",
-                "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982",
-                "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10",
-                "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2",
-                "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea",
-                "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09",
-                "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5",
-                "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149",
-                "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489",
-                "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9",
-                "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80",
-                "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592",
-                "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3",
-                "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6",
-                "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed",
-                "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c",
-                "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200",
-                "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a",
-                "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e",
-                "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d",
-                "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6",
-                "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623",
-                "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669",
-                "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3",
-                "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa",
-                "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9",
-                "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2",
-                "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f",
-                "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1",
-                "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4",
-                "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a",
-                "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8",
-                "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3",
-                "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029",
-                "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f",
-                "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959",
-                "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22",
-                "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7",
-                "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952",
-                "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346",
-                "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e",
-                "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d",
-                "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299",
-                "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd",
-                "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a",
-                "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3",
-                "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037",
-                "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94",
-                "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c",
-                "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858",
-                "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a",
-                "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449",
-                "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c",
-                "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918",
-                "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1",
-                "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c",
-                "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac",
-                "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"
+                "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027",
+                "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087",
+                "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786",
+                "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8",
+                "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09",
+                "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185",
+                "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574",
+                "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e",
+                "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519",
+                "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898",
+                "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269",
+                "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3",
+                "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f",
+                "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6",
+                "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8",
+                "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a",
+                "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73",
+                "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc",
+                "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714",
+                "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2",
+                "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc",
+                "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce",
+                "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d",
+                "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e",
+                "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6",
+                "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269",
+                "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96",
+                "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d",
+                "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a",
+                "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4",
+                "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77",
+                "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d",
+                "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0",
+                "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed",
+                "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068",
+                "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac",
+                "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25",
+                "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8",
+                "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab",
+                "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26",
+                "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2",
+                "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db",
+                "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f",
+                "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5",
+                "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99",
+                "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c",
+                "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d",
+                "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811",
+                "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa",
+                "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a",
+                "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03",
+                "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b",
+                "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04",
+                "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c",
+                "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001",
+                "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458",
+                "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389",
+                "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99",
+                "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985",
+                "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537",
+                "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238",
+                "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f",
+                "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d",
+                "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796",
+                "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a",
+                "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143",
+                "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8",
+                "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c",
+                "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5",
+                "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5",
+                "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711",
+                "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4",
+                "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6",
+                "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c",
+                "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7",
+                "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4",
+                "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b",
+                "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae",
+                "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12",
+                "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c",
+                "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae",
+                "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8",
+                "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887",
+                "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b",
+                "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4",
+                "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f",
+                "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5",
+                "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33",
+                "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519",
+                "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"
             ],
             "markers": "python_full_version >= '3.7.0'",
-            "version": "==3.2.0"
+            "version": "==3.3.2"
+        },
+        "click": {
+            "hashes": [
+                "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28",
+                "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==8.1.7"
+        },
+        "colorama": {
+            "hashes": [
+                "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44",
+                "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"
+            ],
+            "markers": "sys_platform == 'win32'",
+            "version": "==0.4.6"
         },
         "coverage": {
             "extras": [
                 "toml"
             ],
             "hashes": [
-                "sha256:07ea61bcb179f8f05ffd804d2732b09d23a1238642bf7e51dad62082b5019b34",
-                "sha256:1084393c6bda8875c05e04fce5cfe1301a425f758eb012f010eab586f1f3905e",
-                "sha256:13c6cbbd5f31211d8fdb477f0f7b03438591bdd077054076eec362cf2207b4a7",
-                "sha256:211a4576e984f96d9fce61766ffaed0115d5dab1419e4f63d6992b480c2bd60b",
-                "sha256:2d22172f938455c156e9af2612650f26cceea47dc86ca048fa4e0b2d21646ad3",
-                "sha256:34f9f0763d5fa3035a315b69b428fe9c34d4fc2f615262d6be3d3bf3882fb985",
-                "sha256:3558e5b574d62f9c46b76120a5c7c16c4612dc2644c3d48a9f4064a705eaee95",
-                "sha256:36ce5d43a072a036f287029a55b5c6a0e9bd73db58961a273b6dc11a2c6eb9c2",
-                "sha256:37d5576d35fcb765fca05654f66aa71e2808d4237d026e64ac8b397ffa66a56a",
-                "sha256:3c9834d5e3df9d2aba0275c9f67989c590e05732439b3318fa37a725dff51e74",
-                "sha256:438856d3f8f1e27f8e79b5410ae56650732a0dcfa94e756df88c7e2d24851fcd",
-                "sha256:477c9430ad5d1b80b07f3c12f7120eef40bfbf849e9e7859e53b9c93b922d2af",
-                "sha256:49ab200acf891e3dde19e5aa4b0f35d12d8b4bd805dc0be8792270c71bd56c54",
-                "sha256:49dbb19cdcafc130f597d9e04a29d0a032ceedf729e41b181f51cd170e6ee865",
-                "sha256:4c8e31cf29b60859876474034a83f59a14381af50cbe8a9dbaadbf70adc4b214",
-                "sha256:4eddd3153d02204f22aef0825409091a91bf2a20bce06fe0f638f5c19a85de54",
-                "sha256:5247bab12f84a1d608213b96b8af0cbb30d090d705b6663ad794c2f2a5e5b9fe",
-                "sha256:5492a6ce3bdb15c6ad66cb68a0244854d9917478877a25671d70378bdc8562d0",
-                "sha256:56afbf41fa4a7b27f6635bc4289050ac3ab7951b8a821bca46f5b024500e6321",
-                "sha256:59777652e245bb1e300e620ce2bef0d341945842e4eb888c23a7f1d9e143c446",
-                "sha256:60f64e2007c9144375dd0f480a54d6070f00bb1a28f65c408370544091c9bc9e",
-                "sha256:63c5b8ecbc3b3d5eb3a9d873dec60afc0cd5ff9d9f1c75981d8c31cfe4df8527",
-                "sha256:68d8a0426b49c053013e631c0cdc09b952d857efa8f68121746b339912d27a12",
-                "sha256:74c160285f2dfe0acf0f72d425f3e970b21b6de04157fc65adc9fd07ee44177f",
-                "sha256:7a9baf8e230f9621f8e1d00c580394a0aa328fdac0df2b3f8384387c44083c0f",
-                "sha256:7df91fb24c2edaabec4e0eee512ff3bc6ec20eb8dccac2e77001c1fe516c0c84",
-                "sha256:7f297e0c1ae55300ff688568b04ff26b01c13dfbf4c9d2b7d0cb688ac60df479",
-                "sha256:80501d1b2270d7e8daf1b64b895745c3e234289e00d5f0e30923e706f110334e",
-                "sha256:85b7335c22455ec12444cec0d600533a238d6439d8d709d545158c1208483873",
-                "sha256:887665f00ea4e488501ba755a0e3c2cfd6278e846ada3185f42d391ef95e7e70",
-                "sha256:8f39c49faf5344af36042b293ce05c0d9004270d811c7080610b3e713251c9b0",
-                "sha256:90b6e2f0f66750c5a1178ffa9370dec6c508a8ca5265c42fbad3ccac210a7977",
-                "sha256:96d7d761aea65b291a98c84e1250cd57b5b51726821a6f2f8df65db89363be51",
-                "sha256:97af9554a799bd7c58c0179cc8dbf14aa7ab50e1fd5fa73f90b9b7215874ba28",
-                "sha256:97c44f4ee13bce914272589b6b41165bbb650e48fdb7bd5493a38bde8de730a1",
-                "sha256:a67e6bbe756ed458646e1ef2b0778591ed4d1fcd4b146fc3ba2feb1a7afd4254",
-                "sha256:ac0dec90e7de0087d3d95fa0533e1d2d722dcc008bc7b60e1143402a04c117c1",
-                "sha256:ad0f87826c4ebd3ef484502e79b39614e9c03a5d1510cfb623f4a4a051edc6fd",
-                "sha256:b3eb0c93e2ea6445b2173da48cb548364f8f65bf68f3d090404080d338e3a689",
-                "sha256:b543302a3707245d454fc49b8ecd2c2d5982b50eb63f3535244fd79a4be0c99d",
-                "sha256:b859128a093f135b556b4765658d5d2e758e1fae3e7cc2f8c10f26fe7005e543",
-                "sha256:bac329371d4c0d456e8d5f38a9b0816b446581b5f278474e416ea0c68c47dcd9",
-                "sha256:c02cfa6c36144ab334d556989406837336c1d05215a9bdf44c0bc1d1ac1cb637",
-                "sha256:c9737bc49a9255d78da085fa04f628a310c2332b187cd49b958b0e494c125071",
-                "sha256:ccc51713b5581e12f93ccb9c5e39e8b5d4b16776d584c0f5e9e4e63381356482",
-                "sha256:ce2ee86ca75f9f96072295c5ebb4ef2a43cecf2870b0ca5e7a1cbdd929cf67e1",
-                "sha256:d000a739f9feed900381605a12a61f7aaced6beae832719ae0d15058a1e81c1b",
-                "sha256:db76a1bcb51f02b2007adacbed4c88b6dee75342c37b05d1822815eed19edee5",
-                "sha256:e2ac9a1de294773b9fa77447ab7e529cf4fe3910f6a0832816e5f3d538cfea9a",
-                "sha256:e61260ec93f99f2c2d93d264b564ba912bec502f679793c56f678ba5251f0393",
-                "sha256:fac440c43e9b479d1241fe9d768645e7ccec3fb65dc3a5f6e90675e75c3f3e3a",
-                "sha256:fc0ed8d310afe013db1eedd37176d0839dc66c96bcfcce8f6607a73ffea2d6ba"
+                "sha256:04387a4a6ecb330c1878907ce0dc04078ea72a869263e53c72a1ba5bbdf380ca",
+                "sha256:0676cd0ba581e514b7f726495ea75aba3eb20899d824636c6f59b0ed2f88c471",
+                "sha256:0e8d06778e8fbffccfe96331a3946237f87b1e1d359d7fbe8b06b96c95a5407a",
+                "sha256:0eb3c2f32dabe3a4aaf6441dde94f35687224dfd7eb2a7f47f3fd9428e421058",
+                "sha256:109f5985182b6b81fe33323ab4707011875198c41964f014579cf82cebf2bb85",
+                "sha256:13eaf476ec3e883fe3e5fe3707caeb88268a06284484a3daf8250259ef1ba143",
+                "sha256:164fdcc3246c69a6526a59b744b62e303039a81e42cfbbdc171c91a8cc2f9446",
+                "sha256:26776ff6c711d9d835557ee453082025d871e30b3fd6c27fcef14733f67f0590",
+                "sha256:26f66da8695719ccf90e794ed567a1549bb2644a706b41e9f6eae6816b398c4a",
+                "sha256:29f3abe810930311c0b5d1a7140f6395369c3db1be68345638c33eec07535105",
+                "sha256:316543f71025a6565677d84bc4df2114e9b6a615aa39fb165d697dba06a54af9",
+                "sha256:36b0ea8ab20d6a7564e89cb6135920bc9188fb5f1f7152e94e8300b7b189441a",
+                "sha256:3cc9d4bc55de8003663ec94c2f215d12d42ceea128da8f0f4036235a119c88ac",
+                "sha256:485e9f897cf4856a65a57c7f6ea3dc0d4e6c076c87311d4bc003f82cfe199d25",
+                "sha256:5040148f4ec43644702e7b16ca864c5314ccb8ee0751ef617d49aa0e2d6bf4f2",
+                "sha256:51456e6fa099a8d9d91497202d9563a320513fcf59f33991b0661a4a6f2ad450",
+                "sha256:53d7d9158ee03956e0eadac38dfa1ec8068431ef8058fe6447043db1fb40d932",
+                "sha256:5a10a4920def78bbfff4eff8a05c51be03e42f1c3735be42d851f199144897ba",
+                "sha256:5b14b4f8760006bfdb6e08667af7bc2d8d9bfdb648351915315ea17645347137",
+                "sha256:5b2ccb7548a0b65974860a78c9ffe1173cfb5877460e5a229238d985565574ae",
+                "sha256:697d1317e5290a313ef0d369650cfee1a114abb6021fa239ca12b4849ebbd614",
+                "sha256:6ae8c9d301207e6856865867d762a4b6fd379c714fcc0607a84b92ee63feff70",
+                "sha256:707c0f58cb1712b8809ece32b68996ee1e609f71bd14615bd8f87a1293cb610e",
+                "sha256:74775198b702868ec2d058cb92720a3c5a9177296f75bd97317c787daf711505",
+                "sha256:756ded44f47f330666843b5781be126ab57bb57c22adbb07d83f6b519783b870",
+                "sha256:76f03940f9973bfaee8cfba70ac991825611b9aac047e5c80d499a44079ec0bc",
+                "sha256:79287fd95585ed36e83182794a57a46aeae0b64ca53929d1176db56aacc83451",
+                "sha256:799c8f873794a08cdf216aa5d0531c6a3747793b70c53f70e98259720a6fe2d7",
+                "sha256:7d360587e64d006402b7116623cebf9d48893329ef035278969fa3bbf75b697e",
+                "sha256:80b5ee39b7f0131ebec7968baa9b2309eddb35b8403d1869e08f024efd883566",
+                "sha256:815ac2d0f3398a14286dc2cea223a6f338109f9ecf39a71160cd1628786bc6f5",
+                "sha256:83c2dda2666fe32332f8e87481eed056c8b4d163fe18ecc690b02802d36a4d26",
+                "sha256:846f52f46e212affb5bcf131c952fb4075b55aae6b61adc9856222df89cbe3e2",
+                "sha256:936d38794044b26c99d3dd004d8af0035ac535b92090f7f2bb5aa9c8e2f5cd42",
+                "sha256:9864463c1c2f9cb3b5db2cf1ff475eed2f0b4285c2aaf4d357b69959941aa555",
+                "sha256:995ea5c48c4ebfd898eacb098164b3cc826ba273b3049e4a889658548e321b43",
+                "sha256:a1526d265743fb49363974b7aa8d5899ff64ee07df47dd8d3e37dcc0818f09ed",
+                "sha256:a56de34db7b7ff77056a37aedded01b2b98b508227d2d0979d373a9b5d353daa",
+                "sha256:a7c97726520f784239f6c62506bc70e48d01ae71e9da128259d61ca5e9788516",
+                "sha256:b8e99f06160602bc64da35158bb76c73522a4010f0649be44a4e167ff8555952",
+                "sha256:bb1de682da0b824411e00a0d4da5a784ec6496b6850fdf8c865c1d68c0e318dd",
+                "sha256:bf477c355274a72435ceb140dc42de0dc1e1e0bf6e97195be30487d8eaaf1a09",
+                "sha256:bf635a52fc1ea401baf88843ae8708591aa4adff875e5c23220de43b1ccf575c",
+                "sha256:bfd5db349d15c08311702611f3dccbef4b4e2ec148fcc636cf8739519b4a5c0f",
+                "sha256:c530833afc4707fe48524a44844493f36d8727f04dcce91fb978c414a8556cc6",
+                "sha256:cc6d65b21c219ec2072c1293c505cf36e4e913a3f936d80028993dd73c7906b1",
+                "sha256:cd3c1e4cb2ff0083758f09be0f77402e1bdf704adb7f89108007300a6da587d0",
+                "sha256:cfd2a8b6b0d8e66e944d47cdec2f47c48fef2ba2f2dff5a9a75757f64172857e",
+                "sha256:d0ca5c71a5a1765a0f8f88022c52b6b8be740e512980362f7fdbb03725a0d6b9",
+                "sha256:e7defbb9737274023e2d7af02cac77043c86ce88a907c58f42b580a97d5bcca9",
+                "sha256:e9d1bf53c4c8de58d22e0e956a79a5b37f754ed1ffdbf1a260d9dcfa2d8a325e",
+                "sha256:ea81d8f9691bb53f4fb4db603203029643caffc82bf998ab5b59ca05560f4c06"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==7.3.0"
+            "version": "==7.4.0"
+        },
+        "distlib": {
+            "hashes": [
+                "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784",
+                "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"
+            ],
+            "version": "==0.3.8"
         },
         "docutils": {
             "hashes": [
@@ -314,21 +411,38 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==0.16"
         },
+        "filelock": {
+            "hashes": [
+                "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e",
+                "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==3.13.1"
+        },
         "flake8": {
             "hashes": [
-                "sha256:d5b3857f07c030bdb5bf41c7f53799571d75c4491748a3adcd47de929e34cd23",
-                "sha256:ffdfce58ea94c6580c77888a86506937f9a1a227dfcd15f245d694ae20a6b6e5"
+                "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d",
+                "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.6'",
+            "version": "==4.0.1"
+        },
+        "identify": {
+            "hashes": [
+                "sha256:161558f9fe4559e1557e1bff323e8631f6a0e4837f7497767c1782832f16b62d",
+                "sha256:d40ce5fcd762817627670da8a7d8d8e65f24342d14539c59488dc603bf662e34"
             ],
-            "markers": "python_full_version >= '3.8.1'",
-            "version": "==6.1.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.5.33"
         },
         "idna": {
             "hashes": [
-                "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
-                "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
+                "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca",
+                "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==3.4"
+            "version": "==3.6"
         },
         "imagesize": {
             "hashes": [
@@ -340,11 +454,11 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb",
-                "sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743"
+                "sha256:4805911c3a4ec7c3966410053e9ec6a1fecd629117df5adee56dfc9432a1081e",
+                "sha256:f238736bb06590ae52ac1fab06a3a9ef1d8dce2b7a35b5ab329371d6c8f5d2cc"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.8.0"
+            "version": "==7.0.1"
         },
         "iniconfig": {
             "hashes": [
@@ -364,19 +478,19 @@
         },
         "jinja2": {
             "hashes": [
-                "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852",
-                "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"
+                "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa",
+                "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.1.2"
+            "version": "==3.1.3"
         },
         "keyring": {
             "hashes": [
-                "sha256:4901caaf597bfd3bbd78c9a0c7c4c29fcd8310dab2cffefe749e916b6527acd6",
-                "sha256:ca0746a19ec421219f4d713f848fa297a661a8a8c1504867e55bfb5e09091509"
+                "sha256:4446d35d636e6a10b8bce7caa66913dd9eca5fd222ca03a3d42c38608ac30836",
+                "sha256:e730ecffd309658a08ee82535a3b5ec4b4c8669a9be11efb66249d8e0aeb9a25"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==24.2.0"
+            "version": "==24.3.0"
         },
         "markdown-it-py": {
             "hashes": [
@@ -392,8 +506,11 @@
                 "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e",
                 "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431",
                 "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686",
+                "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c",
                 "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559",
                 "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc",
+                "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb",
+                "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939",
                 "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c",
                 "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0",
                 "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4",
@@ -401,6 +518,7 @@
                 "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575",
                 "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba",
                 "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d",
+                "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd",
                 "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3",
                 "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00",
                 "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155",
@@ -409,6 +527,7 @@
                 "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f",
                 "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8",
                 "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b",
+                "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007",
                 "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24",
                 "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea",
                 "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198",
@@ -416,9 +535,12 @@
                 "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee",
                 "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be",
                 "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2",
+                "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1",
                 "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707",
                 "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6",
+                "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c",
                 "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58",
+                "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823",
                 "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779",
                 "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636",
                 "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c",
@@ -437,18 +559,19 @@
                 "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9",
                 "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57",
                 "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc",
-                "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"
+                "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc",
+                "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2",
+                "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"
             ],
             "markers": "python_version >= '3.7'",
             "version": "==2.1.3"
         },
         "mccabe": {
             "hashes": [
-                "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325",
-                "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"
+                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
+                "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==0.7.0"
+            "version": "==0.6.1"
         },
         "mdurl": {
             "hashes": [
@@ -460,19 +583,64 @@
         },
         "more-itertools": {
             "hashes": [
-                "sha256:626c369fa0eb37bac0291bce8259b332fd59ac792fa5497b59837309cd5b114a",
-                "sha256:64e0735fcfdc6f3464ea133afe8ea4483b1c5fe3a3d69852e6503b43a0b222e6"
+                "sha256:686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684",
+                "sha256:8fccb480c43d3e99a00087634c06dd02b0d50fbf088b380de5a41a015ec239e1"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==10.1.0"
+            "version": "==10.2.0"
+        },
+        "mypy-extensions": {
+            "hashes": [
+                "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d",
+                "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==1.0.0"
+        },
+        "nh3": {
+            "hashes": [
+                "sha256:0d02d0ff79dfd8208ed25a39c12cbda092388fff7f1662466e27d97ad011b770",
+                "sha256:3277481293b868b2715907310c7be0f1b9d10491d5adf9fce11756a97e97eddf",
+                "sha256:3b803a5875e7234907f7d64777dfde2b93db992376f3d6d7af7f3bc347deb305",
+                "sha256:427fecbb1031db085eaac9931362adf4a796428ef0163070c484b5a768e71601",
+                "sha256:5f0d77272ce6d34db6c87b4f894f037d55183d9518f948bba236fe81e2bb4e28",
+                "sha256:60684857cfa8fdbb74daa867e5cad3f0c9789415aba660614fe16cd66cbb9ec7",
+                "sha256:6f42f99f0cf6312e470b6c09e04da31f9abaadcd3eb591d7d1a88ea931dca7f3",
+                "sha256:86e447a63ca0b16318deb62498db4f76fc60699ce0a1231262880b38b6cff911",
+                "sha256:8d595df02413aa38586c24811237e95937ef18304e108b7e92c890a06793e3bf",
+                "sha256:9c0d415f6b7f2338f93035bba5c0d8c1b464e538bfbb1d598acd47d7969284f0",
+                "sha256:a5167a6403d19c515217b6bcaaa9be420974a6ac30e0da9e84d4fc67a5d474c5",
+                "sha256:ac19c0d68cd42ecd7ead91a3a032fdfff23d29302dbb1311e641a130dfefba97",
+                "sha256:b1e97221cedaf15a54f5243f2c5894bb12ca951ae4ddfd02a9d4ea9df9e1a29d",
+                "sha256:bc2d086fb540d0fa52ce35afaded4ea526b8fc4d3339f783db55c95de40ef02e",
+                "sha256:d1e30ff2d8d58fb2a14961f7aac1bbb1c51f9bdd7da727be35c63826060b0bf3",
+                "sha256:f3b53ba93bb7725acab1e030bc2ecd012a817040fd7851b332f86e2f9bb98dc6"
+            ],
+            "version": "==0.2.15"
+        },
+        "nodeenv": {
+            "hashes": [
+                "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2",
+                "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'",
+            "version": "==1.8.0"
         },
         "packaging": {
             "hashes": [
-                "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61",
-                "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"
+                "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5",
+                "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==23.1"
+            "version": "==23.2"
+        },
+        "pathspec": {
+            "hashes": [
+                "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08",
+                "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.12.1"
         },
         "pkginfo": {
             "hashes": [
@@ -482,6 +650,14 @@
             "markers": "python_version >= '3.6'",
             "version": "==1.9.6"
         },
+        "platformdirs": {
+            "hashes": [
+                "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380",
+                "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==4.1.0"
+        },
         "pluggy": {
             "hashes": [
                 "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12",
@@ -490,55 +666,73 @@
             "markers": "python_version >= '3.8'",
             "version": "==1.3.0"
         },
+        "pre-commit": {
+            "hashes": [
+                "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7",
+                "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.7'",
+            "version": "==2.20.0"
+        },
+        "py": {
+            "hashes": [
+                "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719",
+                "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==1.11.0"
+        },
         "pycodestyle": {
             "hashes": [
-                "sha256:259bcc17857d8a8b3b4a2327324b79e5f020a13c16074670f9c8c8f872ea76d0",
-                "sha256:5d1013ba8dc7895b548be5afb05740ca82454fd899971563d2ef625d090326f8"
+                "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20",
+                "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==2.11.0"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==2.8.0"
         },
         "pydocstyle": {
             "hashes": [
-                "sha256:118762d452a49d6b05e194ef344a55822987a462831ade91ec5c06fd2169d019",
-                "sha256:7ce43f0c0ac87b07494eb9c0b462c0b73e6ff276807f204d6b53edc72b7e44e1"
+                "sha256:1d41b7c459ba0ee6c345f2eb9ae827cab14a7533a88c5c6f7e94923f72df92dc",
+                "sha256:6987826d6775056839940041beef5c08cc7e3d71d63149b48e36727f70144dc4"
             ],
+            "index": "pypi",
             "markers": "python_version >= '3.6'",
-            "version": "==6.3.0"
+            "version": "==6.1.1"
         },
         "pyflakes": {
             "hashes": [
-                "sha256:4132f6d49cb4dae6819e5379898f2b8cce3c5f23994194c24b77d5da2e36f774",
-                "sha256:a0aae034c444db0071aa077972ba4768d40c830d9539fd45bf4cd3f8f6992efc"
+                "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c",
+                "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==3.1.0"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.4.0"
         },
         "pygments": {
             "hashes": [
-                "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692",
-                "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"
+                "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c",
+                "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.16.1"
+            "version": "==2.17.2"
         },
         "pytest": {
             "hashes": [
-                "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32",
-                "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"
+                "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c",
+                "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.7'",
-            "version": "==7.4.0"
+            "version": "==7.1.2"
         },
         "pytest-cov": {
             "hashes": [
-                "sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6",
-                "sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a"
+                "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6",
+                "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
-            "version": "==4.1.0"
+            "markers": "python_version >= '3.6'",
+            "version": "==3.0.0"
         },
         "pytest-flake8": {
             "hashes": [
@@ -550,19 +744,83 @@
         },
         "pytest-pydocstyle": {
             "hashes": [
-                "sha256:a30b28d49607b2fcd7b24678ab6c4e27a288710a34b3a0f1f90f3497e88771c3"
+                "sha256:1f2d937349cfeb4965c530a0c0f2442b48c03299558db435b65549719510d32b"
             ],
             "index": "pypi",
             "markers": "python_version ~= '3.7'",
-            "version": "==2.3.2"
+            "version": "==2.3.0"
+        },
+        "pywin32-ctypes": {
+            "hashes": [
+                "sha256:3426e063bdd5fd4df74a14fa3cf80a0b42845a87e1d1e81f6549f9daec593a60",
+                "sha256:bf490a1a709baf35d688fe0ecf980ed4de11d2b3e37b51e5442587a75d9957e7"
+            ],
+            "markers": "sys_platform == 'win32'",
+            "version": "==0.2.2"
+        },
+        "pyyaml": {
+            "hashes": [
+                "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5",
+                "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc",
+                "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df",
+                "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741",
+                "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206",
+                "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27",
+                "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595",
+                "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62",
+                "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98",
+                "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696",
+                "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290",
+                "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9",
+                "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d",
+                "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6",
+                "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867",
+                "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47",
+                "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486",
+                "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6",
+                "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3",
+                "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007",
+                "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938",
+                "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0",
+                "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c",
+                "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735",
+                "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d",
+                "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28",
+                "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4",
+                "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
+                "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
+                "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
+                "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd",
+                "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3",
+                "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0",
+                "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515",
+                "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c",
+                "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c",
+                "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924",
+                "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34",
+                "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43",
+                "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859",
+                "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673",
+                "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54",
+                "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a",
+                "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b",
+                "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab",
+                "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa",
+                "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c",
+                "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585",
+                "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d",
+                "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==6.0.1"
         },
         "readme-renderer": {
             "hashes": [
-                "sha256:4f4b11e5893f5a5d725f592c5a343e0dc74f5f273cb3dcf8c42d9703a27073f7",
-                "sha256:a38243d5b6741b700a850026e62da4bd739edc7422071e95fd5c4bb60171df86"
+                "sha256:13d039515c1f24de668e2c93f2e877b9dbe6c6c32328b90a40a49d8b2b85f36d",
+                "sha256:2d55489f83be4992fe4454939d1a051c33edbab778e82761d060c9fc6b308cd1"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==41.0"
+            "version": "==42.0"
         },
         "requests": {
             "hashes": [
@@ -590,27 +848,19 @@
         },
         "rich": {
             "hashes": [
-                "sha256:146a90b3b6b47cac4a73c12866a499e9817426423f57c5a66949c086191a8808",
-                "sha256:fb9d6c0a0f643c99eed3875b5377a184132ba9be4d61516a55273d3554d75a39"
+                "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa",
+                "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"
             ],
             "markers": "python_full_version >= '3.7.0'",
-            "version": "==13.5.2"
+            "version": "==13.7.0"
         },
         "setuptools": {
             "hashes": [
-                "sha256:3d4dfa6d95f1b101d695a6160a7626e15583af71a5f52176efa5d39a054d475d",
-                "sha256:3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b"
+                "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05",
+                "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==68.1.2"
-        },
-        "six": {
-            "hashes": [
-                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
-                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.16.0"
+            "version": "==69.0.3"
         },
         "snowballstemmer": {
             "hashes": [
@@ -684,6 +934,22 @@
             "markers": "python_version >= '3.5'",
             "version": "==1.1.5"
         },
+        "toml": {
+            "hashes": [
+                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
+                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
+            ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'",
+            "version": "==0.10.2"
+        },
+        "tomli": {
+            "hashes": [
+                "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc",
+                "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.1"
+        },
         "twine": {
             "hashes": [
                 "sha256:929bc3c280033347a00f847236564d1c52a3e61b1ac2516c97c48f3ceab756d8",
@@ -695,26 +961,27 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11",
-                "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"
+                "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3",
+                "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.0.4"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.1.0"
         },
-        "webencodings": {
+        "virtualenv": {
             "hashes": [
-                "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
-                "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"
+                "sha256:4238949c5ffe6876362d9c0180fc6c3a824a7b12b80604eeb8085f2ed7460de3",
+                "sha256:bf51c0d9c7dd63ea8e44086fa1e4fb1093a31e963b86959257378aef020e1f1b"
             ],
-            "version": "==0.5.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==20.25.0"
         },
         "zipp": {
             "hashes": [
-                "sha256:679e51dd4403591b2d6838a48de3d283f3d188412a9782faadf845f298736ba0",
-                "sha256:ebc15946aa78bd63458992fc81ec3b6f7b1e92d51c35e6de1c3804e73b799147"
+                "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31",
+                "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.16.2"
+            "version": "==3.17.0"
         }
     },
     "extras": {
@@ -728,100 +995,115 @@
         },
         "attrs": {
             "hashes": [
-                "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04",
-                "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"
+                "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30",
+                "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==23.1.0"
+            "version": "==23.2.0"
         },
         "certifi": {
             "hashes": [
-                "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082",
-                "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
+                "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1",
+                "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2023.7.22"
+            "version": "==2023.11.17"
         },
         "charset-normalizer": {
             "hashes": [
-                "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96",
-                "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c",
-                "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710",
-                "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706",
-                "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020",
-                "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252",
-                "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad",
-                "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329",
-                "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a",
-                "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f",
-                "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6",
-                "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4",
-                "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a",
-                "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46",
-                "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2",
-                "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23",
-                "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace",
-                "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd",
-                "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982",
-                "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10",
-                "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2",
-                "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea",
-                "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09",
-                "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5",
-                "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149",
-                "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489",
-                "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9",
-                "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80",
-                "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592",
-                "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3",
-                "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6",
-                "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed",
-                "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c",
-                "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200",
-                "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a",
-                "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e",
-                "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d",
-                "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6",
-                "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623",
-                "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669",
-                "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3",
-                "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa",
-                "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9",
-                "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2",
-                "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f",
-                "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1",
-                "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4",
-                "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a",
-                "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8",
-                "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3",
-                "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029",
-                "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f",
-                "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959",
-                "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22",
-                "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7",
-                "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952",
-                "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346",
-                "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e",
-                "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d",
-                "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299",
-                "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd",
-                "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a",
-                "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3",
-                "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037",
-                "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94",
-                "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c",
-                "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858",
-                "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a",
-                "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449",
-                "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c",
-                "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918",
-                "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1",
-                "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c",
-                "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac",
-                "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"
+                "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027",
+                "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087",
+                "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786",
+                "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8",
+                "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09",
+                "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185",
+                "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574",
+                "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e",
+                "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519",
+                "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898",
+                "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269",
+                "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3",
+                "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f",
+                "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6",
+                "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8",
+                "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a",
+                "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73",
+                "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc",
+                "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714",
+                "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2",
+                "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc",
+                "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce",
+                "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d",
+                "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e",
+                "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6",
+                "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269",
+                "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96",
+                "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d",
+                "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a",
+                "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4",
+                "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77",
+                "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d",
+                "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0",
+                "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed",
+                "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068",
+                "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac",
+                "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25",
+                "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8",
+                "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab",
+                "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26",
+                "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2",
+                "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db",
+                "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f",
+                "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5",
+                "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99",
+                "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c",
+                "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d",
+                "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811",
+                "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa",
+                "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a",
+                "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03",
+                "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b",
+                "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04",
+                "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c",
+                "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001",
+                "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458",
+                "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389",
+                "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99",
+                "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985",
+                "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537",
+                "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238",
+                "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f",
+                "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d",
+                "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796",
+                "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a",
+                "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143",
+                "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8",
+                "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c",
+                "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5",
+                "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5",
+                "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711",
+                "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4",
+                "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6",
+                "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c",
+                "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7",
+                "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4",
+                "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b",
+                "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae",
+                "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12",
+                "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c",
+                "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae",
+                "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8",
+                "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887",
+                "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b",
+                "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4",
+                "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f",
+                "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5",
+                "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33",
+                "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519",
+                "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"
             ],
             "markers": "python_full_version >= '3.7.0'",
-            "version": "==3.2.0"
+            "version": "==3.3.2"
         },
         "click": {
             "hashes": [
@@ -831,282 +1113,215 @@
             "markers": "python_version >= '3.7'",
             "version": "==8.1.7"
         },
+        "colorama": {
+            "hashes": [
+                "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44",
+                "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"
+            ],
+            "markers": "platform_system == 'Windows'",
+            "version": "==0.4.6"
+        },
         "filelock": {
             "hashes": [
-                "sha256:0ecc1dd2ec4672a10c8550a8182f1bd0c0a5088470ecd5a125e45f49472fac3d",
-                "sha256:f067e40ccc40f2b48395a80fcbd4728262fab54e232e090a4063ab804179efeb"
+                "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e",
+                "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.12.3"
+            "version": "==3.13.1"
         },
         "frozenlist": {
             "hashes": [
-                "sha256:007df07a6e3eb3e33e9a1fe6a9db7af152bbd8a185f9aaa6ece10a3529e3e1c6",
-                "sha256:008eb8b31b3ea6896da16c38c1b136cb9fec9e249e77f6211d479db79a4eaf01",
-                "sha256:09163bdf0b2907454042edb19f887c6d33806adc71fbd54afc14908bfdc22251",
-                "sha256:0c7c1b47859ee2cac3846fde1c1dc0f15da6cec5a0e5c72d101e0f83dcb67ff9",
-                "sha256:0e5c8764c7829343d919cc2dfc587a8db01c4f70a4ebbc49abde5d4b158b007b",
-                "sha256:10ff5faaa22786315ef57097a279b833ecab1a0bfb07d604c9cbb1c4cdc2ed87",
-                "sha256:17ae5cd0f333f94f2e03aaf140bb762c64783935cc764ff9c82dff626089bebf",
-                "sha256:19488c57c12d4e8095a922f328df3f179c820c212940a498623ed39160bc3c2f",
-                "sha256:1a0848b52815006ea6596c395f87449f693dc419061cc21e970f139d466dc0a0",
-                "sha256:1e78fb68cf9c1a6aa4a9a12e960a5c9dfbdb89b3695197aa7064705662515de2",
-                "sha256:261b9f5d17cac914531331ff1b1d452125bf5daa05faf73b71d935485b0c510b",
-                "sha256:2b8bcf994563466db019fab287ff390fffbfdb4f905fc77bc1c1d604b1c689cc",
-                "sha256:38461d02d66de17455072c9ba981d35f1d2a73024bee7790ac2f9e361ef1cd0c",
-                "sha256:490132667476f6781b4c9458298b0c1cddf237488abd228b0b3650e5ecba7467",
-                "sha256:491e014f5c43656da08958808588cc6c016847b4360e327a62cb308c791bd2d9",
-                "sha256:515e1abc578dd3b275d6a5114030b1330ba044ffba03f94091842852f806f1c1",
-                "sha256:556de4430ce324c836789fa4560ca62d1591d2538b8ceb0b4f68fb7b2384a27a",
-                "sha256:5833593c25ac59ede40ed4de6d67eb42928cca97f26feea219f21d0ed0959b79",
-                "sha256:6221d84d463fb110bdd7619b69cb43878a11d51cbb9394ae3105d082d5199167",
-                "sha256:6918d49b1f90821e93069682c06ffde41829c346c66b721e65a5c62b4bab0300",
-                "sha256:6c38721585f285203e4b4132a352eb3daa19121a035f3182e08e437cface44bf",
-                "sha256:71932b597f9895f011f47f17d6428252fc728ba2ae6024e13c3398a087c2cdea",
-                "sha256:7211ef110a9194b6042449431e08c4d80c0481e5891e58d429df5899690511c2",
-                "sha256:764226ceef3125e53ea2cb275000e309c0aa5464d43bd72abd661e27fffc26ab",
-                "sha256:7645a8e814a3ee34a89c4a372011dcd817964ce8cb273c8ed6119d706e9613e3",
-                "sha256:76d4711f6f6d08551a7e9ef28c722f4a50dd0fc204c56b4bcd95c6cc05ce6fbb",
-                "sha256:7f4f399d28478d1f604c2ff9119907af9726aed73680e5ed1ca634d377abb087",
-                "sha256:88f7bc0fcca81f985f78dd0fa68d2c75abf8272b1f5c323ea4a01a4d7a614efc",
-                "sha256:8d0edd6b1c7fb94922bf569c9b092ee187a83f03fb1a63076e7774b60f9481a8",
-                "sha256:901289d524fdd571be1c7be054f48b1f88ce8dddcbdf1ec698b27d4b8b9e5d62",
-                "sha256:93ea75c050c5bb3d98016b4ba2497851eadf0ac154d88a67d7a6816206f6fa7f",
-                "sha256:981b9ab5a0a3178ff413bca62526bb784249421c24ad7381e39d67981be2c326",
-                "sha256:9ac08e601308e41eb533f232dbf6b7e4cea762f9f84f6357136eed926c15d12c",
-                "sha256:a02eb8ab2b8f200179b5f62b59757685ae9987996ae549ccf30f983f40602431",
-                "sha256:a0c6da9aee33ff0b1a451e867da0c1f47408112b3391dd43133838339e410963",
-                "sha256:a6c8097e01886188e5be3e6b14e94ab365f384736aa1fca6a0b9e35bd4a30bc7",
-                "sha256:aa384489fefeb62321b238e64c07ef48398fe80f9e1e6afeff22e140e0850eef",
-                "sha256:ad2a9eb6d9839ae241701d0918f54c51365a51407fd80f6b8289e2dfca977cc3",
-                "sha256:b206646d176a007466358aa21d85cd8600a415c67c9bd15403336c331a10d956",
-                "sha256:b826d97e4276750beca7c8f0f1a4938892697a6bcd8ec8217b3312dad6982781",
-                "sha256:b89ac9768b82205936771f8d2eb3ce88503b1556324c9f903e7156669f521472",
-                "sha256:bd7bd3b3830247580de99c99ea2a01416dfc3c34471ca1298bccabf86d0ff4dc",
-                "sha256:bdf1847068c362f16b353163391210269e4f0569a3c166bc6a9f74ccbfc7e839",
-                "sha256:c11b0746f5d946fecf750428a95f3e9ebe792c1ee3b1e96eeba145dc631a9672",
-                "sha256:c5374b80521d3d3f2ec5572e05adc94601985cc526fb276d0c8574a6d749f1b3",
-                "sha256:ca265542ca427bf97aed183c1676e2a9c66942e822b14dc6e5f42e038f92a503",
-                "sha256:ce31ae3e19f3c902de379cf1323d90c649425b86de7bbdf82871b8a2a0615f3d",
-                "sha256:ceb6ec0a10c65540421e20ebd29083c50e6d1143278746a4ef6bcf6153171eb8",
-                "sha256:d081f13b095d74b67d550de04df1c756831f3b83dc9881c38985834387487f1b",
-                "sha256:d5655a942f5f5d2c9ed93d72148226d75369b4f6952680211972a33e59b1dfdc",
-                "sha256:d5a32087d720c608f42caed0ef36d2b3ea61a9d09ee59a5142d6070da9041b8f",
-                "sha256:d6484756b12f40003c6128bfcc3fa9f0d49a687e171186c2d85ec82e3758c559",
-                "sha256:dd65632acaf0d47608190a71bfe46b209719bf2beb59507db08ccdbe712f969b",
-                "sha256:de343e75f40e972bae1ef6090267f8260c1446a1695e77096db6cfa25e759a95",
-                "sha256:e29cda763f752553fa14c68fb2195150bfab22b352572cb36c43c47bedba70eb",
-                "sha256:e41f3de4df3e80de75845d3e743b3f1c4c8613c3997a912dbf0229fc61a8b963",
-                "sha256:e66d2a64d44d50d2543405fb183a21f76b3b5fd16f130f5c99187c3fb4e64919",
-                "sha256:e74b0506fa5aa5598ac6a975a12aa8928cbb58e1f5ac8360792ef15de1aa848f",
-                "sha256:f0ed05f5079c708fe74bf9027e95125334b6978bf07fd5ab923e9e55e5fbb9d3",
-                "sha256:f61e2dc5ad442c52b4887f1fdc112f97caeff4d9e6ebe78879364ac59f1663e1",
-                "sha256:fec520865f42e5c7f050c2a79038897b1c7d1595e907a9e08e3353293ffc948e"
+                "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7",
+                "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98",
+                "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad",
+                "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5",
+                "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae",
+                "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e",
+                "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a",
+                "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701",
+                "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d",
+                "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6",
+                "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6",
+                "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106",
+                "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75",
+                "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868",
+                "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a",
+                "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0",
+                "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1",
+                "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826",
+                "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec",
+                "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6",
+                "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950",
+                "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19",
+                "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0",
+                "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8",
+                "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a",
+                "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09",
+                "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86",
+                "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c",
+                "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5",
+                "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b",
+                "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b",
+                "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d",
+                "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0",
+                "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea",
+                "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776",
+                "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a",
+                "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897",
+                "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7",
+                "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09",
+                "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9",
+                "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe",
+                "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd",
+                "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742",
+                "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09",
+                "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0",
+                "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932",
+                "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1",
+                "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a",
+                "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49",
+                "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d",
+                "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7",
+                "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480",
+                "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89",
+                "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e",
+                "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b",
+                "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82",
+                "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb",
+                "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068",
+                "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8",
+                "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b",
+                "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb",
+                "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2",
+                "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11",
+                "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b",
+                "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc",
+                "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0",
+                "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497",
+                "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17",
+                "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0",
+                "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2",
+                "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439",
+                "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5",
+                "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac",
+                "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825",
+                "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887",
+                "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced",
+                "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.4.0"
-        },
-        "grpcio": {
-            "hashes": [
-                "sha256:00258cbe3f5188629828363ae8ff78477ce976a6f63fb2bb5e90088396faa82e",
-                "sha256:092fa155b945015754bdf988be47793c377b52b88d546e45c6a9f9579ac7f7b6",
-                "sha256:0f80bf37f09e1caba6a8063e56e2b87fa335add314cf2b78ebf7cb45aa7e3d06",
-                "sha256:20ec6fc4ad47d1b6e12deec5045ec3cd5402d9a1597f738263e98f490fe07056",
-                "sha256:2313b124e475aa9017a9844bdc5eafb2d5abdda9d456af16fc4535408c7d6da6",
-                "sha256:23e7d8849a0e58b806253fd206ac105b328171e01b8f18c7d5922274958cc87e",
-                "sha256:2f708a6a17868ad8bf586598bee69abded4996b18adf26fd2d91191383b79019",
-                "sha256:2f7349786da979a94690cc5c2b804cab4e8774a3cf59be40d037c4342c906649",
-                "sha256:34950353539e7d93f61c6796a007c705d663f3be41166358e3d88c45760c7d98",
-                "sha256:40b72effd4c789de94ce1be2b5f88d7b9b5f7379fe9645f198854112a6567d9a",
-                "sha256:4b089f7ad1eb00a104078bab8015b0ed0ebcb3b589e527ab009c53893fd4e613",
-                "sha256:4faea2cfdf762a664ab90589b66f416274887641ae17817de510b8178356bf73",
-                "sha256:5371bcd861e679d63b8274f73ac281751d34bd54eccdbfcd6aa00e692a82cd7b",
-                "sha256:5613a2fecc82f95d6c51d15b9a72705553aa0d7c932fad7aed7afb51dc982ee5",
-                "sha256:57b183e8b252825c4dd29114d6c13559be95387aafc10a7be645462a0fc98bbb",
-                "sha256:5b7a4ce8f862fe32b2a10b57752cf3169f5fe2915acfe7e6a1e155db3da99e79",
-                "sha256:5e5b58e32ae14658085c16986d11e99abd002ddbf51c8daae8a0671fffb3467f",
-                "sha256:60fe15288a0a65d5c1cb5b4a62b1850d07336e3ba728257a810317be14f0c527",
-                "sha256:6907b1cf8bb29b058081d2aad677b15757a44ef2d4d8d9130271d2ad5e33efca",
-                "sha256:76c44efa4ede1f42a9d5b2fed1fe9377e73a109bef8675fb0728eb80b0b8e8f2",
-                "sha256:7a635589201b18510ff988161b7b573f50c6a48fae9cb567657920ca82022b37",
-                "sha256:7b400807fa749a9eb286e2cd893e501b110b4d356a218426cb9c825a0474ca56",
-                "sha256:82640e57fb86ea1d71ea9ab54f7e942502cf98a429a200b2e743d8672171734f",
-                "sha256:871f9999e0211f9551f368612460442a5436d9444606184652117d6a688c9f51",
-                "sha256:9338bacf172e942e62e5889b6364e56657fbf8ac68062e8b25c48843e7b202bb",
-                "sha256:a8a8e560e8dbbdf29288872e91efd22af71e88b0e5736b0daf7773c1fecd99f0",
-                "sha256:aed90d93b731929e742967e236f842a4a2174dc5db077c8f9ad2c5996f89f63e",
-                "sha256:b363bbb5253e5f9c23d8a0a034dfdf1b7c9e7f12e602fc788c435171e96daccc",
-                "sha256:b4098b6b638d9e0ca839a81656a2fd4bc26c9486ea707e8b1437d6f9d61c3941",
-                "sha256:b53333627283e7241fcc217323f225c37783b5f0472316edcaa4479a213abfa6",
-                "sha256:b670c2faa92124b7397b42303e4d8eb64a4cd0b7a77e35a9e865a55d61c57ef9",
-                "sha256:bb396952cfa7ad2f01061fbc7dc1ad91dd9d69243bcb8110cf4e36924785a0fe",
-                "sha256:c60b83c43faeb6d0a9831f0351d7787a0753f5087cc6fa218d78fdf38e5acef0",
-                "sha256:c6ebecfb7a31385393203eb04ed8b6a08f5002f53df3d59e5e795edb80999652",
-                "sha256:d78d8b86fcdfa1e4c21f8896614b6cc7ee01a2a758ec0c4382d662f2a62cf766",
-                "sha256:d7f8df114d6b4cf5a916b98389aeaf1e3132035420a88beea4e3d977e5f267a5",
-                "sha256:e1cb52fa2d67d7f7fab310b600f22ce1ff04d562d46e9e0ac3e3403c2bb4cc16",
-                "sha256:e3fdf04e402f12e1de8074458549337febb3b45f21076cc02ef4ff786aff687e",
-                "sha256:e503cb45ed12b924b5b988ba9576dc9949b2f5283b8e33b21dcb6be74a7c58d0",
-                "sha256:f19ac6ac0a256cf77d3cc926ef0b4e64a9725cc612f97228cd5dc4bd9dbab03b",
-                "sha256:f1fb0fd4a1e9b11ac21c30c169d169ef434c6e9344ee0ab27cfa6f605f6387b2",
-                "sha256:fada6b07ec4f0befe05218181f4b85176f11d531911b64c715d1875c4736d73a",
-                "sha256:fd173b4cf02b20f60860dc2ffe30115c18972d7d6d2d69df97ac38dee03be5bf",
-                "sha256:fe752639919aad9ffb0dee0d87f29a6467d1ef764f13c4644d212a9a853a078d",
-                "sha256:fee387d2fab144e8a34e0e9c5ca0f45c9376b99de45628265cfa9886b1dbe62b"
-            ],
-            "markers": "python_version >= '3.10'",
-            "version": "==1.57.0"
+            "version": "==1.4.1"
         },
         "idna": {
             "hashes": [
-                "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
-                "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
+                "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca",
+                "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==3.4"
+            "version": "==3.6"
         },
         "jsonschema": {
             "hashes": [
-                "sha256:043dc26a3845ff09d20e4420d6012a9c91c9aa8999fa184e7efcfeccb41e32cb",
-                "sha256:6e1e7569ac13be8139b2dd2c21a55d350066ee3f80df06c608b398cdc6f30e8f"
+                "sha256:4f614fd46d8d61258610998997743ec5492a648b33cf478c1ddc23ed4598a5fa",
+                "sha256:ed6231f0429ecf966f5bc8dfef245998220549cbbcf140f913b7464c52c3b6b3"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.19.0"
+            "version": "==4.20.0"
         },
         "jsonschema-specifications": {
             "hashes": [
-                "sha256:05adf340b659828a004220a9613be00fa3f223f2b82002e273dee62fd50524b1",
-                "sha256:c91a50404e88a1f6ba40636778e2ee08f6e24c5613fe4c53ac24578a5a7f72bb"
+                "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc",
+                "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2023.7.1"
+            "version": "==2023.12.1"
         },
         "msgpack": {
             "hashes": [
-                "sha256:06f5174b5f8ed0ed919da0e62cbd4ffde676a374aba4020034da05fab67b9164",
-                "sha256:0c05a4a96585525916b109bb85f8cb6511db1c6f5b9d9cbcbc940dc6b4be944b",
-                "sha256:137850656634abddfb88236008339fdaba3178f4751b28f270d2ebe77a563b6c",
-                "sha256:17358523b85973e5f242ad74aa4712b7ee560715562554aa2134d96e7aa4cbbf",
-                "sha256:18334484eafc2b1aa47a6d42427da7fa8f2ab3d60b674120bce7a895a0a85bdd",
-                "sha256:1835c84d65f46900920b3708f5ba829fb19b1096c1800ad60bae8418652a951d",
-                "sha256:1967f6129fc50a43bfe0951c35acbb729be89a55d849fab7686004da85103f1c",
-                "sha256:1ab2f3331cb1b54165976a9d976cb251a83183631c88076613c6c780f0d6e45a",
-                "sha256:1c0f7c47f0087ffda62961d425e4407961a7ffd2aa004c81b9c07d9269512f6e",
-                "sha256:20a97bf595a232c3ee6d57ddaadd5453d174a52594bf9c21d10407e2a2d9b3bd",
-                "sha256:20c784e66b613c7f16f632e7b5e8a1651aa5702463d61394671ba07b2fc9e025",
-                "sha256:266fa4202c0eb94d26822d9bfd7af25d1e2c088927fe8de9033d929dd5ba24c5",
-                "sha256:28592e20bbb1620848256ebc105fc420436af59515793ed27d5c77a217477705",
-                "sha256:288e32b47e67f7b171f86b030e527e302c91bd3f40fd9033483f2cacc37f327a",
-                "sha256:3055b0455e45810820db1f29d900bf39466df96ddca11dfa6d074fa47054376d",
-                "sha256:332360ff25469c346a1c5e47cbe2a725517919892eda5cfaffe6046656f0b7bb",
-                "sha256:362d9655cd369b08fda06b6657a303eb7172d5279997abe094512e919cf74b11",
-                "sha256:366c9a7b9057e1547f4ad51d8facad8b406bab69c7d72c0eb6f529cf76d4b85f",
-                "sha256:36961b0568c36027c76e2ae3ca1132e35123dcec0706c4b7992683cc26c1320c",
-                "sha256:379026812e49258016dd84ad79ac8446922234d498058ae1d415f04b522d5b2d",
-                "sha256:382b2c77589331f2cb80b67cc058c00f225e19827dbc818d700f61513ab47bea",
-                "sha256:476a8fe8fae289fdf273d6d2a6cb6e35b5a58541693e8f9f019bfe990a51e4ba",
-                "sha256:48296af57cdb1d885843afd73c4656be5c76c0c6328db3440c9601a98f303d87",
-                "sha256:4867aa2df9e2a5fa5f76d7d5565d25ec76e84c106b55509e78c1ede0f152659a",
-                "sha256:4c075728a1095efd0634a7dccb06204919a2f67d1893b6aa8e00497258bf926c",
-                "sha256:4f837b93669ce4336e24d08286c38761132bc7ab29782727f8557e1eb21b2080",
-                "sha256:4f8d8b3bf1ff2672567d6b5c725a1b347fe838b912772aa8ae2bf70338d5a198",
-                "sha256:525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9",
-                "sha256:5494ea30d517a3576749cad32fa27f7585c65f5f38309c88c6d137877fa28a5a",
-                "sha256:55b56a24893105dc52c1253649b60f475f36b3aa0fc66115bffafb624d7cb30b",
-                "sha256:56a62ec00b636583e5cb6ad313bbed36bb7ead5fa3a3e38938503142c72cba4f",
-                "sha256:57e1f3528bd95cc44684beda696f74d3aaa8a5e58c816214b9046512240ef437",
-                "sha256:586d0d636f9a628ddc6a17bfd45aa5b5efaf1606d2b60fa5d87b8986326e933f",
-                "sha256:5cb47c21a8a65b165ce29f2bec852790cbc04936f502966768e4aae9fa763cb7",
-                "sha256:6c4c68d87497f66f96d50142a2b73b97972130d93677ce930718f68828b382e2",
-                "sha256:821c7e677cc6acf0fd3f7ac664c98803827ae6de594a9f99563e48c5a2f27eb0",
-                "sha256:916723458c25dfb77ff07f4c66aed34e47503b2eb3188b3adbec8d8aa6e00f48",
-                "sha256:9e6ca5d5699bcd89ae605c150aee83b5321f2115695e741b99618f4856c50898",
-                "sha256:9f5ae84c5c8a857ec44dc180a8b0cc08238e021f57abdf51a8182e915e6299f0",
-                "sha256:a2b031c2e9b9af485d5e3c4520f4220d74f4d222a5b8dc8c1a3ab9448ca79c57",
-                "sha256:a61215eac016f391129a013c9e46f3ab308db5f5ec9f25811e811f96962599a8",
-                "sha256:a740fa0e4087a734455f0fc3abf5e746004c9da72fbd541e9b113013c8dc3282",
-                "sha256:a9985b214f33311df47e274eb788a5893a761d025e2b92c723ba4c63936b69b1",
-                "sha256:ab31e908d8424d55601ad7075e471b7d0140d4d3dd3272daf39c5c19d936bd82",
-                "sha256:ac9dd47af78cae935901a9a500104e2dea2e253207c924cc95de149606dc43cc",
-                "sha256:addab7e2e1fcc04bd08e4eb631c2a90960c340e40dfc4a5e24d2ff0d5a3b3edb",
-                "sha256:b1d46dfe3832660f53b13b925d4e0fa1432b00f5f7210eb3ad3bb9a13c6204a6",
-                "sha256:b2de4c1c0538dcb7010902a2b97f4e00fc4ddf2c8cda9749af0e594d3b7fa3d7",
-                "sha256:b5ef2f015b95f912c2fcab19c36814963b5463f1fb9049846994b007962743e9",
-                "sha256:b72d0698f86e8d9ddf9442bdedec15b71df3598199ba33322d9711a19f08145c",
-                "sha256:bae7de2026cbfe3782c8b78b0db9cbfc5455e079f1937cb0ab8d133496ac55e1",
-                "sha256:bf22a83f973b50f9d38e55c6aade04c41ddda19b00c4ebc558930d78eecc64ed",
-                "sha256:c075544284eadc5cddc70f4757331d99dcbc16b2bbd4849d15f8aae4cf36d31c",
-                "sha256:c396e2cc213d12ce017b686e0f53497f94f8ba2b24799c25d913d46c08ec422c",
-                "sha256:cb5aaa8c17760909ec6cb15e744c3ebc2ca8918e727216e79607b7bbce9c8f77",
-                "sha256:cdc793c50be3f01106245a61b739328f7dccc2c648b501e237f0699fe1395b81",
-                "sha256:d25dd59bbbbb996eacf7be6b4ad082ed7eacc4e8f3d2df1ba43822da9bfa122a",
-                "sha256:e42b9594cc3bf4d838d67d6ed62b9e59e201862a25e9a157019e171fbe672dd3",
-                "sha256:e57916ef1bd0fee4f21c4600e9d1da352d8816b52a599c46460e93a6e9f17086",
-                "sha256:ed40e926fa2f297e8a653c954b732f125ef97bdd4c889f243182299de27e2aa9",
-                "sha256:ef8108f8dedf204bb7b42994abf93882da1159728a2d4c5e82012edd92c9da9f",
-                "sha256:f933bbda5a3ee63b8834179096923b094b76f0c7a73c1cfe8f07ad608c58844b",
-                "sha256:fe5c63197c55bce6385d9aee16c4d0641684628f63ace85f73571e65ad1c1e8d"
-            ],
-            "version": "==1.0.5"
-        },
-        "numpy": {
-            "hashes": [
-                "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2",
-                "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55",
-                "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf",
-                "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01",
-                "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca",
-                "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901",
-                "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d",
-                "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4",
-                "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf",
-                "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380",
-                "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044",
-                "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545",
-                "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f",
-                "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f",
-                "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3",
-                "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364",
-                "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9",
-                "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418",
-                "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f",
-                "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295",
-                "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3",
-                "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187",
-                "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926",
-                "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357",
-                "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"
+                "sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862",
+                "sha256:0bfdd914e55e0d2c9e1526de210f6fe8ffe9705f2b1dfcc4aecc92a4cb4b533d",
+                "sha256:1dc93e8e4653bdb5910aed79f11e165c85732067614f180f70534f056da97db3",
+                "sha256:1e2d69948e4132813b8d1131f29f9101bc2c915f26089a6d632001a5c1349672",
+                "sha256:235a31ec7db685f5c82233bddf9858748b89b8119bf4538d514536c485c15fe0",
+                "sha256:27dcd6f46a21c18fa5e5deed92a43d4554e3df8d8ca5a47bf0615d6a5f39dbc9",
+                "sha256:28efb066cde83c479dfe5a48141a53bc7e5f13f785b92ddde336c716663039ee",
+                "sha256:3476fae43db72bd11f29a5147ae2f3cb22e2f1a91d575ef130d2bf49afd21c46",
+                "sha256:36e17c4592231a7dbd2ed09027823ab295d2791b3b1efb2aee874b10548b7524",
+                "sha256:384d779f0d6f1b110eae74cb0659d9aa6ff35aaf547b3955abf2ab4c901c4819",
+                "sha256:38949d30b11ae5f95c3c91917ee7a6b239f5ec276f271f28638dec9156f82cfc",
+                "sha256:3967e4ad1aa9da62fd53e346ed17d7b2e922cba5ab93bdd46febcac39be636fc",
+                "sha256:3e7bf4442b310ff154b7bb9d81eb2c016b7d597e364f97d72b1acc3817a0fdc1",
+                "sha256:3f0c8c6dfa6605ab8ff0611995ee30d4f9fcff89966cf562733b4008a3d60d82",
+                "sha256:484ae3240666ad34cfa31eea7b8c6cd2f1fdaae21d73ce2974211df099a95d81",
+                "sha256:4a7b4f35de6a304b5533c238bee86b670b75b03d31b7797929caa7a624b5dda6",
+                "sha256:4cb14ce54d9b857be9591ac364cb08dc2d6a5c4318c1182cb1d02274029d590d",
+                "sha256:4e71bc4416de195d6e9b4ee93ad3f2f6b2ce11d042b4d7a7ee00bbe0358bd0c2",
+                "sha256:52700dc63a4676669b341ba33520f4d6e43d3ca58d422e22ba66d1736b0a6e4c",
+                "sha256:572efc93db7a4d27e404501975ca6d2d9775705c2d922390d878fcf768d92c87",
+                "sha256:576eb384292b139821c41995523654ad82d1916da6a60cff129c715a6223ea84",
+                "sha256:5b0bf0effb196ed76b7ad883848143427a73c355ae8e569fa538365064188b8e",
+                "sha256:5b6ccc0c85916998d788b295765ea0e9cb9aac7e4a8ed71d12e7d8ac31c23c95",
+                "sha256:5ed82f5a7af3697b1c4786053736f24a0efd0a1b8a130d4c7bfee4b9ded0f08f",
+                "sha256:6d4c80667de2e36970ebf74f42d1088cc9ee7ef5f4e8c35eee1b40eafd33ca5b",
+                "sha256:730076207cb816138cf1af7f7237b208340a2c5e749707457d70705715c93b93",
+                "sha256:7687e22a31e976a0e7fc99c2f4d11ca45eff652a81eb8c8085e9609298916dcf",
+                "sha256:822ea70dc4018c7e6223f13affd1c5c30c0f5c12ac1f96cd8e9949acddb48a61",
+                "sha256:84b0daf226913133f899ea9b30618722d45feffa67e4fe867b0b5ae83a34060c",
+                "sha256:85765fdf4b27eb5086f05ac0491090fc76f4f2b28e09d9350c31aac25a5aaff8",
+                "sha256:8dd178c4c80706546702c59529ffc005681bd6dc2ea234c450661b205445a34d",
+                "sha256:8f5b234f567cf76ee489502ceb7165c2a5cecec081db2b37e35332b537f8157c",
+                "sha256:98bbd754a422a0b123c66a4c341de0474cad4a5c10c164ceed6ea090f3563db4",
+                "sha256:993584fc821c58d5993521bfdcd31a4adf025c7d745bbd4d12ccfecf695af5ba",
+                "sha256:a40821a89dc373d6427e2b44b572efc36a2778d3f543299e2f24eb1a5de65415",
+                "sha256:b291f0ee7961a597cbbcc77709374087fa2a9afe7bdb6a40dbbd9b127e79afee",
+                "sha256:b573a43ef7c368ba4ea06050a957c2a7550f729c31f11dd616d2ac4aba99888d",
+                "sha256:b610ff0f24e9f11c9ae653c67ff8cc03c075131401b3e5ef4b82570d1728f8a9",
+                "sha256:bdf38ba2d393c7911ae989c3bbba510ebbcdf4ecbdbfec36272abe350c454075",
+                "sha256:bfef2bb6ef068827bbd021017a107194956918ab43ce4d6dc945ffa13efbc25f",
+                "sha256:cab3db8bab4b7e635c1c97270d7a4b2a90c070b33cbc00c99ef3f9be03d3e1f7",
+                "sha256:cb70766519500281815dfd7a87d3a178acf7ce95390544b8c90587d76b227681",
+                "sha256:cca1b62fe70d761a282496b96a5e51c44c213e410a964bdffe0928e611368329",
+                "sha256:ccf9a39706b604d884d2cb1e27fe973bc55f2890c52f38df742bc1d79ab9f5e1",
+                "sha256:dc43f1ec66eb8440567186ae2f8c447d91e0372d793dfe8c222aec857b81a8cf",
+                "sha256:dd632777ff3beaaf629f1ab4396caf7ba0bdd075d948a69460d13d44357aca4c",
+                "sha256:e45ae4927759289c30ccba8d9fdce62bb414977ba158286b5ddaf8df2cddb5c5",
+                "sha256:e50ebce52f41370707f1e21a59514e3375e3edd6e1832f5e5235237db933c98b",
+                "sha256:ebbbba226f0a108a7366bf4b59bf0f30a12fd5e75100c630267d94d7f0ad20e5",
+                "sha256:ec79ff6159dffcc30853b2ad612ed572af86c92b5168aa3fc01a67b0fa40665e",
+                "sha256:f0936e08e0003f66bfd97e74ee530427707297b0d0361247e9b4f59ab78ddc8b",
+                "sha256:f26a07a6e877c76a88e3cecac8531908d980d3d5067ff69213653649ec0f60ad",
+                "sha256:f64e376cd20d3f030190e8c32e1c64582eba56ac6dc7d5b0b49a9d44021b52fd",
+                "sha256:f6ffbc252eb0d229aeb2f9ad051200668fc3a9aaa8994e49f0cb2ffe2b7867e7",
+                "sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002",
+                "sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc"
             ],
-            "index": "pypi",
-            "markers": "python_version >= '3.9'",
-            "version": "==1.25.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.0.7"
         },
         "packaging": {
             "hashes": [
-                "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61",
-                "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"
+                "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5",
+                "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==23.1"
+            "version": "==23.2"
         },
         "protobuf": {
             "hashes": [
-                "sha256:237b9a50bd3b7307d0d834c1b0eb1a6cd47d3f4c2da840802cd03ea288ae8880",
-                "sha256:25ae91d21e3ce8d874211110c2f7edd6384816fb44e06b2867afe35139e1fd1c",
-                "sha256:2b23bd6e06445699b12f525f3e92a916f2dcf45ffba441026357dea7fa46f42b",
-                "sha256:3b7b170d3491ceed33f723bbf2d5a260f8a4e23843799a3906f16ef736ef251e",
-                "sha256:4e69965e7e54de4db989289a9b971a099e626f6167a9351e9d112221fc691bc1",
-                "sha256:58e12d2c1aa428ece2281cef09bbaa6938b083bcda606db3da4e02e991a0d924",
-                "sha256:6bd26c1fa9038b26c5c044ee77e0ecb18463e957fefbaeb81a3feb419313a54e",
-                "sha256:77700b55ba41144fc64828e02afb41901b42497b8217b558e4a001f18a85f2e3",
-                "sha256:7fda70797ddec31ddfa3576cbdcc3ddbb6b3078b737a1a87ab9136af0570cd6e",
-                "sha256:839952e759fc40b5d46be319a265cf94920174d88de31657d5622b5d8d6be5cd",
-                "sha256:bb7aa97c252279da65584af0456f802bd4b2de429eb945bbc9b3d61a42a8cd16",
-                "sha256:c00c3c7eb9ad3833806e21e86dca448f46035242a680f81c3fe068ff65e79c74",
-                "sha256:c5cdd486af081bf752225b26809d2d0a85e575b80a84cde5172a05bbb1990099"
+                "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62",
+                "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d",
+                "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61",
+                "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62",
+                "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3",
+                "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9",
+                "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830",
+                "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6",
+                "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0",
+                "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020",
+                "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==4.24.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==4.25.2"
         },
         "pyyaml": {
             "hashes": [
@@ -1166,41 +1381,38 @@
         },
         "ray": {
             "hashes": [
-                "sha256:015a2aa30aba0719d20cdf8fa32c689b68016678cb20f46bd1df8b227c938b84",
-                "sha256:0a5870f9a16cb94080d770f83326d7e2163d88d75be240273cef4b932a071bb2",
-                "sha256:18d033cc468e5171d9995476c33f99a5b79f091c34265c7e9f3d8b1c9042437e",
-                "sha256:1a8de31a9a4049134cf7e97b725a4078c958a964d091cb3e812e31eddd013bd7",
-                "sha256:31f1dd05130e712b9b64ccad9e6eaa82c715bb25a0a45ffd48ebf4953f6fe347",
-                "sha256:3ccf809e5948333c1c8c81694514b5900259e79cbdc8bddd3680695820cafcf2",
-                "sha256:3e5a4bbc29268a64bd2a8d48ed60f32a5bcce285a2a4f4339174947733449e37",
-                "sha256:467b9aa63f09d20e3985457816d703fe27ea388cdcaa88ff5eff222f8074a05c",
-                "sha256:485e4cd46a569416a14a72c06fe7901b0e3902f3023100b375c477975824e707",
-                "sha256:4b4600c93e2e94b6ca75ef4b4cb92d7f98d4be5484273d6fbac4218fb82cf96f",
-                "sha256:56b920a1814decdd20a754b7c5048770684d6d3d242c83aa99da5d3e8c339f13",
-                "sha256:5923849ec0854ab3e5ca8873d47ed7e11074e1213a3c40f8864c9500de034313",
-                "sha256:787ec7f43f5b3ed85728cf4878bdfed0a334d9108b6af75ef3fe5c8d44a7f74d",
-                "sha256:81e2ee7252e2fbfb05a24124774a8de563daa261200a08d9cbc6b499f7262af1",
-                "sha256:8a3cde58dba07da7a62e1f804b3dae5b29de3be052e02e4559bff7e7cb4d4a3b",
-                "sha256:90b780e131f891185f9de2b9c08d1f2d729e5755c7389a1ddaa6f796fae0d787",
-                "sha256:a182a80aebf863b5d4e875bed0a80e83200e84f4f63c4126cef87cc01e43f067",
-                "sha256:a4ef2f52319286720be7f3bfe6043e9fd0b8cb7826cb2ffc90c23c1c42427464",
-                "sha256:abc6a537454506a5fa87137de058d12aeea38da7077aae6f0ebf6199e5f5b2a1",
-                "sha256:b358fd112876c3a249fd8cffbf20b26622817c78b2ade0a725a7036c693f8d70",
-                "sha256:bca66c8e8163f06dc5443623e7b221660529a39574a589ba9257f2188ea8bf6b",
-                "sha256:bdeacaafcbb97e5f1c3c3349e7fcc0c40f691cea2bf057027c5491ea1ac929b0",
-                "sha256:dff21468d621c8dac95b3df320e6c6121f6618f6827243fd75a057c8815c2498",
-                "sha256:e0f8eaf4c4592335722dad474685c2ffc98207b997e47a24b297a60db389a4cb"
+                "sha256:013984b5d76b3ce63ab4616a5e57b4545524003d8b3df27df90007545cc6e364",
+                "sha256:06f34afc29fd392361435aa5425630d3851824e923263607cb0a5404083a23f9",
+                "sha256:13c555fe730fce355726e8dae7a7d6cedbe470a7e125748008ebfc44b0c5827d",
+                "sha256:15e075f647b52ec210538985b4cb2665f64fb76acab77f66f1893653964db64e",
+                "sha256:1751d9672208b7142b9dbc6de9766ffc92e1a7fe522ca45bcc88bbf88ca5d202",
+                "sha256:1dcf0b476f97bd552531279bb8a1c0b677001433e522cc0f33ffe29c920ed693",
+                "sha256:47d9d949e362112213bc53631b08183d1fe254d66d58131377cee913e5891597",
+                "sha256:585aa849afb1cadc0933dc5d251bb8fffe87b7b87b312ca66065b058e2fc2821",
+                "sha256:724ff0103919fb98181010cfbcd0d52a1b78b0dc84cbfd6e7ea0094b74e90a26",
+                "sha256:8de5efb388d503bb35d92f1570b8456cf3f2d01e856a9003814164356d2d75e7",
+                "sha256:93372482171c69e5543aae4cb739bcbe671d5c7d498c0ce761c23813e0f35b84",
+                "sha256:b2211c39bae3f415e32fe9fe23f67acfea4cff80fc37fb794a5767497ac8f2b7",
+                "sha256:b4108832754156cbf296402c5e44ad23758ac190ef923ff91036dbddde6a2d3d",
+                "sha256:bb79596c449c4ba027bc9839299617d8c876b1a5b61f16a1e401aa901ad45183",
+                "sha256:d6f2335a1d7724143e2732e7c4761ee9b572ec924445515808b0951f362a4dbf",
+                "sha256:dabba731106e3a5f0093d2eeae21c822db1f01768e7806eb4f39f06db94eec12",
+                "sha256:e54cef078e75718a56fe65d4b5be14e7193fc0743c6dba3e6d78ad1284e13556",
+                "sha256:eca277062646ef4ce87ffe249a0a816dba0b80c5720708c9973dcb6c17527fa1",
+                "sha256:ef8ba4d6126d8aacfc611b967a23e3e9571edf010756277991e8de9af56bd0ee",
+                "sha256:f245d0a45a32e67e1279bffc02b33ebe73fedd679c00f6b1623681275aa3f488"
             ],
             "index": "pypi",
-            "version": "==2.6.3"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.9.0"
         },
         "referencing": {
             "hashes": [
-                "sha256:449b6669b6121a9e96a7f9e410b245d471e8d48964c67113ce9afe50c8dd7bdf",
-                "sha256:794ad8003c65938edcdbc027f1933215e0d0ccc0291e3ce20a4d87432b59efc0"
+                "sha256:3c57da0513e9563eb7e203ebe9bb3a1b509b042016433bd1e45a2853466c3dd3",
+                "sha256:7e4dc12271d8e15612bfe35792f5ea1c40970dadf8624602e33db2758f7ee554"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.30.2"
+            "version": "==0.32.1"
         },
         "requests": {
             "hashes": [
@@ -1212,114 +1424,116 @@
         },
         "rpds-py": {
             "hashes": [
-                "sha256:00215f6a9058fbf84f9d47536902558eb61f180a6b2a0fa35338d06ceb9a2e5a",
-                "sha256:0028eb0967942d0d2891eae700ae1a27b7fd18604cfcb16a1ef486a790fee99e",
-                "sha256:0155c33af0676fc38e1107679be882077680ad1abb6303956b97259c3177e85e",
-                "sha256:063411228b852fb2ed7485cf91f8e7d30893e69b0acb207ec349db04cccc8225",
-                "sha256:0700c2133ba203c4068aaecd6a59bda22e06a5e46255c9da23cbf68c6942215d",
-                "sha256:08e08ccf5b10badb7d0a5c84829b914c6e1e1f3a716fdb2bf294e2bd01562775",
-                "sha256:0d292cabd7c8335bdd3237ded442480a249dbcdb4ddfac5218799364a01a0f5c",
-                "sha256:15932ec5f224b0e35764dc156514533a4fca52dcfda0dfbe462a1a22b37efd59",
-                "sha256:18f87baa20e02e9277ad8960cd89b63c79c05caf106f4c959a9595c43f2a34a5",
-                "sha256:1a6420a36975e0073acaeee44ead260c1f6ea56812cfc6c31ec00c1c48197173",
-                "sha256:1b401e8b9aece651512e62c431181e6e83048a651698a727ea0eb0699e9f9b74",
-                "sha256:1d7b7b71bcb82d8713c7c2e9c5f061415598af5938666beded20d81fa23e7640",
-                "sha256:23750a9b8a329844ba1fe267ca456bb3184984da2880ed17ae641c5af8de3fef",
-                "sha256:23a059143c1393015c68936370cce11690f7294731904bdae47cc3e16d0b2474",
-                "sha256:26d9fd624649a10e4610fab2bc820e215a184d193e47d0be7fe53c1c8f67f370",
-                "sha256:291c9ce3929a75b45ce8ddde2aa7694fc8449f2bc8f5bd93adf021efaae2d10b",
-                "sha256:298e8b5d8087e0330aac211c85428c8761230ef46a1f2c516d6a2f67fb8803c5",
-                "sha256:2c7c4266c1b61eb429e8aeb7d8ed6a3bfe6c890a1788b18dbec090c35c6b93fa",
-                "sha256:2d68a8e8a3a816629283faf82358d8c93fe5bd974dd2704152394a3de4cec22a",
-                "sha256:344b89384c250ba6a4ce1786e04d01500e4dac0f4137ceebcaad12973c0ac0b3",
-                "sha256:3455ecc46ea443b5f7d9c2f946ce4017745e017b0d0f8b99c92564eff97e97f5",
-                "sha256:3d544a614055b131111bed6edfa1cb0fb082a7265761bcb03321f2dd7b5c6c48",
-                "sha256:3e5c26905aa651cc8c0ddc45e0e5dea2a1296f70bdc96af17aee9d0493280a17",
-                "sha256:3f5cc8c7bc99d2bbcd704cef165ca7d155cd6464c86cbda8339026a42d219397",
-                "sha256:4992266817169997854f81df7f6db7bdcda1609972d8ffd6919252f09ec3c0f6",
-                "sha256:4d55528ef13af4b4e074d067977b1f61408602f53ae4537dccf42ba665c2c7bd",
-                "sha256:576da63eae7809f375932bfcbca2cf20620a1915bf2fedce4b9cc8491eceefe3",
-                "sha256:58fc4d66ee349a23dbf08c7e964120dc9027059566e29cf0ce6205d590ed7eca",
-                "sha256:5b9bf77008f2c55dabbd099fd3ac87009471d223a1c7ebea36873d39511b780a",
-                "sha256:5e7996aed3f65667c6dcc8302a69368435a87c2364079a066750a2eac75ea01e",
-                "sha256:5f7487be65b9c2c510819e744e375bd41b929a97e5915c4852a82fbb085df62c",
-                "sha256:6388e4e95a26717b94a05ced084e19da4d92aca883f392dffcf8e48c8e221a24",
-                "sha256:65af12f70355de29e1092f319f85a3467f4005e959ab65129cb697169ce94b86",
-                "sha256:668d2b45d62c68c7a370ac3dce108ffda482b0a0f50abd8b4c604a813a59e08f",
-                "sha256:71333c22f7cf5f0480b59a0aef21f652cf9bbaa9679ad261b405b65a57511d1e",
-                "sha256:7150b83b3e3ddaac81a8bb6a9b5f93117674a0e7a2b5a5b32ab31fdfea6df27f",
-                "sha256:748e472345c3a82cfb462d0dff998a7bf43e621eed73374cb19f307e97e08a83",
-                "sha256:75dbfd41a61bc1fb0536bf7b1abf272dc115c53d4d77db770cd65d46d4520882",
-                "sha256:7618a082c55cf038eede4a918c1001cc8a4411dfe508dc762659bcd48d8f4c6e",
-                "sha256:780fcb855be29153901c67fc9c5633d48aebef21b90aa72812fa181d731c6b00",
-                "sha256:78d10c431073dc6ebceed35ab22948a016cc2b5120963c13a41e38bdde4a7212",
-                "sha256:7a3a3d3e4f1e3cd2a67b93a0b6ed0f2499e33f47cc568e3a0023e405abdc0ff1",
-                "sha256:7b6975d3763d0952c111700c0634968419268e6bbc0b55fe71138987fa66f309",
-                "sha256:80772e3bda6787510d9620bc0c7572be404a922f8ccdfd436bf6c3778119464c",
-                "sha256:80992eb20755701753e30a6952a96aa58f353d12a65ad3c9d48a8da5ec4690cf",
-                "sha256:841128a22e6ac04070a0f84776d07e9c38c4dcce8e28792a95e45fc621605517",
-                "sha256:861d25ae0985a1dd5297fee35f476b60c6029e2e6e19847d5b4d0a43a390b696",
-                "sha256:872f3dcaa8bf2245944861d7311179d2c0c9b2aaa7d3b464d99a7c2e401f01fa",
-                "sha256:87c93b25d538c433fb053da6228c6290117ba53ff6a537c133b0f2087948a582",
-                "sha256:8856aa76839dc234d3469f1e270918ce6bec1d6a601eba928f45d68a15f04fc3",
-                "sha256:885e023e73ce09b11b89ab91fc60f35d80878d2c19d6213a32b42ff36543c291",
-                "sha256:899b5e7e2d5a8bc92aa533c2d4e55e5ebba095c485568a5e4bedbc163421259a",
-                "sha256:8ce8caa29ebbdcde67e5fd652c811d34bc01f249dbc0d61e5cc4db05ae79a83b",
-                "sha256:8e1c68303ccf7fceb50fbab79064a2636119fd9aca121f28453709283dbca727",
-                "sha256:8e7e2b3577e97fa43c2c2b12a16139b2cedbd0770235d5179c0412b4794efd9b",
-                "sha256:92f05fc7d832e970047662b3440b190d24ea04f8d3c760e33e7163b67308c878",
-                "sha256:97f5811df21703446b42303475b8b855ee07d6ab6cdf8565eff115540624f25d",
-                "sha256:9affee8cb1ec453382c27eb9043378ab32f49cd4bc24a24275f5c39bf186c279",
-                "sha256:a2da4a8c6d465fde36cea7d54bf47b5cf089073452f0e47c8632ecb9dec23c07",
-                "sha256:a6903cdca64f1e301af9be424798328c1fe3b4b14aede35f04510989fc72f012",
-                "sha256:a8ab1adf04ae2d6d65835995218fd3f3eb644fe20655ca8ee233e2c7270ff53b",
-                "sha256:a8edd467551c1102dc0f5754ab55cd0703431cd3044edf8c8e7d9208d63fa453",
-                "sha256:ac00c41dd315d147b129976204839ca9de699d83519ff1272afbe4fb9d362d12",
-                "sha256:ad277f74b1c164f7248afa968700e410651eb858d7c160d109fb451dc45a2f09",
-                "sha256:ae46a50d235f1631d9ec4670503f7b30405103034830bc13df29fd947207f795",
-                "sha256:afe6b5a04b2ab1aa89bad32ca47bf71358e7302a06fdfdad857389dca8fb5f04",
-                "sha256:b1cb078f54af0abd835ca76f93a3152565b73be0f056264da45117d0adf5e99c",
-                "sha256:b25136212a3d064a8f0b9ebbb6c57094c5229e0de76d15c79b76feff26aeb7b8",
-                "sha256:b3226b246facae14909b465061ddcfa2dfeadb6a64f407f24300d42d69bcb1a1",
-                "sha256:b98e75b21fc2ba5285aef8efaf34131d16af1c38df36bdca2f50634bea2d3060",
-                "sha256:bbd7b24d108509a1b9b6679fcc1166a7dd031dbef1f3c2c73788f42e3ebb3beb",
-                "sha256:bed57543c99249ab3a4586ddc8786529fbc33309e5e8a1351802a06ca2baf4c2",
-                "sha256:c0583f69522732bdd79dca4cd3873e63a29acf4a299769c7541f2ca1e4dd4bc6",
-                "sha256:c1e0e9916301e3b3d970814b1439ca59487f0616d30f36a44cead66ee1748c31",
-                "sha256:c651847545422c8131660704c58606d841e228ed576c8f1666d98b3d318f89da",
-                "sha256:c7853f27195598e550fe089f78f0732c66ee1d1f0eaae8ad081589a5a2f5d4af",
-                "sha256:cbae50d352e4717ffc22c566afc2d0da744380e87ed44a144508e3fb9114a3f4",
-                "sha256:cdbed8f21204398f47de39b0a9b180d7e571f02dfb18bf5f1b618e238454b685",
-                "sha256:d08395595c42bcd82c3608762ce734504c6d025eef1c06f42326a6023a584186",
-                "sha256:d4639111e73997567343df6551da9dd90d66aece1b9fc26c786d328439488103",
-                "sha256:d63787f289944cc4bde518ad2b5e70a4f0d6e2ce76324635359c74c113fd188f",
-                "sha256:d6d5f061f6a2aa55790b9e64a23dfd87b6664ab56e24cd06c78eb43986cb260b",
-                "sha256:d7865df1fb564092bcf46dac61b5def25342faf6352e4bc0e61a286e3fa26a3d",
-                "sha256:db6585b600b2e76e98131e0ac0e5195759082b51687ad0c94505970c90718f4a",
-                "sha256:e36d7369363d2707d5f68950a64c4e025991eb0177db01ccb6aa6facae48b69f",
-                "sha256:e7947d9a6264c727a556541b1630296bbd5d0a05068d21c38dde8e7a1c703ef0",
-                "sha256:eb2d59bc196e6d3b1827c7db06c1a898bfa0787c0574af398e65ccf2e97c0fbe",
-                "sha256:ee9c2f6ca9774c2c24bbf7b23086264e6b5fa178201450535ec0859739e6f78d",
-                "sha256:f4760e1b02173f4155203054f77a5dc0b4078de7645c922b208d28e7eb99f3e2",
-                "sha256:f70bec8a14a692be6dbe7ce8aab303e88df891cbd4a39af091f90b6702e28055",
-                "sha256:f869e34d2326e417baee430ae998e91412cc8e7fdd83d979277a90a0e79a5b47",
-                "sha256:f8b9a7cd381970e64849070aca7c32d53ab7d96c66db6c2ef7aa23c6e803f514",
-                "sha256:f99d74ddf9d3b6126b509e81865f89bd1283e3fc1b568b68cd7bd9dfa15583d7",
-                "sha256:f9e7e493ded7042712a374471203dd43ae3fff5b81e3de1a0513fa241af9fd41",
-                "sha256:fc72ae476732cdb7b2c1acb5af23b478b8a0d4b6fcf19b90dd150291e0d5b26b",
-                "sha256:fccbf0cd3411719e4c9426755df90bf3449d9fc5a89f077f4a7f1abd4f70c910",
-                "sha256:ffcf18ad3edf1c170e27e88b10282a2c449aa0358659592462448d71b2000cfc"
+                "sha256:0474df4ade9a3b4af96c3d36eb81856cb9462e4c6657d4caecfd840d2a13f3c9",
+                "sha256:071980663c273bf3d388fe5c794c547e6f35ba3335477072c713a3176bf14a60",
+                "sha256:07aab64e2808c3ebac2a44f67e9dc0543812b715126dfd6fe4264df527556cb6",
+                "sha256:088396c7c70e59872f67462fcac3ecbded5233385797021976a09ebd55961dfe",
+                "sha256:162d7cd9cd311c1b0ff1c55a024b8f38bd8aad1876b648821da08adc40e95734",
+                "sha256:19f00f57fdd38db4bb5ad09f9ead1b535332dbf624200e9029a45f1f35527ebb",
+                "sha256:1bdbc5fcb04a7309074de6b67fa9bc4b418ab3fc435fec1f2779a0eced688d04",
+                "sha256:1be2f033df1b8be8c3167ba3c29d5dca425592ee31e35eac52050623afba5772",
+                "sha256:24f7a2eb3866a9e91f4599851e0c8d39878a470044875c49bd528d2b9b88361c",
+                "sha256:290a81cfbe4673285cdf140ec5cd1658ffbf63ab359f2b352ebe172e7cfa5bf0",
+                "sha256:2946b120718eba9af2b4dd103affc1164a87b9e9ebff8c3e4c05d7b7a7e274e2",
+                "sha256:2bd82db36cd70b3628c0c57d81d2438e8dd4b7b32a6a9f25f24ab0e657cb6c4e",
+                "sha256:2ddef620e70eaffebed5932ce754d539c0930f676aae6212f8e16cd9743dd365",
+                "sha256:2e53b9b25cac9065328901713a7e9e3b12e4f57ef4280b370fbbf6fef2052eef",
+                "sha256:302bd4983bbd47063e452c38be66153760112f6d3635c7eeefc094299fa400a9",
+                "sha256:349cb40897fd529ca15317c22c0eab67f5ac5178b5bd2c6adc86172045210acc",
+                "sha256:358dafc89ce3894c7f486c615ba914609f38277ef67f566abc4c854d23b997fa",
+                "sha256:35953f4f2b3216421af86fd236b7c0c65935936a94ea83ddbd4904ba60757773",
+                "sha256:35ae5ece284cf36464eb160880018cf6088a9ac5ddc72292a6092b6ef3f4da53",
+                "sha256:3b811d182ad17ea294f2ec63c0621e7be92a1141e1012383461872cead87468f",
+                "sha256:3da5a4c56953bdbf6d04447c3410309616c54433146ccdb4a277b9cb499bc10e",
+                "sha256:3dc6a7620ba7639a3db6213da61312cb4aa9ac0ca6e00dc1cbbdc21c2aa6eb57",
+                "sha256:3f91df8e6dbb7360e176d1affd5fb0246d2b88d16aa5ebc7db94fd66b68b61da",
+                "sha256:4022b9dc620e14f30201a8a73898a873c8e910cb642bcd2f3411123bc527f6ac",
+                "sha256:413b9c17388bbd0d87a329d8e30c1a4c6e44e2bb25457f43725a8e6fe4161e9e",
+                "sha256:43d4dd5fb16eb3825742bad8339d454054261ab59fed2fbac84e1d84d5aae7ba",
+                "sha256:44627b6ca7308680a70766454db5249105fa6344853af6762eaad4158a2feebe",
+                "sha256:44a54e99a2b9693a37ebf245937fd6e9228b4cbd64b9cc961e1f3391ec6c7391",
+                "sha256:47713dc4fce213f5c74ca8a1f6a59b622fc1b90868deb8e8e4d993e421b4b39d",
+                "sha256:495a14b72bbe217f2695dcd9b5ab14d4f8066a00f5d209ed94f0aca307f85f6e",
+                "sha256:4c46ad6356e1561f2a54f08367d1d2e70a0a1bb2db2282d2c1972c1d38eafc3b",
+                "sha256:4d6a9f052e72d493efd92a77f861e45bab2f6be63e37fa8ecf0c6fd1a58fedb0",
+                "sha256:509b617ac787cd1149600e731db9274ebbef094503ca25158e6f23edaba1ca8f",
+                "sha256:5552f328eaef1a75ff129d4d0c437bf44e43f9436d3996e8eab623ea0f5fcf73",
+                "sha256:5a80e2f83391ad0808b4646732af2a7b67550b98f0cae056cb3b40622a83dbb3",
+                "sha256:5cf6af100ffb5c195beec11ffaa8cf8523057f123afa2944e6571d54da84cdc9",
+                "sha256:5e6caa3809e50690bd92fa490f5c38caa86082c8c3315aa438bce43786d5e90d",
+                "sha256:5ef00873303d678aaf8b0627e111fd434925ca01c657dbb2641410f1cdaef261",
+                "sha256:69ac7ea9897ec201ce68b48582f3eb34a3f9924488a5432a93f177bf76a82a7e",
+                "sha256:6a61226465bda9283686db8f17d02569a98e4b13c637be5a26d44aa1f1e361c2",
+                "sha256:6d904c5693e08bad240f16d79305edba78276be87061c872a4a15e2c301fa2c0",
+                "sha256:6dace7b26a13353e24613417ce2239491b40a6ad44e5776a18eaff7733488b44",
+                "sha256:6df15846ee3fb2e6397fe25d7ca6624af9f89587f3f259d177b556fed6bebe2c",
+                "sha256:703d95c75a72e902544fda08e965885525e297578317989fd15a6ce58414b41d",
+                "sha256:726ac36e8a3bb8daef2fd482534cabc5e17334052447008405daca7ca04a3108",
+                "sha256:781ef8bfc091b19960fc0142a23aedadafa826bc32b433fdfe6fd7f964d7ef44",
+                "sha256:80443fe2f7b3ea3934c5d75fb0e04a5dbb4a8e943e5ff2de0dec059202b70a8b",
+                "sha256:83640a5d7cd3bff694747d50436b8b541b5b9b9782b0c8c1688931d6ee1a1f2d",
+                "sha256:84c5a4d1f9dd7e2d2c44097fb09fffe728629bad31eb56caf97719e55575aa82",
+                "sha256:882ce6e25e585949c3d9f9abd29202367175e0aab3aba0c58c9abbb37d4982ff",
+                "sha256:888a97002e986eca10d8546e3c8b97da1d47ad8b69726dcfeb3e56348ebb28a3",
+                "sha256:8aad80645a011abae487d356e0ceb359f4938dfb6f7bcc410027ed7ae4f7bb8b",
+                "sha256:8cb6fe8ecdfffa0e711a75c931fb39f4ba382b4b3ccedeca43f18693864fe850",
+                "sha256:8d6b6937ae9eac6d6c0ca3c42774d89fa311f55adff3970fb364b34abde6ed3d",
+                "sha256:90123853fc8b1747f80b0d354be3d122b4365a93e50fc3aacc9fb4c2488845d6",
+                "sha256:96f957d6ab25a78b9e7fc9749d754b98eac825a112b4e666525ce89afcbd9ed5",
+                "sha256:981d135c7cdaf6cd8eadae1c950de43b976de8f09d8e800feed307140d3d6d00",
+                "sha256:9b32f742ce5b57201305f19c2ef7a184b52f6f9ba6871cc042c2a61f0d6b49b8",
+                "sha256:9f0350ef2fba5f34eb0c9000ea328e51b9572b403d2f7f3b19f24085f6f598e8",
+                "sha256:a297a4d08cc67c7466c873c78039d87840fb50d05473db0ec1b7b03d179bf322",
+                "sha256:a3d7e2ea25d3517c6d7e5a1cc3702cffa6bd18d9ef8d08d9af6717fc1c700eed",
+                "sha256:a4b682c5775d6a3d21e314c10124599976809455ee67020e8e72df1769b87bc3",
+                "sha256:a4ebb8b20bd09c5ce7884c8f0388801100f5e75e7f733b1b6613c713371feefc",
+                "sha256:a61f659665a39a4d17d699ab3593d7116d66e1e2e3f03ef3fb8f484e91908808",
+                "sha256:a9880b4656efe36ccad41edc66789e191e5ee19a1ea8811e0aed6f69851a82f4",
+                "sha256:ac08472f41ea77cd6a5dae36ae7d4ed3951d6602833af87532b556c1b4601d63",
+                "sha256:adc0c3d6fc6ae35fee3e4917628983f6ce630d513cbaad575b4517d47e81b4bb",
+                "sha256:af27423662f32d7501a00c5e7342f7dbd1e4a718aea7a239781357d15d437133",
+                "sha256:b2e75e17bd0bb66ee34a707da677e47c14ee51ccef78ed6a263a4cc965a072a1",
+                "sha256:b634c5ec0103c5cbebc24ebac4872b045cccb9456fc59efdcf6fe39775365bd2",
+                "sha256:b6f5549d6ed1da9bfe3631ca9483ae906f21410be2445b73443fa9f017601c6f",
+                "sha256:bd4b677d929cf1f6bac07ad76e0f2d5de367e6373351c01a9c0a39f6b21b4a8b",
+                "sha256:bf721ede3eb7b829e4a9b8142bd55db0bdc82902720548a703f7e601ee13bdc3",
+                "sha256:c647ca87fc0ebe808a41de912e9a1bfef9acb85257e5d63691364ac16b81c1f0",
+                "sha256:ca57468da2d9a660bcf8961637c85f2fbb2aa64d9bc3f9484e30c3f9f67b1dd7",
+                "sha256:cad0f59ee3dc35526039f4bc23642d52d5f6616b5f687d846bfc6d0d6d486db0",
+                "sha256:cc97f0640e91d7776530f06e6836c546c1c752a52de158720c4224c9e8053cad",
+                "sha256:ccd4e400309e1f34a5095bf9249d371f0fd60f8a3a5c4a791cad7b99ce1fd38d",
+                "sha256:cffa76b385dfe1e38527662a302b19ffb0e7f5cf7dd5e89186d2c94a22dd9d0c",
+                "sha256:d0dd7ed2f16df2e129496e7fbe59a34bc2d7fc8db443a606644d069eb69cbd45",
+                "sha256:d452817e0d9c749c431a1121d56a777bd7099b720b3d1c820f1725cb40928f58",
+                "sha256:d8dda2a806dfa4a9b795950c4f5cc56d6d6159f7d68080aedaff3bdc9b5032f5",
+                "sha256:dcbe1f8dd179e4d69b70b1f1d9bb6fd1e7e1bdc9c9aad345cdeb332e29d40748",
+                "sha256:e0441fb4fdd39a230477b2ca9be90868af64425bfe7b122b57e61e45737a653b",
+                "sha256:e04e56b4ca7a770593633556e8e9e46579d66ec2ada846b401252a2bdcf70a6d",
+                "sha256:e061de3b745fe611e23cd7318aec2c8b0e4153939c25c9202a5811ca911fd733",
+                "sha256:e93ec1b300acf89730cf27975ef574396bc04edecc358e9bd116fb387a123239",
+                "sha256:e9e557db6a177470316c82f023e5d571811c9a4422b5ea084c85da9aa3c035fc",
+                "sha256:eab36eae3f3e8e24b05748ec9acc66286662f5d25c52ad70cadab544e034536b",
+                "sha256:ec23fcad480e77ede06cf4127a25fc440f7489922e17fc058f426b5256ee0edb",
+                "sha256:ec2e1cf025b2c0f48ec17ff3e642661da7ee332d326f2e6619366ce8e221f018",
+                "sha256:ed99b4f7179d2111702020fd7d156e88acd533f5a7d3971353e568b6051d5c97",
+                "sha256:ee94cb58c0ba2c62ee108c2b7c9131b2c66a29e82746e8fa3aa1a1effbd3dcf1",
+                "sha256:f19afcfc0dd0dca35694df441e9b0f95bc231b512f51bded3c3d8ca32153ec19",
+                "sha256:f1b9d9260e06ea017feb7172976ab261e011c1dc2f8883c7c274f6b2aabfe01a",
+                "sha256:f28ac0e8e7242d140f99402a903a2c596ab71550272ae9247ad78f9a932b5698",
+                "sha256:f42e25c016927e2a6b1ce748112c3ab134261fc2ddc867e92d02006103e1b1b7",
+                "sha256:f4bd4578e44f26997e9e56c96dedc5f1af43cc9d16c4daa29c771a00b2a26851",
+                "sha256:f811771019f063bbd0aa7bb72c8a934bc13ebacb4672d712fc1639cfd314cccc"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.10.0"
+            "version": "==0.16.2"
         },
         "urllib3": {
             "hashes": [
-                "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11",
-                "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"
+                "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3",
+                "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.0.4"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.1.0"
         }
     }
 }

From 2afe7ecb6aafa561418eefac7a733a4b33eff9b7 Mon Sep 17 00:00:00 2001
From: LukasDrews97 <drews.lukas@yahoo.com>
Date: Thu, 11 Jan 2024 16:07:13 +0100
Subject: [PATCH 02/65] add 'calibration_method' parameter

---
 hiclass/HierarchicalClassifier.py       | 14 ++++++++++++++
 hiclass/LocalClassifierPerLevel.py      |  4 ++++
 hiclass/LocalClassifierPerNode.py       |  4 ++++
 hiclass/LocalClassifierPerParentNode.py |  4 ++++
 4 files changed, 26 insertions(+)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index 7b8f1dba..8e211a65 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -67,6 +67,7 @@ def __init__(
         n_jobs: int = 1,
         bert: bool = False,
         classifier_abbreviation: str = "",
+        calibration_method: str = None,
     ):
         """
         Initialize a local hierarchical classifier.
@@ -92,6 +93,8 @@ def __init__(
             If True, skip scikit-learn's checks and sample_weight passing for BERT.
         classifier_abbreviation : str, default=""
             The abbreviation of the local hierarchical classifier to be displayed during logging.
+        calibration_method : {"ivap", "cvap", "platt", "isotonic"}, str, default=None
+            If set, use the desired method to calibrate probabilities returned by predict_proba().
         """
         self.local_classifier = local_classifier
         self.verbose = verbose
@@ -100,6 +103,7 @@ def __init__(
         self.n_jobs = n_jobs
         self.bert = bert
         self.classifier_abbreviation = classifier_abbreviation
+        self.calibration_method = calibration_method
 
     def fit(self, X, y, sample_weight=None):
         """
@@ -175,6 +179,16 @@ def _pre_fit(self, X, y, sample_weight):
         # Initialize local classifiers in DAG
         self._initialize_local_classifiers()
 
+    def _calibrate(self, X, y):
+        # Create calibrator object
+        # seed
+        # fit calibrator using calibration data
+        # predict_proba can then use the calibrator to predict calibrated probabilities
+        pass
+
+    def predict_ood():
+        pass
+
     def _create_logger(self):
         # Create logger
         self.logger_ = logging.getLogger(self.classifier_abbreviation)
diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index ce0c8f5d..7b62d347 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -48,6 +48,7 @@ def __init__(
         replace_classifiers: bool = True,
         n_jobs: int = 1,
         bert: bool = False,
+        calibration_method: str = None,
     ):
         """
         Initialize a local classifier per level.
@@ -71,6 +72,8 @@ def __init__(
             If :code:`Ray` is installed it is used, otherwise it defaults to :code:`Joblib`.
         bert : bool, default=False
             If True, skip scikit-learn's checks and sample_weight passing for BERT.
+        calibration_method : {"ivap", "cvap", "platt", "isotonic"}, str, default=None
+            If set, use the desired method to calibrate probabilities returned by predict_proba().
         """
         super().__init__(
             local_classifier=local_classifier,
@@ -80,6 +83,7 @@ def __init__(
             n_jobs=n_jobs,
             classifier_abbreviation="LCPL",
             bert=bert,
+            calibration_method=calibration_method,
         )
 
     def fit(self, X, y, sample_weight=None):
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index c26c57bb..fafdbb05 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -43,6 +43,7 @@ def __init__(
         replace_classifiers: bool = True,
         n_jobs: int = 1,
         bert: bool = False,
+        calibration_method: str = None,
     ):
         """
         Initialize a local classifier per node.
@@ -77,6 +78,8 @@ def __init__(
             If :code:`Ray` is installed it is used, otherwise it defaults to :code:`Joblib`.
         bert : bool, default=False
             If True, skip scikit-learn's checks and sample_weight passing for BERT.
+        calibration_method : {"ivap", "cvap", "platt", "isotonic"}, str, default=None
+            If set, use the desired method to calibrate probabilities returned by predict_proba().
         """
         super().__init__(
             local_classifier=local_classifier,
@@ -86,6 +89,7 @@ def __init__(
             n_jobs=n_jobs,
             classifier_abbreviation="LCPN",
             bert=bert,
+            calibration_method=calibration_method,
         )
         self.binary_policy = binary_policy
 
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index 14895fd7..8fd43baf 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -41,6 +41,7 @@ def __init__(
         replace_classifiers: bool = True,
         n_jobs: int = 1,
         bert: bool = False,
+        calibration_method: str = None,
     ):
         """
         Initialize a local classifier per parent node.
@@ -64,6 +65,8 @@ def __init__(
             If :code:`Ray` is installed it is used, otherwise it defaults to :code:`Joblib`.
         bert : bool, default=False
             If True, skip scikit-learn's checks and sample_weight passing for BERT.
+        calibration_method : {"ivap", "cvap", "platt", "isotonic"}, str, default=None
+            If set, use the desired method to calibrate probabilities returned by predict_proba().
         """
         super().__init__(
             local_classifier=local_classifier,
@@ -73,6 +76,7 @@ def __init__(
             n_jobs=n_jobs,
             classifier_abbreviation="LCPPN",
             bert=bert,
+            calibration_method=calibration_method,
         )
 
     def fit(self, X, y, sample_weight=None):

From 211d24904c08f6a1795c4356a56d347680703b27 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Tue, 16 Jan 2024 08:37:02 +0100
Subject: [PATCH 03/65] update

---
 hiclass/HierarchicalClassifier.py          |   1 +
 hiclass/calibration/BinaryCalibrator.py    |  11 +
 hiclass/calibration/Calibrator.py          |  83 +++++++
 hiclass/calibration/IsotonicRegression.py  |  24 ++
 hiclass/calibration/PlattScaling.py        |  21 ++
 hiclass/calibration/VennAbersCalibrator.py | 255 +++++++++++++++++++++
 hiclass/calibration/__init__.py            |   0
 7 files changed, 395 insertions(+)
 create mode 100644 hiclass/calibration/BinaryCalibrator.py
 create mode 100644 hiclass/calibration/Calibrator.py
 create mode 100644 hiclass/calibration/IsotonicRegression.py
 create mode 100644 hiclass/calibration/PlattScaling.py
 create mode 100644 hiclass/calibration/VennAbersCalibrator.py
 create mode 100644 hiclass/calibration/__init__.py

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index 8e211a65..153d5316 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -8,6 +8,7 @@
 from sklearn.base import BaseEstimator
 from sklearn.linear_model import LogisticRegression
 from sklearn.utils.validation import _check_sample_weight
+from calibration.Calibrator import _Calibrator
 
 try:
     import ray
diff --git a/hiclass/calibration/BinaryCalibrator.py b/hiclass/calibration/BinaryCalibrator.py
new file mode 100644
index 00000000..97ec4965
--- /dev/null
+++ b/hiclass/calibration/BinaryCalibrator.py
@@ -0,0 +1,11 @@
+import abc
+
+class BinaryCalibrator(abc.ABC):
+    
+    @abc.abstractmethod
+    def fit(self, y, scores, X=None): #pragma: no cover
+        ...
+
+    @abc.abstractmethod
+    def predict_proba(self, scores, X=None): #pragma: no cover
+        ...
diff --git a/hiclass/calibration/Calibrator.py b/hiclass/calibration/Calibrator.py
new file mode 100644
index 00000000..a14c2078
--- /dev/null
+++ b/hiclass/calibration/Calibrator.py
@@ -0,0 +1,83 @@
+import numpy as np
+from sklearn.base import BaseEstimator
+from sklearn.preprocessing import LabelBinarizer
+from sklearn.preprocessing import LabelEncoder
+from calibration.VennAbersCalibrator import InductiveVennAbersCalibrator, CrossVennAbersCalibrator
+from calibration.IsotonicRegression import IsotonicRegression
+from calibration.PlattScaling import PlattScaling
+
+class _Calibrator(BaseEstimator):
+    available_methods = ["ivap", "cvap", "sigmoid", "isotonic"]
+
+    def __init__(self, estimator, method="ivap", **method_params) -> None:
+        assert callable(getattr(estimator, 'predict_proba', None))
+        self.estimator = estimator
+        self.method_params = method_params
+        self.classes_ = self.estimator.classes_
+        self.multiclass = False
+        if method not in self.available_methods:
+            raise ValueError(f"{method} is not a valid calibration method.")
+        self.method = method
+    
+    def fit(self, X, y):
+        '''
+        X = all datapoints if method == cvap and X_cal else
+        y = all labels if method == cvap and y_cal else
+        
+        '''
+        calibration_scores = self.estimator.predict_proba(X)
+
+        if calibration_scores.shape[1] > 2:
+            self.multiclass = True
+        
+        self.calibrators = []
+
+        if self.multiclass:
+            # binarize multiclass labels
+            label_binarizer = LabelBinarizer(sparse_output=False)
+            binary_labels = label_binarizer.fit_transform(y).T
+
+            # split scores into k one vs rest splits
+            score_splits = [calibration_scores[:, i] for i in range(calibration_scores.shape[1])]
+
+            for idx, split in enumerate(score_splits):
+                # create a calibrator for each step
+                calibrator = self._create_calibrator(self.method, self.method_params)
+                calibrator.fit(binary_labels[idx], split, X)
+                self.calibrators.append(calibrator)
+
+        else:
+            self.label_encoder = LabelEncoder()
+            encoded_y = self.label_encoder.fit_transform(y)
+            calibrator = self._create_calibrator(self.method, self.method_params)
+            calibrator.fit(encoded_y, calibration_scores[:, 1], X)
+            self.calibrators.append(calibrator)
+
+    def predict_proba(self, X):
+        test_scores = self.estimator.predict_proba(X)
+
+        if self.multiclass:
+            score_splits = [test_scores[:, i] for i in range(test_scores.shape[1])]
+
+            probabilities = np.zeros((len(X), len(self.estimator.classes_)))
+            for idx, split in enumerate(score_splits):
+                probabilities[:, idx] = self.calibrators[idx].predict_proba(split)
+        
+            probabilities /= probabilities.sum(axis=1, keepdims=True)
+        
+        else:
+            probabilities = np.zeros((len(X), 2))
+            probabilities[:, 1] = self.calibrators[0].predict_proba(test_scores[:, 1])
+            probabilities[:, 0] = 1.0 - probabilities[:, 1]
+
+        return probabilities
+
+    def _create_calibrator(self, name, params):
+        if name == "ivap":
+            return InductiveVennAbersCalibrator(**params)
+        elif name == "cvap":
+            return CrossVennAbersCalibrator(self.estimator, **params)
+        elif name == "sigmoid":
+            return PlattScaling()
+        elif name == "isotonic":
+            return IsotonicRegression(params)
\ No newline at end of file
diff --git a/hiclass/calibration/IsotonicRegression.py b/hiclass/calibration/IsotonicRegression.py
new file mode 100644
index 00000000..6a3e0fac
--- /dev/null
+++ b/hiclass/calibration/IsotonicRegression.py
@@ -0,0 +1,24 @@
+from calibration.BinaryCalibrator import BinaryCalibrator
+from sklearn.isotonic import IsotonicRegression as SkLearnIR
+from sklearn.exceptions import NotFittedError
+
+
+
+class IsotonicRegression(BinaryCalibrator):
+    name = "IsotonicRegression"
+
+    def __init__(self, params={}) -> None:
+        self.fitted = False
+        if "out_of_bounds" not in params:
+            params["out_of_bounds"] = "clip"
+        self.isotonic_regression = SkLearnIR(**params)
+
+    def fit(self, y, scores, X=None):
+        self.isotonic_regression.fit(scores, y)
+        self.fitted = True
+        return self
+
+    def predict_proba(self, scores, X=None):
+        if not self.fitted:
+            raise NotFittedError(f"This {self.name} calibrator is not fitted yet. Call 'fit' with appropriate arguments before using this calibrator.")
+        return self.isotonic_regression.predict(scores)
diff --git a/hiclass/calibration/PlattScaling.py b/hiclass/calibration/PlattScaling.py
new file mode 100644
index 00000000..f06ab625
--- /dev/null
+++ b/hiclass/calibration/PlattScaling.py
@@ -0,0 +1,21 @@
+from calibration.BinaryCalibrator import BinaryCalibrator
+from sklearn.calibration import _SigmoidCalibration
+from sklearn.exceptions import NotFittedError
+
+class PlattScaling(BinaryCalibrator):
+    name = "PlattScaling"
+
+    def __init__(self) -> None:
+        self.fitted = False
+        self.platt_scaling = _SigmoidCalibration()
+
+    def fit(self, y, scores, X=None):
+        self.platt_scaling.fit(scores, y)
+        self.fitted = True
+        return self
+
+    def predict_proba(self, scores, X=None):
+        if not self.fitted:
+            raise NotFittedError(f"This {self.name} calibrator is not fitted yet. Call 'fit' with appropriate arguments before using this calibrator.")
+        return self.platt_scaling.predict(scores)
+
diff --git a/hiclass/calibration/VennAbersCalibrator.py b/hiclass/calibration/VennAbersCalibrator.py
new file mode 100644
index 00000000..383deecb
--- /dev/null
+++ b/hiclass/calibration/VennAbersCalibrator.py
@@ -0,0 +1,255 @@
+import numpy as np
+from sklearn.exceptions import NotFittedError
+from calibration.BinaryCalibrator import BinaryCalibrator
+from scipy.stats import gmean
+
+
+class InductiveVennAbersCalibrator(BinaryCalibrator):
+    name = "InductiveVennAbersCalibrator"
+
+    def __init__(self):
+        self.fitted = False
+
+    def fit(self, y, scores, X=None):
+        positive_label = 1
+        unique_labels = np.unique(y)
+        assert len(unique_labels) <= 2
+
+        y = np.where(y == positive_label, 1, 0)
+        y = y.reshape(-1) # make sure it's a 1D array
+        # sort all scores s1, ..., sk in increasing order
+
+        order_idx = np.lexsort([y, scores])
+        ordered_calibration_scores, ordered_calibration_labels = scores[order_idx], y[order_idx]
+        # remove duplicates
+        unique_elements, unique_idx, unique_element_counts = np.unique(ordered_calibration_scores, return_index=True, return_counts=True)
+        #self.unique_elements = unique_elements
+        ordered_unique_calibration_scores, ordered_unique_calibration_labels = ordered_calibration_scores[unique_idx], ordered_calibration_labels[unique_idx]
+
+        self.k_distinct = len(unique_idx)
+
+        def compute_csd(un_el, un_el_counts, ocs, ocl, oucs):
+        
+            # Count the frequencies of each s'j
+            w = dict(zip(un_el, un_el_counts))
+        
+            y = np.zeros(self.k_distinct)
+            csd = np.zeros((self.k_distinct + 1, 2))
+        
+            for j in range(self.k_distinct):
+                s_j = oucs[j]
+                matching_idx = np.where(ocs == s_j)
+                matching_labels = ocl[matching_idx]
+                y[j] = np.sum(matching_labels) / w[un_el[j]]
+        
+            csd[1:, 0] = np.cumsum(un_el_counts)
+            csd[1:, 1] = np.cumsum(y * un_el_counts)
+
+            return list(csd)
+
+
+        def slope(top, next_to_top):
+                return (next_to_top[1] - top[1]) / (next_to_top[0] - top[0])
+            
+        def at_or_above(p, cur_slope, top, next_to_top):
+                intersection_point = (p[0], top[1] + cur_slope * (p[0] - top[0]))
+                return p[1] >= intersection_point[1]
+
+        def non_left_angle_turn(next_to_top, top, p_i):
+            next_to_top = np.array(next_to_top)
+            top = np.array(top)
+            p_i = np.array(p_i)
+            res = np.cross((top - next_to_top), (p_i - top))
+            return res <= 0
+            
+        def non_right_angle_turn(next_to_top, top, p_i):
+            next_to_top = np.array(next_to_top)
+            top = np.array(top)
+            p_i = np.array(p_i)
+            res = np.cross((top - next_to_top), (p_i - top))
+            return res >= 0
+
+        def initialize_f1_corners(csd):
+            stack = []
+            # append P_{-1} and P_0
+            stack.append(csd[0])
+            stack.append(csd[1])
+
+            for i in range(2, len(csd)):
+                while len(stack) > 1 and non_left_angle_turn(next_to_top=stack[-2], top=stack[-1], p_i=csd[i]):
+                    stack.pop()
+                stack.append(csd[i])
+
+            return stack
+
+        
+        def initialize_f0_corners(csd):
+            stack = []
+            # append p_{k'+1}, p_{k'}
+            stack.append(csd[-1])
+            stack.append(csd[-2])
+
+            for i in range(len(csd)-3, -1, -1):
+                while len(stack) > 1 and non_right_angle_turn(next_to_top=stack[-2], top=stack[-1], p_i=csd[i]):
+                    stack.pop()
+                stack.append(csd[i])
+            return stack
+        
+
+        point_addition = lambda p1, p2: tuple((p1[0] + p2[0], p1[1] + p2[1]))
+        point_subtraction = lambda p1, p2: tuple((p1[0] - p2[0], p1[1] - p2[1]))
+
+        def compute_f1(prev_stack, csd):
+            F1 = np.zeros(self.k_distinct+1)
+            stack = []
+            while prev_stack:
+                stack.append(prev_stack.pop())
+
+            for i in range(2, self.k_distinct+2):
+                F1[i-1] = slope(top=stack[-1], next_to_top=stack[-2]) 
+                # p_{i-1}
+                csd[i-1] = point_subtraction(point_addition(csd[i-2], csd[i]), csd[i-1])
+                p_temp = csd[i-1]
+                
+                if at_or_above(p_temp, F1[i-1], top=stack[-1], next_to_top=stack[-2]):
+                    continue
+                    
+                stack.pop()
+                while len(stack) > 1 and non_left_angle_turn(p_temp, stack[-1], stack[-2]):
+                    stack.pop()
+                stack.append(p_temp)
+            return F1
+
+        
+        def compute_f0(prev_stack, csd):
+            F0 = np.zeros(self.k_distinct+1)
+            stack = []
+            while prev_stack:
+                stack.append(prev_stack.pop())
+
+            for i in range(self.k_distinct, 0, -1):
+                F0[i] = slope(top=stack[-1], next_to_top=stack[-2])
+                csd[i] = point_subtraction(point_addition(csd[i-1], csd[i+1]), csd[i])
+                
+                if at_or_above(csd[i], F0[i], top=stack[-1], next_to_top=stack[-2]):
+                    continue
+                stack.pop()
+                while len(stack) > 1 and non_right_angle_turn(csd[i], stack[-1], stack[-2]):
+                    stack.pop()
+                stack.append(csd[i])
+            return F0
+
+        csd_1 = compute_csd(
+            unique_elements, 
+            unique_element_counts, 
+            ordered_calibration_scores, 
+            ordered_calibration_labels, 
+            ordered_unique_calibration_scores
+        )
+        csd_0 = csd_1.copy()
+        csd_0.append((csd_0[-1][0] + 1, csd_0[-1][1] + 0))
+        
+        csd_1.insert(0, (-1,-1))
+        
+        f1_stack = initialize_f1_corners(csd_1)
+        f0_stack = initialize_f0_corners(csd_0)
+                
+        self.F1 = compute_f1(f1_stack, csd_1)
+        self.F0 = compute_f0(f0_stack, csd_0)
+        self.unique_elements = unique_elements
+        self.fitted = True
+
+        return self
+
+
+    def predict_proba(self, scores, X=None):
+        if not self.fitted:
+            raise NotFittedError(f"This {self.name} calibrator is not fitted yet. Call 'fit' with appropriate arguments before using this calibrator.")
+        lower = np.searchsorted(self.unique_elements, scores, side="left")
+        upper = np.searchsorted(self.unique_elements[:-1], scores, side="right")+1
+
+        p0 = self.F0[lower]
+        p1 = self.F1[upper]
+
+        return p1 / (1 - p0 + p1)
+    
+    def predict_intervall(self, scores):
+        lower = np.searchsorted(self.unique_elements, scores, side="left")
+        upper = np.searchsorted(self.unique_elements[:-1], scores, side="right")+1
+        p0 = self.F0[lower]
+        p1 = self.F1[upper]
+
+        return np.array(list(zip(p0, p1)))
+
+    '''
+    def predict(self, X):
+        pred_scores = self.predict_proba(X)
+        return self.classes_[np.argmax(pred_scores, axis=1)]
+    '''
+
+class CrossVennAbersCalibrator(BinaryCalibrator):
+    name = "CrossVennAbersCalibrator"
+
+    def __init__(self, estimator, n_folds=5) -> None:
+        self.fitted = False
+        self.n_folds = n_folds
+        self.estimator_type = type(estimator)
+        self.estimator_params = estimator.get_params()
+
+    def fit(self, y, scores, X):
+        #positive_label = 1
+        unique_labels = np.unique(y)
+        assert len(unique_labels) == 2
+        #self.negative_label = unique_labels[unique_labels != positive_label][0]
+
+        # split training set X into self.n_folds folds
+        splits_x = np.array_split(X, self.n_folds, axis=0)
+        splits_y = np.array_split(y, self.n_folds, axis=0)
+
+        # create random permutation
+        perm = np.random.default_rng().permutation(self.n_folds)
+        splits_x = [splits_x[i] for i in perm]
+        splits_y = [splits_y[i] for i in perm]
+        self.ivaps = []
+
+        for i in range(self.n_folds):
+            X_train = np.concatenate(splits_x[:i] + splits_x[i+1:], axis=0)
+            y_train = np.concatenate(splits_y[:i] + splits_y[i+1:], axis=0)
+            X_cal = splits_x[i]
+            y_cal = splits_y[i]
+
+            # train underlying model with x_train and y_train
+            model = self.estimator_type()
+            model.set_params(**self.estimator_params)
+            model.fit(X_train, y_train)
+
+            # calibrate IVAP with left out dataset
+            calibration_scores = model.predict_proba(X_cal)
+
+            calibrator = InductiveVennAbersCalibrator()
+            calibrator.fit(y_cal, calibration_scores[:, 1])
+            self.ivaps.append(calibrator)
+        self.fitted = True
+
+        return self
+
+
+    def predict_proba(self, scores):
+        if not self.fitted:
+            raise NotFittedError(f"This {self.name} calibrator is not fitted yet. Call 'fit' with appropriate arguments before using this calibrator.")
+        res = []
+        for calibrator in self.ivaps:
+            res.append(calibrator.predict_intervall(scores))
+        
+        res = np.array(res)
+        p0 = res[:, :, 0]
+        p1 = res[:, :, 1]
+
+        p1_gm = gmean(p1)
+        return p1_gm / (gmean(1 - p0) + p1_gm)
+
+    '''
+    def predict(self, X):
+        pred_scores = self.predict_proba(X)
+        return self.classes_[np.argmax(pred_scores, axis=1)]
+    '''
diff --git a/hiclass/calibration/__init__.py b/hiclass/calibration/__init__.py
new file mode 100644
index 00000000..e69de29b

From eb9b75353d350777e5eb90707e8420887270f3f5 Mon Sep 17 00:00:00 2001
From: LukasDrews97 <drews.lukas@yahoo.com>
Date: Thu, 18 Jan 2024 18:19:49 +0100
Subject: [PATCH 04/65] update

---
 hiclass/HierarchicalClassifier.py          | 15 ++++++++++++++-
 hiclass/calibration/Calibrator.py          |  6 +++---
 hiclass/calibration/IsotonicRegression.py  |  2 +-
 hiclass/calibration/PlattScaling.py        |  2 +-
 hiclass/calibration/VennAbersCalibrator.py |  2 +-
 5 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index 153d5316..75889196 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -8,7 +8,7 @@
 from sklearn.base import BaseEstimator
 from sklearn.linear_model import LogisticRegression
 from sklearn.utils.validation import _check_sample_weight
-from calibration.Calibrator import _Calibrator
+from hiclass.calibration.Calibrator import _Calibrator
 
 try:
     import ray
@@ -146,6 +146,13 @@ def _pre_fit(self, X, y, sample_weight):
         else:
             self.X_ = np.array(X)
             self.y_ = np.array(y)
+        
+        if self.calibration_method:
+            n_train_samples = int(0.7 * self.X_.shape[0])
+            print(self.X_.shape)
+            print(self.X_[:10])
+            self.X_, self.X_cal_ = np.split(self.X_, [n_train_samples], axis=0)
+            self.y_, self.y_cal = np.split(self.y_, [n_train_samples], axis=0)
 
         if sample_weight is not None:
             self.sample_weight_ = _check_sample_weight(sample_weight, X)
@@ -353,6 +360,12 @@ def _fit_node_classifier(
         for classifier, node in zip(classifiers, nodes):
             self.hierarchy_.nodes[node]["classifier"] = classifier
 
+    def _fit_node_calibrator():
+        pass
+
+    def _create_train_calibration_split(X, y, sample_weight):
+        pass
+
     @staticmethod
     def _fit_classifier(self, node):
         raise NotImplementedError("Method should be implemented in the LCPN and LCPPN")
diff --git a/hiclass/calibration/Calibrator.py b/hiclass/calibration/Calibrator.py
index a14c2078..bd92f89f 100644
--- a/hiclass/calibration/Calibrator.py
+++ b/hiclass/calibration/Calibrator.py
@@ -2,9 +2,9 @@
 from sklearn.base import BaseEstimator
 from sklearn.preprocessing import LabelBinarizer
 from sklearn.preprocessing import LabelEncoder
-from calibration.VennAbersCalibrator import InductiveVennAbersCalibrator, CrossVennAbersCalibrator
-from calibration.IsotonicRegression import IsotonicRegression
-from calibration.PlattScaling import PlattScaling
+from hiclass.calibration.VennAbersCalibrator import InductiveVennAbersCalibrator, CrossVennAbersCalibrator
+from hiclass.calibration.IsotonicRegression import IsotonicRegression
+from hiclass.calibration.PlattScaling import PlattScaling
 
 class _Calibrator(BaseEstimator):
     available_methods = ["ivap", "cvap", "sigmoid", "isotonic"]
diff --git a/hiclass/calibration/IsotonicRegression.py b/hiclass/calibration/IsotonicRegression.py
index 6a3e0fac..b50bd2a8 100644
--- a/hiclass/calibration/IsotonicRegression.py
+++ b/hiclass/calibration/IsotonicRegression.py
@@ -1,4 +1,4 @@
-from calibration.BinaryCalibrator import BinaryCalibrator
+from hiclass.calibration.BinaryCalibrator import BinaryCalibrator
 from sklearn.isotonic import IsotonicRegression as SkLearnIR
 from sklearn.exceptions import NotFittedError
 
diff --git a/hiclass/calibration/PlattScaling.py b/hiclass/calibration/PlattScaling.py
index f06ab625..7f7fc579 100644
--- a/hiclass/calibration/PlattScaling.py
+++ b/hiclass/calibration/PlattScaling.py
@@ -1,4 +1,4 @@
-from calibration.BinaryCalibrator import BinaryCalibrator
+from hiclass.calibration.BinaryCalibrator import BinaryCalibrator
 from sklearn.calibration import _SigmoidCalibration
 from sklearn.exceptions import NotFittedError
 
diff --git a/hiclass/calibration/VennAbersCalibrator.py b/hiclass/calibration/VennAbersCalibrator.py
index 383deecb..b1d91c1a 100644
--- a/hiclass/calibration/VennAbersCalibrator.py
+++ b/hiclass/calibration/VennAbersCalibrator.py
@@ -1,6 +1,6 @@
 import numpy as np
 from sklearn.exceptions import NotFittedError
-from calibration.BinaryCalibrator import BinaryCalibrator
+from hiclass.calibration.BinaryCalibrator import BinaryCalibrator
 from scipy.stats import gmean
 
 

From ded07f15fd1fe936e12ada5a344fc7a12ec1fb5d Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Tue, 23 Jan 2024 02:53:52 +0100
Subject: [PATCH 05/65] update

---
 hiclass/HierarchicalClassifier.py |  48 ++++++++----
 hiclass/LocalClassifierPerNode.py | 123 ++++++++++++++++++++++++++----
 hiclass/calibration/Calibrator.py |   4 +-
 3 files changed, 144 insertions(+), 31 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index 75889196..49e51e58 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -8,7 +8,7 @@
 from sklearn.base import BaseEstimator
 from sklearn.linear_model import LogisticRegression
 from sklearn.utils.validation import _check_sample_weight
-from hiclass.calibration.Calibrator import _Calibrator
+from sklearn.utils.validation import check_array, check_is_fitted
 
 try:
     import ray
@@ -146,13 +146,6 @@ def _pre_fit(self, X, y, sample_weight):
         else:
             self.X_ = np.array(X)
             self.y_ = np.array(y)
-        
-        if self.calibration_method:
-            n_train_samples = int(0.7 * self.X_.shape[0])
-            print(self.X_.shape)
-            print(self.X_[:10])
-            self.X_, self.X_cal_ = np.split(self.X_, [n_train_samples], axis=0)
-            self.y_, self.y_cal = np.split(self.y_, [n_train_samples], axis=0)
 
         if sample_weight is not None:
             self.sample_weight_ = _check_sample_weight(sample_weight, X)
@@ -188,11 +181,29 @@ def _pre_fit(self, X, y, sample_weight):
         self._initialize_local_classifiers()
 
     def _calibrate(self, X, y):
+        # check if fitted
+        check_is_fitted(self)
+
+        # Input validation
+        if not self.bert:
+            X = check_array(X, accept_sparse="csr", allow_nd=True, ensure_2d=False)
+        else:
+            X = np.array(X)
+
+        self.X_cal = X
+        self.y_cal = y
+
+        self.cal_binary_policy_ = self._initialize_binary_policy(calibration=True)
+        self.logger_.info("Calibrating")
+
+        #Create a calibrator for each local classifier
+        self._initialize_local_calibrators()
+        self._calibrate_digraph()
+
         # Create calibrator object
         # seed
         # fit calibrator using calibration data
         # predict_proba can then use the calibrator to predict calibrated probabilities
-        pass
 
     def predict_ood():
         pass
@@ -322,6 +333,10 @@ def _initialize_local_classifiers(self):
             self.local_classifier_ = LogisticRegression()
         else:
             self.local_classifier_ = self.local_classifier
+    
+        @abc.abstractmethod
+        def _initialize_local_calibrators(self):
+            raise NotImplementedError("Method should be implemented in the LCPN and LCPPN")
 
     def _convert_to_1d(self, y):
         # Convert predictions to 1D if there is only 1 column
@@ -360,15 +375,22 @@ def _fit_node_classifier(
         for classifier, node in zip(classifiers, nodes):
             self.hierarchy_.nodes[node]["classifier"] = classifier
 
-    def _fit_node_calibrator():
-        pass
+    def _fit_node_calibrator(self, nodes, local_mode: bool = False, use_joblib: bool = False
+    ):
+        # TODO: add support for multithreading
+        calibrators = [self._fit_calibrator(self, node) for node in nodes]
+        for calibrator, node in zip(calibrators, nodes):
+            self.hierarchy_.nodes[node]["calibrator"] = calibrator
 
-    def _create_train_calibration_split(X, y, sample_weight):
-        pass
 
     @staticmethod
     def _fit_classifier(self, node):
         raise NotImplementedError("Method should be implemented in the LCPN and LCPPN")
+    
+    @staticmethod
+    def _fit_calibrator(self, node):
+        raise NotImplementedError("Method should be implemented in the LCPN and LCPPN")
+
 
     def _clean_up(self):
         self.logger_.info("Cleaning up variables that can take a lot of disk space")
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index fafdbb05..a73a6582 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -13,6 +13,8 @@
 from hiclass import BinaryPolicy
 from hiclass.ConstantClassifier import ConstantClassifier
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
+from hiclass.calibration.Calibrator import _Calibrator
+
 
 
 class LocalClassifierPerNode(BaseEstimator, HierarchicalClassifier):
@@ -93,7 +95,7 @@ def __init__(
         )
         self.binary_policy = binary_policy
 
-    def fit(self, X, y, sample_weight=None):
+    def fit(self, X, y, sample_weight=None, calibration_step=False):
         """
         Fit a local classifier per node.
 
@@ -114,22 +116,25 @@ def fit(self, X, y, sample_weight=None):
         self : object
             Fitted estimator.
         """
-        # Execute common methods necessary before fitting
-        super()._pre_fit(X, y, sample_weight)
+        if calibration_step:
+            super()._calibrate(X, y)
+        else:
+            # Execute common methods necessary before fitting
+            super()._pre_fit(X, y, sample_weight)
 
-        # Initialize policy
-        self._initialize_binary_policy()
+            # Initialize policy
+            self.binary_policy_ = self._initialize_binary_policy(calibration=False)
 
-        # Fit local classifiers in DAG
-        super().fit(X, y)
+            # Fit local classifiers in DAG
+            super().fit(X, y)
 
-        # TODO: Store the classes seen during fit
+            # TODO: Store the classes seen during fit
 
-        # TODO: Add function to allow user to change local classifier
+            # TODO: Add function to allow user to change local classifier
 
-        # TODO: Add parameter to receive hierarchy as parameter in constructor
+            # TODO: Add parameter to receive hierarchy as parameter in constructor
 
-        # Return the classifier
+            # Return the classifier
         return self
 
     def predict(self, X):
@@ -199,14 +204,63 @@ def predict(self, X):
         self._remove_separator(y)
 
         return y
+    
+    def predict_proba(self, X):
+        # Check if fit has been called
+        check_is_fitted(self)
 
-    def _initialize_binary_policy(self):
+        # Input validation
+        if not self.bert:
+            X = check_array(X, accept_sparse="csr", allow_nd=True, ensure_2d=False)
+        else:
+            X = np.array(X)
+        
+        bfs = nx.bfs_successors(self.hierarchy_, source=self.root_)
+        self.logger_.info("Predicting Probability")
+
+        y = np.empty((X.shape[0], self.max_levels_), dtype=self.dtype_)
+
+        for predecessor, successors in bfs:
+            self.logger_.info(f"pre: {predecessor}")
+            self.logger_.info(f"suc: {successors}")
+            self.logger_.info(f"")
+
+            '''
+            if predecessor == self.root_:
+                mask = [True] * X.shape[0]
+                subset_x = X[mask]
+            else:
+                mask = np.isin(y, predecessor).any(axis=1)
+                subset_x = X[mask]
+
+            for i, successor in enumerate(successors):
+                successor_name = str(successor).split(self.separator_)[-1]
+                self.logger_.info(f"Predicting proba for node '{successor_name}'")
+                calibrator = self.hierarchy_.nodes[successor]["calibrator"]
+
+                probs = calibrator.predict_proba(subset_x)
+                label = np.argmax(probs, axis=1)
+                print(label)
+            '''
+
+                
+
+
+
+        
+
+    def _initialize_binary_policy(self, calibration=False):
         if isinstance(self.binary_policy, str):
             self.logger_.info(f"Initializing {self.binary_policy} binary policy")
             try:
-                self.binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
+                if calibration:
+                    X, y, sample_weight = self.X_cal, self.y_cal, None
+                else:
+                    X, y, sample_weight = self.X_, self.y_, self.sample_weight_
+                binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
                     self.binary_policy.lower()
-                ](self.hierarchy_, self.X_, self.y_, self.sample_weight_)
+                ](self.hierarchy_, X, y, sample_weight)
+                return binary_policy_
             except KeyError:
                 self.logger_.error(
                     f"Policy {self.binary_policy} not implemented. Available policies are:\n"
@@ -218,6 +272,7 @@ def _initialize_binary_policy(self):
             raise ValueError(
                 f"Binary policy type must str, not {type(self.binary_policy)}."
             )
+            
 
     def _initialize_local_classifiers(self):
         super()._initialize_local_classifiers()
@@ -229,6 +284,18 @@ def _initialize_local_classifiers(self):
                     "classifier": deepcopy(self.local_classifier_)
                 }
         nx.set_node_attributes(self.hierarchy_, local_classifiers)
+    
+    def _initialize_local_calibrators(self):
+        local_calibrators = {}
+        for node in self.hierarchy_.nodes:
+            # Skip only root node
+            if node != self.root_:
+                # get classifier from node
+                local_classifier = self.hierarchy_.nodes[node]["classifier"]
+                local_calibrators[node] = {
+                    "calibrator": _Calibrator(estimator=local_classifier, method=self.calibration_method)
+                    }
+        nx.set_node_attributes(self.hierarchy_, local_calibrators)   
 
     def _fit_digraph(self, local_mode: bool = False, use_joblib: bool = False):
         self.logger_.info("Fitting local classifiers")
@@ -237,10 +304,17 @@ def _fit_digraph(self, local_mode: bool = False, use_joblib: bool = False):
         nodes.remove(self.root_)
         self._fit_node_classifier(nodes, local_mode, use_joblib)
 
+    def _calibrate_digraph(self, local_mode: bool = False, use_joblib: bool = False):
+        nodes = list(self.hierarchy_.nodes)
+        # TODO: only valid for LocalClassifierPerNode, move to subclass!
+        nodes.remove(self.root_)
+        self._fit_node_calibrator(nodes, local_mode, use_joblib)
+
     @staticmethod
     def _fit_classifier(self, node):
         classifier = self.hierarchy_.nodes[node]["classifier"]
         X, y, sample_weight = self.binary_policy_.get_binary_examples(node)
+        self.logger_.info("fitting model " + "node: " + str(node) + " " + str(X.shape) + " : " + str(y.shape))
         unique_y = np.unique(y)
         if len(unique_y) == 1 and self.replace_classifiers:
             classifier = ConstantClassifier()
@@ -252,7 +326,24 @@ def _fit_classifier(self, node):
         else:
             classifier.fit(X, y)
         return classifier
+    
+    @staticmethod
+    def _fit_calibrator(self, node):
+        # TODO: use binary policy
+        try:
+            calibrator = self.hierarchy_.nodes[node]["calibrator"]
+        except KeyError:
+            self.logger_.info("no calibrator for " + "node: " + str(node))
+        X, y, sample_weight = self.cal_binary_policy_.get_binary_examples(node)
+        self.logger_.info("fitting calibrator " + "node: " + str(node) + " " + str(X.shape) + " : " + str(y.shape))
+        #TODO
+        unique_y = np.unique(y)
+        if len(unique_y) == 1 and self.replace_classifiers:
+            calibrator = ConstantClassifier()
+        calibrator.fit(X, y)
+        return calibrator
 
     def _clean_up(self):
-        super()._clean_up()
-        del self.binary_policy_
+        #super()._clean_up()
+        #del self.binary_policy_
+        pass
diff --git a/hiclass/calibration/Calibrator.py b/hiclass/calibration/Calibrator.py
index bd92f89f..6993104a 100644
--- a/hiclass/calibration/Calibrator.py
+++ b/hiclass/calibration/Calibrator.py
@@ -59,14 +59,14 @@ def predict_proba(self, X):
         if self.multiclass:
             score_splits = [test_scores[:, i] for i in range(test_scores.shape[1])]
 
-            probabilities = np.zeros((len(X), len(self.estimator.classes_)))
+            probabilities = np.zeros((X.shape[0], len(self.estimator.classes_)))
             for idx, split in enumerate(score_splits):
                 probabilities[:, idx] = self.calibrators[idx].predict_proba(split)
         
             probabilities /= probabilities.sum(axis=1, keepdims=True)
         
         else:
-            probabilities = np.zeros((len(X), 2))
+            probabilities = np.zeros((X.shape[0], 2))
             probabilities[:, 1] = self.calibrators[0].predict_proba(test_scores[:, 1])
             probabilities[:, 0] = 1.0 - probabilities[:, 1]
 

From c85c0f86711b5c8913a5654eff39c6d9ac943b9b Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Fri, 9 Feb 2024 11:11:37 +0100
Subject: [PATCH 06/65] add predict_proba() method for LocalClassifierPerNode

---
 hiclass/HierarchicalClassifier.py |  44 ++++++++-----
 hiclass/LocalClassifierPerNode.py | 104 +++++++++++++++++++-----------
 2 files changed, 95 insertions(+), 53 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index 49e51e58..5df55d81 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -151,6 +151,11 @@ def _pre_fit(self, X, y, sample_weight):
             self.sample_weight_ = _check_sample_weight(sample_weight, X)
         else:
             self.sample_weight_ = None
+        
+        self.max_level_dimensions = np.array([len(np.unique(self.y_[:, level])) for level in range(self.y_.shape[1])])
+
+        self.classes_ = [np.unique(self.y_[:, level]) for level in range(self.y_.shape[1])]
+        self.class_to_index_mapping = [{self.classes_[level][index]: index for index in range(len(self.classes_[level]))} for level in range(self.y_.shape[1])]
 
         self.y_ = make_leveled(self.y_)
 
@@ -159,7 +164,7 @@ def _pre_fit(self, X, y, sample_weight):
 
         # Avoids creating more columns in prediction if edges are a->b and b->c,
         # which would generate the prediction a->b->c
-        self._disambiguate()
+        self.y_ = self._disambiguate(self.y_)
 
         # Create DAG from self.y_ and store to self.hierarchy_
         self._create_digraph()
@@ -172,7 +177,7 @@ def _pre_fit(self, X, y, sample_weight):
         self._assert_digraph_is_dag()
 
         # If y is 1D, convert to 2D for binary policies
-        self._convert_1d_y_to_2d()
+        self.y_ = self._convert_1d_y_to_2d(self.y_)
 
         # Detect root(s) and add artificial root to DAG
         self._add_artificial_root()
@@ -181,6 +186,10 @@ def _pre_fit(self, X, y, sample_weight):
         self._initialize_local_classifiers()
 
     def _calibrate(self, X, y):
+
+        if not self.calibration_method:
+            raise ValueError("No calibration method specified")
+
         # check if fitted
         check_is_fitted(self)
 
@@ -193,6 +202,10 @@ def _calibrate(self, X, y):
         self.X_cal = X
         self.y_cal = y
 
+        self.y_cal = make_leveled(self.y_cal)
+        self.y_cal = self._disambiguate(self.y_cal)
+        self.y_cal = self._convert_1d_y_to_2d(self.y_cal)
+
         self.cal_binary_policy_ = self._initialize_binary_policy(calibration=True)
         self.logger_.info("Calibrating")
 
@@ -200,11 +213,6 @@ def _calibrate(self, X, y):
         self._initialize_local_calibrators()
         self._calibrate_digraph()
 
-        # Create calibrator object
-        # seed
-        # fit calibrator using calibration data
-        # predict_proba can then use the calibrator to predict calibrated probabilities
-
     def predict_ood():
         pass
 
@@ -229,18 +237,19 @@ def _create_logger(self):
             # Add ch to logger
             self.logger_.addHandler(ch)
 
-    def _disambiguate(self):
+    def _disambiguate(self, y):
         self.separator_ = "::HiClass::Separator::"
-        if self.y_.ndim == 2:
+        if y.ndim == 2:
             new_y = []
-            for i in range(self.y_.shape[0]):
-                row = [str(self.y_[i, 0])]
-                for j in range(1, self.y_.shape[1]):
+            for i in range(y.shape[0]):
+                row = [str(y[i, 0])]
+                for j in range(1, y.shape[1]):
                     parent = str(row[-1])
-                    child = str(self.y_[i, j])
+                    child = str(y[i, j])
                     row.append(parent + self.separator_ + child)
                 new_y.append(np.asarray(row, dtype=np.str_))
-            self.y_ = np.array(new_y)
+            return np.array(new_y)
+        return y
 
     def _create_digraph(self):
         # Create DiGraph
@@ -308,10 +317,11 @@ def _assert_digraph_is_dag(self):
             self.logger_.error("Cycle detected in graph")
             raise ValueError("Graph is not directed acyclic")
 
-    def _convert_1d_y_to_2d(self):
+    def _convert_1d_y_to_2d(self, y):
         # This conversion is necessary for the binary policies
-        if self.y_.ndim == 1:
-            self.y_ = np.reshape(self.y_, (-1, 1))
+        #if y.ndim == 1:
+            #self.y_ = np.reshape(self.y_, (-1, 1))
+        return np.reshape(y, (-1, 1)) if y.ndim == 1 else y
 
     def _add_artificial_root(self):
         # Detect root(s)
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index a73a6582..2d40d394 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -46,6 +46,7 @@ def __init__(
         n_jobs: int = 1,
         bert: bool = False,
         calibration_method: str = None,
+        return_all_probabilities: bool = False,
     ):
         """
         Initialize a local classifier per node.
@@ -82,6 +83,8 @@ def __init__(
             If True, skip scikit-learn's checks and sample_weight passing for BERT.
         calibration_method : {"ivap", "cvap", "platt", "isotonic"}, str, default=None
             If set, use the desired method to calibrate probabilities returned by predict_proba().
+        return_all_probabilities : bool, default=False
+            If True, return probabilities from all levels. Otherwise, return only probabilities from the last level.
         """
         super().__init__(
             local_classifier=local_classifier,
@@ -94,8 +97,9 @@ def __init__(
             calibration_method=calibration_method,
         )
         self.binary_policy = binary_policy
+        self.return_all_probabilities = return_all_probabilities
 
-    def fit(self, X, y, sample_weight=None, calibration_step=False):
+    def fit(self, X, y, sample_weight=None):
         """
         Fit a local classifier per node.
 
@@ -116,25 +120,22 @@ def fit(self, X, y, sample_weight=None, calibration_step=False):
         self : object
             Fitted estimator.
         """
-        if calibration_step:
-            super()._calibrate(X, y)
-        else:
-            # Execute common methods necessary before fitting
-            super()._pre_fit(X, y, sample_weight)
+        # Execute common methods necessary before fitting
+        super()._pre_fit(X, y, sample_weight)
 
-            # Initialize policy
-            self.binary_policy_ = self._initialize_binary_policy(calibration=False)
+        # Initialize policy
+        self.binary_policy_ = self._initialize_binary_policy(calibration=False)
 
-            # Fit local classifiers in DAG
-            super().fit(X, y)
+        # Fit local classifiers in DAG
+        super().fit(X, y)
 
-            # TODO: Store the classes seen during fit
+        # TODO: Store the classes seen during fit
 
-            # TODO: Add function to allow user to change local classifier
+        # TODO: Add function to allow user to change local classifier
 
-            # TODO: Add parameter to receive hierarchy as parameter in constructor
+        # TODO: Add parameter to receive hierarchy as parameter in constructor
 
-            # Return the classifier
+        # Return the classifier
         return self
 
     def predict(self, X):
@@ -184,6 +185,7 @@ def predict(self, X):
                 for i, successor in enumerate(successors):
                     successor_name = str(successor).split(self.separator_)[-1]
                     self.logger_.info(f"Predicting for node '{successor_name}'")
+                    # TODO: use calibrator if using calibration to predict class
                     classifier = self.hierarchy_.nodes[successor]["classifier"]
                     positive_index = np.where(classifier.classes_ == 1)[0]
                     probabilities[:, i] = classifier.predict_proba(subset_x)[
@@ -215,17 +217,26 @@ def predict_proba(self, X):
         else:
             X = np.array(X)
         
+        if not self.calibration_method:
+            self.logger_.info("It is not recommended to use predict_proba() without calibration")
         bfs = nx.bfs_successors(self.hierarchy_, source=self.root_)
         self.logger_.info("Predicting Probability")
 
         y = np.empty((X.shape[0], self.max_levels_), dtype=self.dtype_)
+        level_probability_list = []
+        last_level = -1
 
         for predecessor, successors in bfs:
-            self.logger_.info(f"pre: {predecessor}")
-            self.logger_.info(f"suc: {successors}")
-            self.logger_.info(f"")
+            level = nx.shortest_path_length(
+                self.hierarchy_, self.root_, predecessor
+            )
+            level_dimension = self.max_level_dimensions[level]
+
+            if last_level != level:
+                last_level = level
+                cur_level_probabilities = np.zeros((X.shape[0], level_dimension))
+                level_probability_list.append(cur_level_probabilities)
 
-            '''
             if predecessor == self.root_:
                 mask = [True] * X.shape[0]
                 subset_x = X[mask]
@@ -233,21 +244,38 @@ def predict_proba(self, X):
                 mask = np.isin(y, predecessor).any(axis=1)
                 subset_x = X[mask]
 
-            for i, successor in enumerate(successors):
-                successor_name = str(successor).split(self.separator_)[-1]
-                self.logger_.info(f"Predicting proba for node '{successor_name}'")
-                calibrator = self.hierarchy_.nodes[successor]["calibrator"]
+            if subset_x.shape[0] > 0:
+                local_probabilities = np.zeros((subset_x.shape[0], len(successors)))
+                for i, successor in enumerate(successors):
+                    successor_name = str(successor).split(self.separator_)[-1]
+                    self.logger_.info(f"Predicting probabilities for node '{successor_name}'")
+                    classifier = self.hierarchy_.nodes[successor]["classifier"]
+                    # use classifier as a fallback if no calibrator is available
+                    calibrator = self.hierarchy_.nodes[successor].get("calibrator", classifier)
+                    positive_index = np.where(calibrator.classes_ == 1)[0]
+                    proba = calibrator.predict_proba(subset_x)[:, positive_index][:, 0]
+                    local_probabilities[:, i] = proba
+                    class_index = self.class_to_index_mapping[level][successor_name]
+                    level_probability_list[-1][mask, class_index] = proba
+
+                highest_local_probability = np.argmax(local_probabilities, axis=1)
+                path_prediction = []
+                for i in highest_local_probability:
+                    path_prediction.append(successors[i])
+                path_prediction = np.array(path_prediction)
+
+                y[mask, level] = path_prediction
 
-                probs = calibrator.predict_proba(subset_x)
-                label = np.argmax(probs, axis=1)
-                print(label)
-            '''
+        y = self._convert_to_1d(y)
+        self._remove_separator(y)
 
-                
+        # normalize probabilities
+        for level_probabilities in level_probability_list:
+            level_probabilities /= level_probabilities.sum(axis=1, keepdims=True)
 
+        return level_probability_list if self.return_all_probabilities else level_probability_list[-1]
 
 
-        
 
     def _initialize_binary_policy(self, calibration=False):
         if isinstance(self.binary_policy, str):
@@ -306,7 +334,7 @@ def _fit_digraph(self, local_mode: bool = False, use_joblib: bool = False):
 
     def _calibrate_digraph(self, local_mode: bool = False, use_joblib: bool = False):
         nodes = list(self.hierarchy_.nodes)
-        # TODO: only valid for LocalClassifierPerNode, move to subclass!
+        # Remove root because it does not need to be fitted
         nodes.remove(self.root_)
         self._fit_node_calibrator(nodes, local_mode, use_joblib)
 
@@ -314,9 +342,10 @@ def _calibrate_digraph(self, local_mode: bool = False, use_joblib: bool = False)
     def _fit_classifier(self, node):
         classifier = self.hierarchy_.nodes[node]["classifier"]
         X, y, sample_weight = self.binary_policy_.get_binary_examples(node)
-        self.logger_.info("fitting model " + "node: " + str(node) + " " + str(X.shape) + " : " + str(y.shape))
+        self.logger_.info("fitting model " + "node: " + str(node) + " " + str(X.shape) + " : " + str(y.shape) + " unique labels: " + str(len(np.unique(y))))
         unique_y = np.unique(y)
         if len(unique_y) == 1 and self.replace_classifiers:
+            self.logger_.info("adding constant classifier")
             classifier = ConstantClassifier()
         if not self.bert:
             try:
@@ -336,14 +365,17 @@ def _fit_calibrator(self, node):
             self.logger_.info("no calibrator for " + "node: " + str(node))
         X, y, sample_weight = self.cal_binary_policy_.get_binary_examples(node)
         self.logger_.info("fitting calibrator " + "node: " + str(node) + " " + str(X.shape) + " : " + str(y.shape))
-        #TODO
+        if len(y) == 0:
+            self.logger_.info(f"No calibration samples to fit calibrator for node: {str(node)}") 
+            return None
         unique_y = np.unique(y)
-        if len(unique_y) == 1 and self.replace_classifiers:
-            calibrator = ConstantClassifier()
+        self.logger_.info(f"unique y: {unique_y}")
+        #if len(unique_y) == 1 and self.replace_classifiers:
+        #    self.logger_.info("adding constant calibrator")
+        #    calibrator = ConstantClassifier()
         calibrator.fit(X, y)
         return calibrator
 
     def _clean_up(self):
-        #super()._clean_up()
-        #del self.binary_policy_
-        pass
+        super()._clean_up()
+        del self.binary_policy_

From 6e69c956e936ea3db09b04a2df811f12a6d95da8 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Fri, 9 Feb 2024 13:54:12 +0100
Subject: [PATCH 07/65] fix tests

---
 hiclass/HierarchicalClassifier.py    | 15 +++++++++------
 hiclass/LocalClassifierPerNode.py    | 20 ++++++++------------
 tests/test_HierarchicalClassifier.py |  6 +++---
 tests/test_LocalClassifierPerNode.py |  7 ++++---
 4 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index 5df55d81..cf0f42d6 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -151,13 +151,17 @@ def _pre_fit(self, X, y, sample_weight):
             self.sample_weight_ = _check_sample_weight(sample_weight, X)
         else:
             self.sample_weight_ = None
-        
-        self.max_level_dimensions = np.array([len(np.unique(self.y_[:, level])) for level in range(self.y_.shape[1])])
-
-        self.classes_ = [np.unique(self.y_[:, level]) for level in range(self.y_.shape[1])]
-        self.class_to_index_mapping = [{self.classes_[level][index]: index for index in range(len(self.classes_[level]))} for level in range(self.y_.shape[1])]
 
         self.y_ = make_leveled(self.y_)
+        
+        if self.y_.ndim > 1:
+            self.max_level_dimensions_ = np.array([len(np.unique(self.y_[:, level])) for level in range(self.y_.shape[1])])
+            self.classes_ = [np.unique(self.y_[:, level]).astype("str") for level in range(self.y_.shape[1])]
+            self.class_to_index_mapping_ = [{self.classes_[level][index]: index for index in range(len(self.classes_[level]))} for level in range(self.y_.shape[1])]
+        else:
+            self.max_level_dimensions_ = np.array([len(np.unique(self.y_))])
+            self.classes_ = [np.unique(self.y_).astype("str")]
+            self.class_to_index_mapping_ = [{self.classes_[0][index] : index for index in range(len(self.classes_[0]))}]
 
         # Create and configure logger
         self._create_logger()
@@ -186,7 +190,6 @@ def _pre_fit(self, X, y, sample_weight):
         self._initialize_local_classifiers()
 
     def _calibrate(self, X, y):
-
         if not self.calibration_method:
             raise ValueError("No calibration method specified")
 
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index 2d40d394..d79f2ded 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -230,7 +230,7 @@ def predict_proba(self, X):
             level = nx.shortest_path_length(
                 self.hierarchy_, self.root_, predecessor
             )
-            level_dimension = self.max_level_dimensions[level]
+            level_dimension = self.max_level_dimensions_[level]
 
             if last_level != level:
                 last_level = level
@@ -255,7 +255,7 @@ def predict_proba(self, X):
                     positive_index = np.where(calibrator.classes_ == 1)[0]
                     proba = calibrator.predict_proba(subset_x)[:, positive_index][:, 0]
                     local_probabilities[:, i] = proba
-                    class_index = self.class_to_index_mapping[level][successor_name]
+                    class_index = self.class_to_index_mapping_[level][successor_name]
                     level_probability_list[-1][mask, class_index] = proba
 
                 highest_local_probability = np.argmax(local_probabilities, axis=1)
@@ -282,12 +282,13 @@ def _initialize_binary_policy(self, calibration=False):
             self.logger_.info(f"Initializing {self.binary_policy} binary policy")
             try:
                 if calibration:
-                    X, y, sample_weight = self.X_cal, self.y_cal, None
-                else:
-                    X, y, sample_weight = self.X_, self.y_, self.sample_weight_
-                binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
+                    binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
                     self.binary_policy.lower()
-                ](self.hierarchy_, X, y, sample_weight)
+                ](self.hierarchy_, self.X_cal, self.y_cal, None)
+                else:
+                    binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
+                        self.binary_policy.lower()
+                    ](self.hierarchy_, self.X_, self.y_, self.sample_weight_)
                 return binary_policy_
             except KeyError:
                 self.logger_.error(
@@ -358,7 +359,6 @@ def _fit_classifier(self, node):
     
     @staticmethod
     def _fit_calibrator(self, node):
-        # TODO: use binary policy
         try:
             calibrator = self.hierarchy_.nodes[node]["calibrator"]
         except KeyError:
@@ -369,10 +369,6 @@ def _fit_calibrator(self, node):
             self.logger_.info(f"No calibration samples to fit calibrator for node: {str(node)}") 
             return None
         unique_y = np.unique(y)
-        self.logger_.info(f"unique y: {unique_y}")
-        #if len(unique_y) == 1 and self.replace_classifiers:
-        #    self.logger_.info("adding constant calibrator")
-        #    calibrator = ConstantClassifier()
         calibrator.fit(X, y)
         return calibrator
 
diff --git a/tests/test_HierarchicalClassifier.py b/tests/test_HierarchicalClassifier.py
index d800ff47..b6876752 100644
--- a/tests/test_HierarchicalClassifier.py
+++ b/tests/test_HierarchicalClassifier.py
@@ -21,7 +21,7 @@ def test_disambiguate_str(ambiguous_node_str):
     ground_truth = np.array(
         [["a", "a::HiClass::Separator::b"], ["b", "b::HiClass::Separator::c"]]
     )
-    ambiguous_node_str._disambiguate()
+    ambiguous_node_str.y_ = ambiguous_node_str._disambiguate(ambiguous_node_str.y_)
     assert_array_equal(ground_truth, ambiguous_node_str.y_)
 
 
@@ -36,7 +36,7 @@ def test_disambiguate_int(ambiguous_node_int):
     ground_truth = np.array(
         [["1", "1::HiClass::Separator::2"], ["2", "2::HiClass::Separator::3"]]
     )
-    ambiguous_node_int._disambiguate()
+    ambiguous_node_int.y_ = ambiguous_node_int._disambiguate(ambiguous_node_int.y_)
     assert_array_equal(ground_truth, ambiguous_node_int.y_)
 
 
@@ -128,7 +128,7 @@ def test_assert_digraph_is_dag(cyclic_graph):
 
 def test_convert_1d_y_to_2d(graph_1d):
     ground_truth = np.array([["a"], ["b"], ["c"], ["d"]])
-    graph_1d._convert_1d_y_to_2d()
+    graph_1d.y_ = graph_1d._convert_1d_y_to_2d(graph_1d.y_)
     assert_array_equal(ground_truth, graph_1d.y_)
 
 
diff --git a/tests/test_LocalClassifierPerNode.py b/tests/test_LocalClassifierPerNode.py
index 670c9823..dadd04eb 100644
--- a/tests/test_LocalClassifierPerNode.py
+++ b/tests/test_LocalClassifierPerNode.py
@@ -23,15 +23,15 @@ def test_sklearn_compatible_estimator(estimator, check):
 def digraph_with_policy():
     digraph = LocalClassifierPerNode(binary_policy="exclusive")
     digraph.hierarchy_ = nx.DiGraph([("a", "b")])
-    digraph.X_ = np.array([1, 2])
-    digraph.y_ = np.array(["a", "b"])
+    digraph.X_ = np.array([[1, 2]])
+    digraph.y_ = np.array([["a", "b"]])
     digraph.logger_ = logging.getLogger("LCPN")
     digraph.sample_weight_ = None
     return digraph
 
 
 def test_initialize_binary_policy(digraph_with_policy):
-    digraph_with_policy._initialize_binary_policy()
+    digraph_with_policy.binary_policy_ = digraph_with_policy._initialize_binary_policy()
     assert isinstance(digraph_with_policy.binary_policy_, ExclusivePolicy)
 
 
@@ -46,6 +46,7 @@ def digraph_with_unknown_policy():
 
 def test_initialize_unknown_binary_policy(digraph_with_unknown_policy):
     with pytest.raises(KeyError):
+        digraph_with_unknown_policy.binary_policy_ = digraph_with_unknown_policy._initialize_binary_policy()
         digraph_with_unknown_policy._initialize_binary_policy()
 
 

From a0a3206a2f8bcffe3759c19bf49b5cf9f5686063 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Fri, 9 Feb 2024 14:10:02 +0100
Subject: [PATCH 08/65] add scipy as dependency

---
 Pipfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Pipfile b/Pipfile
index 401d8f4b..3ad6d898 100644
--- a/Pipfile
+++ b/Pipfile
@@ -7,6 +7,7 @@ name = "pypi"
 networkx = "*"
 numpy = "*"
 scikit-learn = "*"
+scipy = "*"
 
 [dev-packages]
 pytest = "7.1.2"

From 40a4c9e1b96f84324a55bb51a14965a11d78aa52 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Fri, 9 Feb 2024 14:10:57 +0100
Subject: [PATCH 09/65] add scipy as dependency

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index c156f4d8..e759b4f8 100644
--- a/setup.py
+++ b/setup.py
@@ -26,7 +26,7 @@
 KEYWORDS = ["hierarchical classification"]
 DACS_SOFTWARE = "https://gitlab.com/dacs-hpi"
 # What packages are required for this module to be executed?
-REQUIRED = ["networkx", "numpy", "scikit-learn"]
+REQUIRED = ["networkx", "numpy", "scikit-learn", "scipy"]
 
 # What packages are optional?
 # 'fancy feature': ['django'],}

From e522365d813d4e52d3beed234cf14d634e647c72 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Fri, 9 Feb 2024 15:55:30 +0100
Subject: [PATCH 10/65] flake8, pydocstyle

---
 hiclass/HierarchicalClassifier.py             |  38 ++++---
 hiclass/LocalClassifierPerNode.py             |  45 +++++---
 hiclass/_calibration/BinaryCalibrator.py      |  12 +++
 .../Calibrator.py                             |  46 +++++---
 .../IsotonicRegression.py                     |   5 +-
 .../PlattScaling.py                           |   6 +-
 .../VennAbersCalibrator.py                    | 101 +++++++-----------
 .../{calibration => _calibration}/__init__.py |   0
 hiclass/calibration/BinaryCalibrator.py       |  11 --
 9 files changed, 142 insertions(+), 122 deletions(-)
 create mode 100644 hiclass/_calibration/BinaryCalibrator.py
 rename hiclass/{calibration => _calibration}/Calibrator.py (72%)
 rename hiclass/{calibration => _calibration}/IsotonicRegression.py (87%)
 rename hiclass/{calibration => _calibration}/PlattScaling.py (85%)
 rename hiclass/{calibration => _calibration}/VennAbersCalibrator.py (75%)
 rename hiclass/{calibration => _calibration}/__init__.py (100%)
 delete mode 100644 hiclass/calibration/BinaryCalibrator.py

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index cf0f42d6..07147c1e 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -153,7 +153,7 @@ def _pre_fit(self, X, y, sample_weight):
             self.sample_weight_ = None
 
         self.y_ = make_leveled(self.y_)
-        
+
         if self.y_.ndim > 1:
             self.max_level_dimensions_ = np.array([len(np.unique(self.y_[:, level])) for level in range(self.y_.shape[1])])
             self.classes_ = [np.unique(self.y_[:, level]).astype("str") for level in range(self.y_.shape[1])]
@@ -189,7 +189,24 @@ def _pre_fit(self, X, y, sample_weight):
         # Initialize local classifiers in DAG
         self._initialize_local_classifiers()
 
-    def _calibrate(self, X, y):
+    def calibrate(self, X, y):
+        """
+        Fit a local calibrator per node.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            The calibration input samples. Internally, its dtype will be converted
+            to ``dtype=np.float32``. If a sparse matrix is provided, it will be
+            converted into a sparse ``csc_matrix``.
+        y : array-like of shape (n_samples, n_levels)
+            The target values, i.e., hierarchical class labels for classification.
+
+        Returns
+        -------
+        self : object
+            Calibrated estimator.
+        """
         if not self.calibration_method:
             raise ValueError("No calibration method specified")
 
@@ -212,11 +229,12 @@ def _calibrate(self, X, y):
         self.cal_binary_policy_ = self._initialize_binary_policy(calibration=True)
         self.logger_.info("Calibrating")
 
-        #Create a calibrator for each local classifier
+        # Create a calibrator for each local classifier
         self._initialize_local_calibrators()
         self._calibrate_digraph()
+        return self
 
-    def predict_ood():
+    def _predict_ood():
         pass
 
     def _create_logger(self):
@@ -321,9 +339,6 @@ def _assert_digraph_is_dag(self):
             raise ValueError("Graph is not directed acyclic")
 
     def _convert_1d_y_to_2d(self, y):
-        # This conversion is necessary for the binary policies
-        #if y.ndim == 1:
-            #self.y_ = np.reshape(self.y_, (-1, 1))
         return np.reshape(y, (-1, 1)) if y.ndim == 1 else y
 
     def _add_artificial_root(self):
@@ -346,7 +361,7 @@ def _initialize_local_classifiers(self):
             self.local_classifier_ = LogisticRegression()
         else:
             self.local_classifier_ = self.local_classifier
-    
+
         @abc.abstractmethod
         def _initialize_local_calibrators(self):
             raise NotImplementedError("Method should be implemented in the LCPN and LCPPN")
@@ -388,23 +403,20 @@ def _fit_node_classifier(
         for classifier, node in zip(classifiers, nodes):
             self.hierarchy_.nodes[node]["classifier"] = classifier
 
-    def _fit_node_calibrator(self, nodes, local_mode: bool = False, use_joblib: bool = False
-    ):
+    def _fit_node_calibrator(self, nodes, local_mode: bool = False, use_joblib: bool = False):
         # TODO: add support for multithreading
         calibrators = [self._fit_calibrator(self, node) for node in nodes]
         for calibrator, node in zip(calibrators, nodes):
             self.hierarchy_.nodes[node]["calibrator"] = calibrator
 
-
     @staticmethod
     def _fit_classifier(self, node):
         raise NotImplementedError("Method should be implemented in the LCPN and LCPPN")
-    
+
     @staticmethod
     def _fit_calibrator(self, node):
         raise NotImplementedError("Method should be implemented in the LCPN and LCPPN")
 
-
     def _clean_up(self):
         self.logger_.info("Cleaning up variables that can take a lot of disk space")
         del self.X_
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index d79f2ded..6ee9e38e 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -13,8 +13,7 @@
 from hiclass import BinaryPolicy
 from hiclass.ConstantClassifier import ConstantClassifier
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
-from hiclass.calibration.Calibrator import _Calibrator
-
+from hiclass._calibration.Calibrator import _Calibrator
 
 
 class LocalClassifierPerNode(BaseEstimator, HierarchicalClassifier):
@@ -206,8 +205,26 @@ def predict(self, X):
         self._remove_separator(y)
 
         return y
-    
+
     def predict_proba(self, X):
+        """
+        Predict class probabilities for the given data.
+
+        Hierarchical labels are returned.
+        If return_all_probabilities=True: Returns the probabilities for each level.
+        Else: Returns the probabilities for the lowest level.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            The input samples. Internally, its dtype will be converted
+            to ``dtype=np.float32``. If a sparse matrix is provided, it will be
+            converted into a sparse ``csr_matrix``.
+        Returns
+        -------
+        T : ndarray of shape (n_samples,n_classes) or List[ndarray(n_samples,n_classes)]
+            The predicted probabilities of the lowest levels or of all levels.
+        """
         # Check if fit has been called
         check_is_fitted(self)
 
@@ -216,7 +233,7 @@ def predict_proba(self, X):
             X = check_array(X, accept_sparse="csr", allow_nd=True, ensure_2d=False)
         else:
             X = np.array(X)
-        
+
         if not self.calibration_method:
             self.logger_.info("It is not recommended to use predict_proba() without calibration")
         bfs = nx.bfs_successors(self.hierarchy_, source=self.root_)
@@ -275,16 +292,14 @@ def predict_proba(self, X):
 
         return level_probability_list if self.return_all_probabilities else level_probability_list[-1]
 
-
-
     def _initialize_binary_policy(self, calibration=False):
         if isinstance(self.binary_policy, str):
             self.logger_.info(f"Initializing {self.binary_policy} binary policy")
             try:
                 if calibration:
                     binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
-                    self.binary_policy.lower()
-                ](self.hierarchy_, self.X_cal, self.y_cal, None)
+                        self.binary_policy.lower()
+                    ](self.hierarchy_, self.X_cal, self.y_cal, None)
                 else:
                     binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
                         self.binary_policy.lower()
@@ -301,7 +316,6 @@ def _initialize_binary_policy(self, calibration=False):
             raise ValueError(
                 f"Binary policy type must str, not {type(self.binary_policy)}."
             )
-            
 
     def _initialize_local_classifiers(self):
         super()._initialize_local_classifiers()
@@ -313,7 +327,7 @@ def _initialize_local_classifiers(self):
                     "classifier": deepcopy(self.local_classifier_)
                 }
         nx.set_node_attributes(self.hierarchy_, local_classifiers)
-    
+
     def _initialize_local_calibrators(self):
         local_calibrators = {}
         for node in self.hierarchy_.nodes:
@@ -323,8 +337,8 @@ def _initialize_local_calibrators(self):
                 local_classifier = self.hierarchy_.nodes[node]["classifier"]
                 local_calibrators[node] = {
                     "calibrator": _Calibrator(estimator=local_classifier, method=self.calibration_method)
-                    }
-        nx.set_node_attributes(self.hierarchy_, local_calibrators)   
+                }
+        nx.set_node_attributes(self.hierarchy_, local_calibrators)
 
     def _fit_digraph(self, local_mode: bool = False, use_joblib: bool = False):
         self.logger_.info("Fitting local classifiers")
@@ -342,6 +356,8 @@ def _calibrate_digraph(self, local_mode: bool = False, use_joblib: bool = False)
     @staticmethod
     def _fit_classifier(self, node):
         classifier = self.hierarchy_.nodes[node]["classifier"]
+        self.logger_.info("policy: " + str(self.binary_policy_))
+        self.logger_.info("node: " + str(node))
         X, y, sample_weight = self.binary_policy_.get_binary_examples(node)
         self.logger_.info("fitting model " + "node: " + str(node) + " " + str(X.shape) + " : " + str(y.shape) + " unique labels: " + str(len(np.unique(y))))
         unique_y = np.unique(y)
@@ -356,7 +372,7 @@ def _fit_classifier(self, node):
         else:
             classifier.fit(X, y)
         return classifier
-    
+
     @staticmethod
     def _fit_calibrator(self, node):
         try:
@@ -366,9 +382,8 @@ def _fit_calibrator(self, node):
         X, y, sample_weight = self.cal_binary_policy_.get_binary_examples(node)
         self.logger_.info("fitting calibrator " + "node: " + str(node) + " " + str(X.shape) + " : " + str(y.shape))
         if len(y) == 0:
-            self.logger_.info(f"No calibration samples to fit calibrator for node: {str(node)}") 
+            self.logger_.info(f"No calibration samples to fit calibrator for node: {str(node)}")
             return None
-        unique_y = np.unique(y)
         calibrator.fit(X, y)
         return calibrator
 
diff --git a/hiclass/_calibration/BinaryCalibrator.py b/hiclass/_calibration/BinaryCalibrator.py
new file mode 100644
index 00000000..d2fcd015
--- /dev/null
+++ b/hiclass/_calibration/BinaryCalibrator.py
@@ -0,0 +1,12 @@
+import abc
+
+
+class _BinaryCalibrator(abc.ABC):
+
+    @abc.abstractmethod
+    def fit(self, y, scores, X=None):  # pragma: no cover
+        ...
+
+    @abc.abstractmethod
+    def predict_proba(self, scores, X=None):  # pragma: no cover
+        ...
diff --git a/hiclass/calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
similarity index 72%
rename from hiclass/calibration/Calibrator.py
rename to hiclass/_calibration/Calibrator.py
index 6993104a..ad9b87ed 100644
--- a/hiclass/calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -2,9 +2,10 @@
 from sklearn.base import BaseEstimator
 from sklearn.preprocessing import LabelBinarizer
 from sklearn.preprocessing import LabelEncoder
-from hiclass.calibration.VennAbersCalibrator import InductiveVennAbersCalibrator, CrossVennAbersCalibrator
-from hiclass.calibration.IsotonicRegression import IsotonicRegression
-from hiclass.calibration.PlattScaling import PlattScaling
+from hiclass._calibration.VennAbersCalibrator import _InductiveVennAbersCalibrator, _CrossVennAbersCalibrator
+from hiclass._calibration.IsotonicRegression import _IsotonicRegression
+from hiclass._calibration.PlattScaling import _PlattScaling
+
 
 class _Calibrator(BaseEstimator):
     available_methods = ["ivap", "cvap", "sigmoid", "isotonic"]
@@ -18,18 +19,30 @@ def __init__(self, estimator, method="ivap", **method_params) -> None:
         if method not in self.available_methods:
             raise ValueError(f"{method} is not a valid calibration method.")
         self.method = method
-    
+
     def fit(self, X, y):
-        '''
-        X = all datapoints if method == cvap and X_cal else
-        y = all labels if method == cvap and y_cal else
-        
-        '''
+        """
+        Fit a calibrator.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            The calibration input samples. Internally, its dtype will be converted
+            to ``dtype=np.float32``. If a sparse matrix is provided, it will be
+            converted into a sparse ``csc_matrix``.
+        y : array-like of shape (n_samples, n_levels)
+            The target values, i.e., hierarchical class labels for classification.
+
+        Returns
+        -------
+        self : object
+            Calibrated estimator.
+        """
         calibration_scores = self.estimator.predict_proba(X)
 
         if calibration_scores.shape[1] > 2:
             self.multiclass = True
-        
+
         self.calibrators = []
 
         if self.multiclass:
@@ -52,6 +65,7 @@ def fit(self, X, y):
             calibrator = self._create_calibrator(self.method, self.method_params)
             calibrator.fit(encoded_y, calibration_scores[:, 1], X)
             self.calibrators.append(calibrator)
+        return self
 
     def predict_proba(self, X):
         test_scores = self.estimator.predict_proba(X)
@@ -62,9 +76,9 @@ def predict_proba(self, X):
             probabilities = np.zeros((X.shape[0], len(self.estimator.classes_)))
             for idx, split in enumerate(score_splits):
                 probabilities[:, idx] = self.calibrators[idx].predict_proba(split)
-        
+
             probabilities /= probabilities.sum(axis=1, keepdims=True)
-        
+
         else:
             probabilities = np.zeros((X.shape[0], 2))
             probabilities[:, 1] = self.calibrators[0].predict_proba(test_scores[:, 1])
@@ -74,10 +88,10 @@ def predict_proba(self, X):
 
     def _create_calibrator(self, name, params):
         if name == "ivap":
-            return InductiveVennAbersCalibrator(**params)
+            return _InductiveVennAbersCalibrator(**params)
         elif name == "cvap":
-            return CrossVennAbersCalibrator(self.estimator, **params)
+            return _CrossVennAbersCalibrator(self.estimator, **params)
         elif name == "sigmoid":
-            return PlattScaling()
+            return _PlattScaling()
         elif name == "isotonic":
-            return IsotonicRegression(params)
\ No newline at end of file
+            return _IsotonicRegression(params)
diff --git a/hiclass/calibration/IsotonicRegression.py b/hiclass/_calibration/IsotonicRegression.py
similarity index 87%
rename from hiclass/calibration/IsotonicRegression.py
rename to hiclass/_calibration/IsotonicRegression.py
index b50bd2a8..3ec7c0d0 100644
--- a/hiclass/calibration/IsotonicRegression.py
+++ b/hiclass/_calibration/IsotonicRegression.py
@@ -1,10 +1,9 @@
-from hiclass.calibration.BinaryCalibrator import BinaryCalibrator
+from hiclass._calibration.BinaryCalibrator import _BinaryCalibrator
 from sklearn.isotonic import IsotonicRegression as SkLearnIR
 from sklearn.exceptions import NotFittedError
 
 
-
-class IsotonicRegression(BinaryCalibrator):
+class _IsotonicRegression(_BinaryCalibrator):
     name = "IsotonicRegression"
 
     def __init__(self, params={}) -> None:
diff --git a/hiclass/calibration/PlattScaling.py b/hiclass/_calibration/PlattScaling.py
similarity index 85%
rename from hiclass/calibration/PlattScaling.py
rename to hiclass/_calibration/PlattScaling.py
index 7f7fc579..d8a1c0f2 100644
--- a/hiclass/calibration/PlattScaling.py
+++ b/hiclass/_calibration/PlattScaling.py
@@ -1,8 +1,9 @@
-from hiclass.calibration.BinaryCalibrator import BinaryCalibrator
+from hiclass._calibration.BinaryCalibrator import _BinaryCalibrator
 from sklearn.calibration import _SigmoidCalibration
 from sklearn.exceptions import NotFittedError
 
-class PlattScaling(BinaryCalibrator):
+
+class _PlattScaling(_BinaryCalibrator):
     name = "PlattScaling"
 
     def __init__(self) -> None:
@@ -18,4 +19,3 @@ def predict_proba(self, scores, X=None):
         if not self.fitted:
             raise NotFittedError(f"This {self.name} calibrator is not fitted yet. Call 'fit' with appropriate arguments before using this calibrator.")
         return self.platt_scaling.predict(scores)
-
diff --git a/hiclass/calibration/VennAbersCalibrator.py b/hiclass/_calibration/VennAbersCalibrator.py
similarity index 75%
rename from hiclass/calibration/VennAbersCalibrator.py
rename to hiclass/_calibration/VennAbersCalibrator.py
index b1d91c1a..81459caa 100644
--- a/hiclass/calibration/VennAbersCalibrator.py
+++ b/hiclass/_calibration/VennAbersCalibrator.py
@@ -1,10 +1,10 @@
 import numpy as np
 from sklearn.exceptions import NotFittedError
-from hiclass.calibration.BinaryCalibrator import BinaryCalibrator
+from hiclass._calibration.BinaryCalibrator import _BinaryCalibrator
 from scipy.stats import gmean
 
 
-class InductiveVennAbersCalibrator(BinaryCalibrator):
+class _InductiveVennAbersCalibrator(_BinaryCalibrator):
     name = "InductiveVennAbersCalibrator"
 
     def __init__(self):
@@ -16,44 +16,42 @@ def fit(self, y, scores, X=None):
         assert len(unique_labels) <= 2
 
         y = np.where(y == positive_label, 1, 0)
-        y = y.reshape(-1) # make sure it's a 1D array
+        y = y.reshape(-1)  # make sure it's a 1D array
         # sort all scores s1, ..., sk in increasing order
 
         order_idx = np.lexsort([y, scores])
         ordered_calibration_scores, ordered_calibration_labels = scores[order_idx], y[order_idx]
         # remove duplicates
         unique_elements, unique_idx, unique_element_counts = np.unique(ordered_calibration_scores, return_index=True, return_counts=True)
-        #self.unique_elements = unique_elements
-        ordered_unique_calibration_scores, ordered_unique_calibration_labels = ordered_calibration_scores[unique_idx], ordered_calibration_labels[unique_idx]
+        ordered_unique_calibration_scores, _ = ordered_calibration_scores[unique_idx], ordered_calibration_labels[unique_idx]
 
         self.k_distinct = len(unique_idx)
 
         def compute_csd(un_el, un_el_counts, ocs, ocl, oucs):
-        
+
             # Count the frequencies of each s'j
             w = dict(zip(un_el, un_el_counts))
-        
+
             y = np.zeros(self.k_distinct)
             csd = np.zeros((self.k_distinct + 1, 2))
-        
+
             for j in range(self.k_distinct):
                 s_j = oucs[j]
                 matching_idx = np.where(ocs == s_j)
                 matching_labels = ocl[matching_idx]
                 y[j] = np.sum(matching_labels) / w[un_el[j]]
-        
+
             csd[1:, 0] = np.cumsum(un_el_counts)
             csd[1:, 1] = np.cumsum(y * un_el_counts)
 
             return list(csd)
 
-
         def slope(top, next_to_top):
-                return (next_to_top[1] - top[1]) / (next_to_top[0] - top[0])
-            
+            return (next_to_top[1] - top[1]) / (next_to_top[0] - top[0])
+
         def at_or_above(p, cur_slope, top, next_to_top):
-                intersection_point = (p[0], top[1] + cur_slope * (p[0] - top[0]))
-                return p[1] >= intersection_point[1]
+            intersection_point = (p[0], top[1] + cur_slope * (p[0] - top[0]))
+            return p[1] >= intersection_point[1]
 
         def non_left_angle_turn(next_to_top, top, p_i):
             next_to_top = np.array(next_to_top)
@@ -61,7 +59,7 @@ def non_left_angle_turn(next_to_top, top, p_i):
             p_i = np.array(p_i)
             res = np.cross((top - next_to_top), (p_i - top))
             return res <= 0
-            
+
         def non_right_angle_turn(next_to_top, top, p_i):
             next_to_top = np.array(next_to_top)
             top = np.array(top)
@@ -82,55 +80,52 @@ def initialize_f1_corners(csd):
 
             return stack
 
-        
         def initialize_f0_corners(csd):
             stack = []
             # append p_{k'+1}, p_{k'}
             stack.append(csd[-1])
             stack.append(csd[-2])
 
-            for i in range(len(csd)-3, -1, -1):
+            for i in range(len(csd) - 3, -1, -1):
                 while len(stack) > 1 and non_right_angle_turn(next_to_top=stack[-2], top=stack[-1], p_i=csd[i]):
                     stack.pop()
                 stack.append(csd[i])
             return stack
-        
 
         point_addition = lambda p1, p2: tuple((p1[0] + p2[0], p1[1] + p2[1]))
         point_subtraction = lambda p1, p2: tuple((p1[0] - p2[0], p1[1] - p2[1]))
 
         def compute_f1(prev_stack, csd):
-            F1 = np.zeros(self.k_distinct+1)
+            F1 = np.zeros(self.k_distinct + 1)
             stack = []
             while prev_stack:
                 stack.append(prev_stack.pop())
 
-            for i in range(2, self.k_distinct+2):
-                F1[i-1] = slope(top=stack[-1], next_to_top=stack[-2]) 
+            for i in range(2, self.k_distinct + 2):
+                F1[i - 1] = slope(top=stack[-1], next_to_top=stack[-2])
                 # p_{i-1}
-                csd[i-1] = point_subtraction(point_addition(csd[i-2], csd[i]), csd[i-1])
-                p_temp = csd[i-1]
-                
-                if at_or_above(p_temp, F1[i-1], top=stack[-1], next_to_top=stack[-2]):
+                csd[i - 1] = point_subtraction(point_addition(csd[i - 2], csd[i]), csd[i - 1])
+                p_temp = csd[i - 1]
+
+                if at_or_above(p_temp, F1[i - 1], top=stack[-1], next_to_top=stack[-2]):
                     continue
-                    
+
                 stack.pop()
                 while len(stack) > 1 and non_left_angle_turn(p_temp, stack[-1], stack[-2]):
                     stack.pop()
                 stack.append(p_temp)
             return F1
 
-        
         def compute_f0(prev_stack, csd):
-            F0 = np.zeros(self.k_distinct+1)
+            F0 = np.zeros(self.k_distinct + 1)
             stack = []
             while prev_stack:
                 stack.append(prev_stack.pop())
 
             for i in range(self.k_distinct, 0, -1):
                 F0[i] = slope(top=stack[-1], next_to_top=stack[-2])
-                csd[i] = point_subtraction(point_addition(csd[i-1], csd[i+1]), csd[i])
-                
+                csd[i] = point_subtraction(point_addition(csd[i - 1], csd[i + 1]), csd[i])
+
                 if at_or_above(csd[i], F0[i], top=stack[-1], next_to_top=stack[-2]):
                     continue
                 stack.pop()
@@ -140,20 +135,19 @@ def compute_f0(prev_stack, csd):
             return F0
 
         csd_1 = compute_csd(
-            unique_elements, 
-            unique_element_counts, 
-            ordered_calibration_scores, 
-            ordered_calibration_labels, 
+            unique_elements,
+            unique_element_counts,
+            ordered_calibration_scores,
+            ordered_calibration_labels,
             ordered_unique_calibration_scores
         )
         csd_0 = csd_1.copy()
         csd_0.append((csd_0[-1][0] + 1, csd_0[-1][1] + 0))
-        
-        csd_1.insert(0, (-1,-1))
-        
+        csd_1.insert(0, (-1, -1))
+
         f1_stack = initialize_f1_corners(csd_1)
         f0_stack = initialize_f0_corners(csd_0)
-                
+
         self.F1 = compute_f1(f1_stack, csd_1)
         self.F0 = compute_f0(f0_stack, csd_0)
         self.unique_elements = unique_elements
@@ -161,33 +155,27 @@ def compute_f0(prev_stack, csd):
 
         return self
 
-
     def predict_proba(self, scores, X=None):
         if not self.fitted:
             raise NotFittedError(f"This {self.name} calibrator is not fitted yet. Call 'fit' with appropriate arguments before using this calibrator.")
         lower = np.searchsorted(self.unique_elements, scores, side="left")
-        upper = np.searchsorted(self.unique_elements[:-1], scores, side="right")+1
+        upper = np.searchsorted(self.unique_elements[:-1], scores, side="right") + 1
 
         p0 = self.F0[lower]
         p1 = self.F1[upper]
 
         return p1 / (1 - p0 + p1)
-    
+
     def predict_intervall(self, scores):
         lower = np.searchsorted(self.unique_elements, scores, side="left")
-        upper = np.searchsorted(self.unique_elements[:-1], scores, side="right")+1
+        upper = np.searchsorted(self.unique_elements[:-1], scores, side="right") + 1
         p0 = self.F0[lower]
         p1 = self.F1[upper]
 
         return np.array(list(zip(p0, p1)))
 
-    '''
-    def predict(self, X):
-        pred_scores = self.predict_proba(X)
-        return self.classes_[np.argmax(pred_scores, axis=1)]
-    '''
 
-class CrossVennAbersCalibrator(BinaryCalibrator):
+class _CrossVennAbersCalibrator(_BinaryCalibrator):
     name = "CrossVennAbersCalibrator"
 
     def __init__(self, estimator, n_folds=5) -> None:
@@ -197,10 +185,8 @@ def __init__(self, estimator, n_folds=5) -> None:
         self.estimator_params = estimator.get_params()
 
     def fit(self, y, scores, X):
-        #positive_label = 1
         unique_labels = np.unique(y)
         assert len(unique_labels) == 2
-        #self.negative_label = unique_labels[unique_labels != positive_label][0]
 
         # split training set X into self.n_folds folds
         splits_x = np.array_split(X, self.n_folds, axis=0)
@@ -213,8 +199,8 @@ def fit(self, y, scores, X):
         self.ivaps = []
 
         for i in range(self.n_folds):
-            X_train = np.concatenate(splits_x[:i] + splits_x[i+1:], axis=0)
-            y_train = np.concatenate(splits_y[:i] + splits_y[i+1:], axis=0)
+            X_train = np.concatenate(splits_x[:i] + splits_x[i + 1:], axis=0)
+            y_train = np.concatenate(splits_y[:i] + splits_y[i + 1:], axis=0)
             X_cal = splits_x[i]
             y_cal = splits_y[i]
 
@@ -226,30 +212,23 @@ def fit(self, y, scores, X):
             # calibrate IVAP with left out dataset
             calibration_scores = model.predict_proba(X_cal)
 
-            calibrator = InductiveVennAbersCalibrator()
+            calibrator = _InductiveVennAbersCalibrator()
             calibrator.fit(y_cal, calibration_scores[:, 1])
             self.ivaps.append(calibrator)
         self.fitted = True
 
         return self
 
-
     def predict_proba(self, scores):
         if not self.fitted:
             raise NotFittedError(f"This {self.name} calibrator is not fitted yet. Call 'fit' with appropriate arguments before using this calibrator.")
         res = []
         for calibrator in self.ivaps:
             res.append(calibrator.predict_intervall(scores))
-        
+
         res = np.array(res)
         p0 = res[:, :, 0]
         p1 = res[:, :, 1]
 
         p1_gm = gmean(p1)
         return p1_gm / (gmean(1 - p0) + p1_gm)
-
-    '''
-    def predict(self, X):
-        pred_scores = self.predict_proba(X)
-        return self.classes_[np.argmax(pred_scores, axis=1)]
-    '''
diff --git a/hiclass/calibration/__init__.py b/hiclass/_calibration/__init__.py
similarity index 100%
rename from hiclass/calibration/__init__.py
rename to hiclass/_calibration/__init__.py
diff --git a/hiclass/calibration/BinaryCalibrator.py b/hiclass/calibration/BinaryCalibrator.py
deleted file mode 100644
index 97ec4965..00000000
--- a/hiclass/calibration/BinaryCalibrator.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import abc
-
-class BinaryCalibrator(abc.ABC):
-    
-    @abc.abstractmethod
-    def fit(self, y, scores, X=None): #pragma: no cover
-        ...
-
-    @abc.abstractmethod
-    def predict_proba(self, scores, X=None): #pragma: no cover
-        ...

From 505af200f592a7f09cb56f1af588265af0c47544 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Tue, 27 Feb 2024 20:37:32 +0100
Subject: [PATCH 11/65] add stratified sampling to cvap

---
 hiclass/HierarchicalClassifier.py           | 57 +++++++++++++++-----
 hiclass/LocalClassifierPerNode.py           | 21 +++++---
 hiclass/_calibration/Calibrator.py          |  1 +
 hiclass/_calibration/VennAbersCalibrator.py | 59 ++++++++++++++++-----
 4 files changed, 107 insertions(+), 31 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index 07147c1e..a53b3f0f 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -4,6 +4,7 @@
 
 import networkx as nx
 import numpy as np
+import scipy
 from joblib import Parallel, delayed
 from sklearn.base import BaseEstimator
 from sklearn.linear_model import LogisticRegression
@@ -133,7 +134,8 @@ def fit(self, X, y, sample_weight=None):
         self._fit_digraph()
 
         # Delete unnecessary variables
-        self._clean_up()
+        if not self.calibration_method == "cvap":
+            self._clean_up()
 
     def _pre_fit(self, X, y, sample_weight):
         # Check that X and y have correct shape
@@ -218,13 +220,26 @@ def calibrate(self, X, y):
             X = check_array(X, accept_sparse="csr", allow_nd=True, ensure_2d=False)
         else:
             X = np.array(X)
+        
+        if self.calibration_method == "cvap":
+            # combine train and calibration dataset for cross validation
+            if isinstance(self.X_, scipy.sparse._csr.csr_matrix):
+                self.X_cross_val = scipy.sparse.vstack([self.X_, X])
+                self.logger_.info(f"CV Dataset X: {str(type(self.X_cross_val))} {str(self.X_cross_val.shape)}")
+            else:
+                self.X_cross_val = np.hstack([self.X_, X])
+                self.logger_.info(f"CV Dataset X: {str(type(self.X_cross_val))} {str(self.X_cross_val.shape)}")
+            self.y_cross_val = np.vstack([self.y_, y])
+            self.y_cross_val = make_leveled(self.y_cross_val)
+            self.y_cross_val = self._disambiguate(self.y_cross_val)
+            self.y_cross_val = self._convert_1d_y_to_2d(self.y_cross_val)
+        else:
+            self.X_cal = X
+            self.y_cal = y
 
-        self.X_cal = X
-        self.y_cal = y
-
-        self.y_cal = make_leveled(self.y_cal)
-        self.y_cal = self._disambiguate(self.y_cal)
-        self.y_cal = self._convert_1d_y_to_2d(self.y_cal)
+            self.y_cal = make_leveled(self.y_cal)
+            self.y_cal = self._disambiguate(self.y_cal)
+            self.y_cal = self._convert_1d_y_to_2d(self.y_cal)
 
         self.cal_binary_policy_ = self._initialize_binary_policy(calibration=True)
         self.logger_.info("Calibrating")
@@ -232,6 +247,8 @@ def calibrate(self, X, y):
         # Create a calibrator for each local classifier
         self._initialize_local_calibrators()
         self._calibrate_digraph()
+
+        self._clean_up()
         return self
 
     def _predict_ood():
@@ -405,9 +422,15 @@ def _fit_node_classifier(
 
     def _fit_node_calibrator(self, nodes, local_mode: bool = False, use_joblib: bool = False):
         # TODO: add support for multithreading
-        calibrators = [self._fit_calibrator(self, node) for node in nodes]
+        #calibrators = [self._fit_calibrator(self, node) for node in nodes]
+        calibrators = []
+        for idx, node in enumerate(nodes):
+            self.logger_.info(f"calibrating node {idx+1}/{len(nodes)}")
+            calibrators.append(self._fit_calibrator(self, node))
+
         for calibrator, node in zip(calibrators, nodes):
-            self.hierarchy_.nodes[node]["calibrator"] = calibrator
+            if calibrator:
+                self.hierarchy_.nodes[node]["calibrator"] = calibrator
 
     @staticmethod
     def _fit_classifier(self, node):
@@ -419,7 +442,17 @@ def _fit_calibrator(self, node):
 
     def _clean_up(self):
         self.logger_.info("Cleaning up variables that can take a lot of disk space")
-        del self.X_
-        del self.y_
-        if self.sample_weight_ is not None:
+        if hasattr(self, 'X_'):
+            del self.X_
+        if hasattr(self, 'y_'):
+            del self.y_
+        if hasattr(self, 'sample_weight') and self.sample_weight_ is not None:
             del self.sample_weight_
+        if hasattr(self, 'X_cal'):
+            del self.X_cal
+        if hasattr(self, 'y_cal'):
+            del self.y_cal
+        if hasattr(self, 'y_cross_val'):
+            del self.y_cross_val
+        if hasattr(self, 'X_cross_val'):
+            del self.X_cross_val
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index 6ee9e38e..d96c8d00 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -83,7 +83,7 @@ def __init__(
         calibration_method : {"ivap", "cvap", "platt", "isotonic"}, str, default=None
             If set, use the desired method to calibrate probabilities returned by predict_proba().
         return_all_probabilities : bool, default=False
-            If True, return probabilities from all levels. Otherwise, return only probabilities from the last level.
+            If True, return probabilities for all levels. Otherwise, return only probabilities for the last level.
         """
         super().__init__(
             local_classifier=local_classifier,
@@ -296,10 +296,14 @@ def _initialize_binary_policy(self, calibration=False):
         if isinstance(self.binary_policy, str):
             self.logger_.info(f"Initializing {self.binary_policy} binary policy")
             try:
-                if calibration:
+                if calibration and self.calibration_method != "cvap":
                     binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
                         self.binary_policy.lower()
                     ](self.hierarchy_, self.X_cal, self.y_cal, None)
+                elif calibration and self.calibration_method == "cvap":
+                    binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
+                        self.binary_policy.lower()
+                    ](self.hierarchy_, self.X_cross_val, self.y_cross_val, None)
                 else:
                     binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
                         self.binary_policy.lower()
@@ -356,8 +360,6 @@ def _calibrate_digraph(self, local_mode: bool = False, use_joblib: bool = False)
     @staticmethod
     def _fit_classifier(self, node):
         classifier = self.hierarchy_.nodes[node]["classifier"]
-        self.logger_.info("policy: " + str(self.binary_policy_))
-        self.logger_.info("node: " + str(node))
         X, y, sample_weight = self.binary_policy_.get_binary_examples(node)
         self.logger_.info("fitting model " + "node: " + str(node) + " " + str(X.shape) + " : " + str(y.shape) + " unique labels: " + str(len(np.unique(y))))
         unique_y = np.unique(y)
@@ -379,14 +381,19 @@ def _fit_calibrator(self, node):
             calibrator = self.hierarchy_.nodes[node]["calibrator"]
         except KeyError:
             self.logger_.info("no calibrator for " + "node: " + str(node))
-        X, y, sample_weight = self.cal_binary_policy_.get_binary_examples(node)
+            return None
+        X, y, _ = self.cal_binary_policy_.get_binary_examples(node)
         self.logger_.info("fitting calibrator " + "node: " + str(node) + " " + str(X.shape) + " : " + str(y.shape))
-        if len(y) == 0:
+        if len(y) == 0 or len(np.unique(y)) < 2:
             self.logger_.info(f"No calibration samples to fit calibrator for node: {str(node)}")
+            self.hierarchy_.nodes[node].pop('calibrator', None)
             return None
         calibrator.fit(X, y)
         return calibrator
 
     def _clean_up(self):
         super()._clean_up()
-        del self.binary_policy_
+        if hasattr(self, 'binary_policy_'):
+            del self.binary_policy_
+        if hasattr(self, 'cal_binary_policy_'):
+            del self.cal_binary_policy_
diff --git a/hiclass/_calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
index ad9b87ed..f362dd3f 100644
--- a/hiclass/_calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -62,6 +62,7 @@ def fit(self, X, y):
         else:
             self.label_encoder = LabelEncoder()
             encoded_y = self.label_encoder.fit_transform(y)
+            print(f"after encoding: {np.unique(encoded_y)}")
             calibrator = self._create_calibrator(self.method, self.method_params)
             calibrator.fit(encoded_y, calibration_scores[:, 1], X)
             self.calibrators.append(calibrator)
diff --git a/hiclass/_calibration/VennAbersCalibrator.py b/hiclass/_calibration/VennAbersCalibrator.py
index 81459caa..1fa45ebf 100644
--- a/hiclass/_calibration/VennAbersCalibrator.py
+++ b/hiclass/_calibration/VennAbersCalibrator.py
@@ -1,5 +1,6 @@
 import numpy as np
 from sklearn.exceptions import NotFittedError
+from sklearn.model_selection import StratifiedKFold
 from hiclass._calibration.BinaryCalibrator import _BinaryCalibrator
 from scipy.stats import gmean
 
@@ -188,21 +189,19 @@ def fit(self, y, scores, X):
         unique_labels = np.unique(y)
         assert len(unique_labels) == 2
 
-        # split training set X into self.n_folds folds
-        splits_x = np.array_split(X, self.n_folds, axis=0)
-        splits_y = np.array_split(y, self.n_folds, axis=0)
-
-        # create random permutation
-        perm = np.random.default_rng().permutation(self.n_folds)
-        splits_x = [splits_x[i] for i in perm]
-        splits_y = [splits_y[i] for i in perm]
+        splits_x, splits_y = create_n_splits(X, y, self.n_folds)
         self.ivaps = []
 
         for i in range(self.n_folds):
-            X_train = np.concatenate(splits_x[:i] + splits_x[i + 1:], axis=0)
-            y_train = np.concatenate(splits_y[:i] + splits_y[i + 1:], axis=0)
-            X_cal = splits_x[i]
-            y_cal = splits_y[i]
+            X_train, X_cal = splits_x[i][0], splits_x[i][1]
+            y_train, y_cal = splits_y[i][0], splits_y[i][1]
+
+            if len(np.unique(y_train)) < 2 or len(np.unique(y_cal)) < 2:
+                print("skip cv split due to lack of positive samples!")
+                continue
+
+            elements, _, counts = np.unique(y_train, return_index=True, return_counts=True)
+            print(f"y_train : {np.unique(elements)} : {np.unique(counts)}")
 
             # train underlying model with x_train and y_train
             model = self.estimator_type()
@@ -215,6 +214,7 @@ def fit(self, y, scores, X):
             calibrator = _InductiveVennAbersCalibrator()
             calibrator.fit(y_cal, calibration_scores[:, 1])
             self.ivaps.append(calibrator)
+
         self.fitted = True
 
         return self
@@ -226,9 +226,44 @@ def predict_proba(self, scores):
         for calibrator in self.ivaps:
             res.append(calibrator.predict_intervall(scores))
 
+        if len(res) == 0:
+            return np.zeros_like(scores, dtype=np.float32)
+        
         res = np.array(res)
         p0 = res[:, :, 0]
         p1 = res[:, :, 1]
 
         p1_gm = gmean(p1)
         return p1_gm / (gmean(1 - p0) + p1_gm)
+
+def split_csr_matrix(matrix, n_folds):
+    n_rows = matrix.shape[0]
+    rows_per_fold = n_rows // n_folds
+    remainder = n_rows % n_folds
+
+    folds = []
+    start_idx = 0
+    for i in range(n_folds):
+        # Determine the number of rows for this chunk
+        # Add an extra row to some chunks to account for the remainder
+        extra_row = 1 if i < remainder else 0
+        end_idx = start_idx + rows_per_fold + extra_row
+        
+        # Slice the matrix to create the chunk
+        fold = matrix[start_idx:end_idx]
+        folds.append(fold)
+        
+        # Update the start index for the next chunk
+        start_idx = end_idx
+
+    return folds
+
+def create_n_splits(X, y, n_folds):
+    splitter = StratifiedKFold(n_splits=n_folds)
+    splits_x = []
+    splits_y = []
+    for train_index, cal_index in splitter.split(X, y):
+        splits_x.append((X[train_index], X[cal_index]))
+        splits_y.append((y[train_index], y[cal_index]))
+    
+    return splits_x, splits_y

From fd3c66c8fb6d3ec5e5c2955e4db13ed956a7bd08 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Tue, 27 Feb 2024 21:06:48 +0100
Subject: [PATCH 12/65] add tests for calibration

---
 tests/test_calibration.py | 232 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 232 insertions(+)
 create mode 100644 tests/test_calibration.py

diff --git a/tests/test_calibration.py b/tests/test_calibration.py
new file mode 100644
index 00000000..469a8a7e
--- /dev/null
+++ b/tests/test_calibration.py
@@ -0,0 +1,232 @@
+import pytest
+import numpy as np
+from unittest.mock import Mock
+from numpy.testing import assert_array_almost_equal, assert_array_equal
+from sklearn.linear_model import LogisticRegression
+from sklearn.exceptions import NotFittedError
+
+from hiclass._calibration.VennAbersCalibrator import _InductiveVennAbersCalibrator, _CrossVennAbersCalibrator
+from hiclass._calibration.PlattScaling import _PlattScaling
+from hiclass._calibration.IsotonicRegression import _IsotonicRegression
+from hiclass._calibration.Calibrator import _Calibrator
+
+@pytest.fixture
+def binary_calibration_data():
+    prob = np.array([[0.37, 0.63],
+                     [0.39, 0.61],
+                     [0.42, 0.58],
+                     [0.51, 0.49],
+                     [0.51, 0.49],
+                     [0.45, 0.55],
+                     [0.48, 0.52],
+                     [0.60, 0.40],
+                     [0.54, 0.46],
+                     [0.57, 0.43],
+                     [0.57, 0.43],
+                     [0.62, 0.38]]) 
+    
+    assert_array_equal(np.sum(prob, axis=1), np.ones(len(prob)))
+
+    ground_truth_labels = np.array([1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0])
+    return prob, ground_truth_labels
+
+@pytest.fixture
+def binary_test_scores():
+    X = np.array([[0.22, 0.78],
+                  [0.48, 0.52],
+                  [0.66, 0.34],
+                  [0.01, 0.99],
+                  [0.77, 0.23],
+                  [0.50, 0.50],
+                  [1.00, 0.00]])
+    
+    assert_array_equal(np.sum(X, axis=1), np.ones(len(X)))
+    return X
+
+@pytest.fixture
+def binary_cal_X():
+    X = np.array([[1, 2], 
+                  [3, 4], 
+                  [5, 6], 
+                  [7, 8], 
+                  [1, 2], 
+                  [3, 4], 
+                  [5, 6], 
+                  [7, 8], 
+                  [1, 2], 
+                  [3, 4], 
+                  [5, 6], 
+                  [7, 8]])
+    return X
+
+@pytest.fixture
+def binary_mock_estimator(binary_calibration_data, binary_test_scores):
+    # return calibration scores or test scores depending on input size
+    side_effect = lambda X: binary_calibration_data[0] if len(X) == len(binary_calibration_data[0]) else binary_test_scores
+
+    lr = LogisticRegression()
+    binary_estimator = Mock(spec=lr)
+    binary_estimator.predict_proba.side_effect = side_effect
+    binary_estimator.classes_ = np.array([0, 1])
+    binary_estimator.get_params.return_value = lr.get_params()
+    return binary_estimator
+
+@pytest.fixture
+def multiclass_calibration_data():
+    prob = np.array([[0.12, 0.24, 0.64],
+                     [0.44, 0.22, 0.34],
+                     [0.30, 0.40, 0.30],
+                     [0.11, 0.33, 0.56],
+                     [0.44, 0.22, 0.34],
+                     [0.66, 0.33, 0.01],
+                     [0.14, 0.66, 0.20],
+                     [0.44, 0.12, 0.44],
+                     [0.64, 0.24, 0.12],
+                     [0.20, 0.77, 0.03]
+    ])
+
+    assert_array_equal(np.sum(prob, axis=1), np.ones(len(prob)))
+
+    ground_truth_labels = np.array([2, 0, 1, 2, 0, 0, 1, 0, 0, 1])
+    return prob, ground_truth_labels
+
+@pytest.fixture
+def multiclass_test_scores():
+    X = np.array([[0.23, 0.47, 0.30],
+                  [0.44, 0.21, 0.35],
+                  [0.22, 0.22, 0.56],
+                  [0.25, 0.50, 0.25],
+                  [0.01, 0.72, 0.27],
+                  [0.12, 0.35, 0.53],
+                  [0.30, 0.23, 0.47]
+    ])
+    
+    assert_array_equal(np.sum(X, axis=1), np.ones(len(X)))
+    return X
+
+@pytest.fixture
+def multiclass_mock_estimator(multiclass_calibration_data, multiclass_test_scores):
+    # return calibration scores or test scores depending on input size
+    side_effect = lambda X: multiclass_calibration_data[0] if len(X) == len(multiclass_calibration_data[0]) else multiclass_test_scores
+
+    multiclass_estimator = Mock(spec=LogisticRegression)
+    multiclass_estimator.predict_proba.side_effect = side_effect
+    multiclass_estimator.classes_ = np.array([0, 1, 2])
+    return multiclass_estimator
+
+def test_inductive_venn_abers_calibrator(binary_calibration_data, binary_test_scores):
+    scores, ground_truth_labels = binary_calibration_data
+    test_scores = binary_test_scores
+    calibrator = _InductiveVennAbersCalibrator()
+    calibrator.fit(scores=scores[:, 1], y=ground_truth_labels)
+
+    intervalls = calibrator.predict_intervall(test_scores[:, 1])
+    proba = calibrator.predict_proba(test_scores[:, 1])
+
+    assert_array_almost_equal(calibrator.F1, np.array([0, 0.33333333, 0.375, 0.375, 0.4, 0.5, 0.5, 0.66666666, 0.66666666, 1.0, 1.0]))
+    assert_array_almost_equal(calibrator.F0, np.array([0, 0, 0.2, 0.2, 0.2, 0.25, 0.25, 0.33333333, 0.33333333, 0.5, 0.66666666]))
+    assert_array_almost_equal(intervalls, np.array([[0.66666666, 1.0], [0.25, 0.66666666], [0, 0.33333333], [0.66666666, 1.0], [0, 0.33333333], [0.25, 0.5], [0,0.33333333]]))
+    assert_array_almost_equal(proba, np.array([0.74999999, 0.47058823, 0.24999999, 0.74999999, 0.24999999, 0.4, 0.24999999]))
+
+def test_cross_venn_abers_calibrator(binary_calibration_data, binary_test_scores, binary_cal_X):
+    cal_scores, y_cal = binary_calibration_data
+    test_scores = binary_test_scores
+
+    calibrator = _CrossVennAbersCalibrator(LogisticRegression(), n_folds=5)
+    calibrator.fit(y_cal, cal_scores, X=binary_cal_X)
+    proba = calibrator.predict_proba(test_scores[:, 1])
+    expected = np.array([0.602499, 0.51438, 0.397501, 0.602499, 0.346239, 0.51438, 0.328119])
+    assert len(calibrator.ivaps) == 5
+    assert_array_almost_equal(proba, expected)
+
+def test_platt_scaling(binary_calibration_data, binary_test_scores):
+    cal_scores, cal_labels = binary_calibration_data
+    calibrator = _PlattScaling()
+    calibrator.fit(cal_labels, cal_scores[:, 1])
+    proba = calibrator.predict_proba(binary_test_scores[:, 1])
+
+    assert proba.shape == (len(binary_test_scores),)
+    assert_array_almost_equal(proba, np.array([0.8827398, 0.46130191, 0.15976229, 0.97756289, 0.07045925, 0.42011212, 0.01095897]))
+
+def test_isotonic_regression(binary_calibration_data, binary_test_scores):
+    cal_scores, cal_labels = binary_calibration_data
+    calibrator = _IsotonicRegression()
+    calibrator.fit(cal_labels, cal_scores[:, 1])
+    proba = calibrator.predict_proba(binary_test_scores[:, 1])
+    
+    assert proba.shape == (len(binary_test_scores),)
+    assert_array_almost_equal(proba, np.array([1.0, 0.33333333, 0.0, 1.0, 0.0, 0.33333333, 0.0]))
+
+def test_illegal_calibration_method_raises_error(binary_mock_estimator):
+    with pytest.raises(ValueError, match="abc is not a valid calibration method."):
+        _Calibrator(binary_mock_estimator, method="abc")
+
+def test_not_fitted_calibrator_throws_error(binary_test_scores, binary_mock_estimator):
+    for calibrator in [_PlattScaling(), 
+                       _IsotonicRegression(), 
+                       _InductiveVennAbersCalibrator(), 
+                       _CrossVennAbersCalibrator(binary_mock_estimator)]:
+            with pytest.raises(NotFittedError, match=f"This {calibrator.name} calibrator is not fitted yet. Call 'fit' with appropriate arguments before using this calibrator."):
+                calibrator.predict_proba(binary_test_scores)
+
+def test_valid_calibration(binary_calibration_data, binary_test_scores, binary_cal_X, binary_mock_estimator):
+    _, y_cal = binary_calibration_data
+
+    for method in _Calibrator.available_methods:
+        lr = LogisticRegression()
+        lr.fit(binary_cal_X, y_cal)
+
+        calibrator = _Calibrator(lr, method)
+        calibrator.fit(X=binary_cal_X, y=y_cal)
+        proba = calibrator.predict_proba(binary_test_scores)
+
+        assert proba.shape == (len(binary_test_scores),2)
+
+def test_multiclass_calibration(multiclass_calibration_data, multiclass_test_scores, multiclass_mock_estimator):
+    scores, y_cal = multiclass_calibration_data
+
+    calibrator = _Calibrator(multiclass_mock_estimator, method="ivap")
+    calibrator.fit(X=scores, y=y_cal)
+    assert len(calibrator.calibrators) == scores.shape[1]
+
+    proba = calibrator.predict_proba(multiclass_test_scores)
+    assert proba.shape == multiclass_test_scores.shape
+    assert_array_almost_equal(np.sum(proba, axis=1), np.ones(len(proba)))
+
+    expected = np.array([[0.27777778, 0.55555556, 0.16666667],
+                         [0.52173913, 0.13043478, 0.34782609],
+                         [0.33333333, 0.16666667, 0.5       ],
+                         [0.28571429, 0.57142857, 0.14285714],
+                         [0.13483146, 0.70786517, 0.15730337],
+                         [0.16666667, 0.41666667, 0.41666667],
+                         [0.42857143, 0.14285714, 0.42857143]])
+    
+    assert_array_almost_equal(proba, expected)
+
+def test_multiclass_probability_merge(multiclass_mock_estimator, multiclass_calibration_data, multiclass_test_scores):
+    calibrator = _Calibrator(estimator=multiclass_mock_estimator, method="ivap")
+    X_cal, y_cal = multiclass_calibration_data
+    calibrator.fit(X_cal, y_cal)
+
+    calibrator_1 = Mock(spec=_PlattScaling)
+    calibrator_1.predict_proba.return_value = np.array([0.40, 0.60, 0.20, 0.80, 0.75, 0.25, 0.85])
+    
+    calibrator_2 = Mock(spec=_PlattScaling)
+    calibrator_2.predict_proba.return_value = np.array([0.30, 0.70, 0.35, 0.65, 0.30, 0.70, 0.20])
+
+    calibrator_3 = Mock(spec=_PlattScaling)
+    calibrator_3.predict_proba.return_value = np.array([0.55, 0.45, 0.10, 0.90, 0.45, 0.55, 0.40])
+
+    calibrator.calibrators = [calibrator_1, calibrator_2, calibrator_3]
+    proba = calibrator.predict_proba(multiclass_test_scores)
+
+    expected = np.array([[0.32, 0.24, 0.44],
+                         [0.34, 0.40, 0.26],
+                         [0.31, 0.54, 0.15],
+                         [0.34, 0.28, 0.38],
+                         [0.5,  0.20, 0.30],
+                         [0.17, 0.46, 0.37],
+                         [0.59, 0.14, 0.27]])
+    
+    assert_array_equal(np.sum(expected, axis=1), np.ones(len(expected)))
+    assert_array_almost_equal(proba, expected, decimal=2)

From 521741fa511fbb6277a0e9c615e5f3266dbc0501 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Tue, 27 Feb 2024 21:17:50 +0100
Subject: [PATCH 13/65] add local brier score + test

---
 hiclass/metrics.py    | 10 ++++++++++
 tests/test_metrics.py | 31 ++++++++++++++++++++++++++++++-
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/hiclass/metrics.py b/hiclass/metrics.py
index 5e9fe441..e6e5ae02 100644
--- a/hiclass/metrics.py
+++ b/hiclass/metrics.py
@@ -1,6 +1,7 @@
 """Helper functions to compute hierarchical evaluation metrics."""
 import numpy as np
 from sklearn.utils import check_array
+from sklearn.preprocessing import LabelEncoder
 
 from hiclass.HierarchicalClassifier import make_leveled
 
@@ -247,3 +248,12 @@ def _compute_macro(y_true: np.ndarray, y_pred: np.ndarray, _micro_function):
         sample_score = _micro_function(np.array([ground_truth]), np.array([prediction]))
         overall_sum = overall_sum + sample_score
     return overall_sum / len(y_true)
+
+
+def _multiclass_brier_score(y_true, y_prob):
+    '''
+    Assumes that y_true is ordered
+    '''
+    label_encoder = LabelEncoder()
+    y_true = label_encoder.fit_transform(y_true)
+    return (1/y_prob.shape[0]) * np.sum(np.sum(np.square(y_prob - np.eye(y_prob.shape[1])[y_true]), axis=1))
\ No newline at end of file
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index dceaf440..31a8d1bf 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -1,8 +1,10 @@
 import numpy as np
 import pytest
 from pytest import approx
+import math
+from numpy.testing import assert_array_almost_equal, assert_array_equal
 
-from hiclass.metrics import precision, recall, f1
+from hiclass.metrics import precision, recall, f1, _multiclass_brier_score
 
 
 # TODO: add tests for 3D dataframe (not sure if it's possible to have 3D dataframes)
@@ -334,3 +336,30 @@ def test_empty_levels_2d_list_2():
     y_pred = [["1"], ["2", "3"], ["4", "5", "6"]]
     assert 1 == f1(y_true, y_pred)
     assert 1 == f1(y_true, y_true)
+
+
+@pytest.fixture
+def uncertainty_data():
+    prob = np.array([[0.88, 0.06, 0.06],
+                     [0.22, 0.48, 0.30],
+                     [0.33, 0.33, 0.34],
+                     [0.49, 0.40, 0.11],
+                     [0.23, 0.03, 0.74],
+                     [0.21, 0.67, 0.12],
+                     [0.34, 0.34, 0.32],
+                     [0.02, 0.77, 0.21],
+                     [0.44, 0.42, 0.14],
+                     [0.85, 0.13, 0.02]])
+
+    assert_array_equal(np.sum(prob, axis=1), np.ones(len(prob)))
+
+    y_pred = np.array([0, 1, 2, 0, 2, 1, 0, 1, 0, 0])
+    y_true = np.array([0, 2, 0, 0, 2, 1, 1, 1, 0, 0])
+
+    return prob, y_pred, y_true
+
+
+def test_local_brier_score(uncertainty_data):
+    prob, _, y_true = uncertainty_data
+    brier_score = _multiclass_brier_score(y_true, prob)
+    assert math.isclose(brier_score, 0.34852)

From d437d0eac4bbbdad242eabe96ed2e1944eb9542e Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Wed, 28 Feb 2024 21:22:31 +0100
Subject: [PATCH 14/65] fix cvap

---
 hiclass/_calibration/VennAbersCalibrator.py | 47 +++++++++++----------
 hiclass/metrics.py                          |  4 +-
 2 files changed, 27 insertions(+), 24 deletions(-)

diff --git a/hiclass/_calibration/VennAbersCalibrator.py b/hiclass/_calibration/VennAbersCalibrator.py
index 1fa45ebf..0cfa3665 100644
--- a/hiclass/_calibration/VennAbersCalibrator.py
+++ b/hiclass/_calibration/VennAbersCalibrator.py
@@ -188,32 +188,38 @@ def __init__(self, estimator, n_folds=5) -> None:
     def fit(self, y, scores, X):
         unique_labels = np.unique(y)
         assert len(unique_labels) == 2
-
-        splits_x, splits_y = create_n_splits(X, y, self.n_folds)
         self.ivaps = []
 
-        for i in range(self.n_folds):
-            X_train, X_cal = splits_x[i][0], splits_x[i][1]
-            y_train, y_cal = splits_y[i][0], splits_y[i][1]
+        try:
+            splits_x, splits_y = create_n_splits(X, y, self.n_folds)
+        except ValueError:
+            splits_x, splits_y = [], []
 
-            if len(np.unique(y_train)) < 2 or len(np.unique(y_cal)) < 2:
-                print("skip cv split due to lack of positive samples!")
-                continue
+        if len(splits_x) == 0 or any([(len(np.unique(y_train)) < 2 or len(np.unique(y_cal)) < 2) for y_train, y_cal in splits_y]):
+            print("skip cv split due to lack of positive samples!")
+            calibrator = _InductiveVennAbersCalibrator()
+            calibrator.fit(y, scores)
+            self.ivaps.append(calibrator)
 
-            elements, _, counts = np.unique(y_train, return_index=True, return_counts=True)
-            print(f"y_train : {np.unique(elements)} : {np.unique(counts)}")
+        else:
+            for i in range(self.n_folds):
+                X_train, X_cal = splits_x[i][0], splits_x[i][1]
+                y_train, y_cal = splits_y[i][0], splits_y[i][1]
 
-            # train underlying model with x_train and y_train
-            model = self.estimator_type()
-            model.set_params(**self.estimator_params)
-            model.fit(X_train, y_train)
+                elements, _, counts = np.unique(y_train, return_index=True, return_counts=True)
+                print(f"y_train : {np.unique(elements)} : {np.unique(counts)}")
 
-            # calibrate IVAP with left out dataset
-            calibration_scores = model.predict_proba(X_cal)
+                # train underlying model with x_train and y_train
+                model = self.estimator_type()
+                model.set_params(**self.estimator_params)
+                model.fit(X_train, y_train)
 
-            calibrator = _InductiveVennAbersCalibrator()
-            calibrator.fit(y_cal, calibration_scores[:, 1])
-            self.ivaps.append(calibrator)
+                # calibrate IVAP with left out dataset
+                calibration_scores = model.predict_proba(X_cal)
+
+                calibrator = _InductiveVennAbersCalibrator()
+                calibrator.fit(y_cal, calibration_scores[:, 1])
+                self.ivaps.append(calibrator)
 
         self.fitted = True
 
@@ -225,9 +231,6 @@ def predict_proba(self, scores):
         res = []
         for calibrator in self.ivaps:
             res.append(calibrator.predict_intervall(scores))
-
-        if len(res) == 0:
-            return np.zeros_like(scores, dtype=np.float32)
         
         res = np.array(res)
         p0 = res[:, :, 0]
diff --git a/hiclass/metrics.py b/hiclass/metrics.py
index e6e5ae02..996c82e0 100644
--- a/hiclass/metrics.py
+++ b/hiclass/metrics.py
@@ -250,10 +250,10 @@ def _compute_macro(y_true: np.ndarray, y_pred: np.ndarray, _micro_function):
     return overall_sum / len(y_true)
 
 
-def _multiclass_brier_score(y_true, y_prob):
+def _multiclass_brier_score(y_true: np.ndarray, y_prob: np.ndarray):
     '''
     Assumes that y_true is ordered
     '''
     label_encoder = LabelEncoder()
     y_true = label_encoder.fit_transform(y_true)
-    return (1/y_prob.shape[0]) * np.sum(np.sum(np.square(y_prob - np.eye(y_prob.shape[1])[y_true]), axis=1))
\ No newline at end of file
+    return (1/y_prob.shape[0]) * np.sum(np.sum(np.square(y_prob - np.eye(y_prob.shape[1])[y_true]), axis=1))

From c0ddc09a28a9f883c2aa0aa109120690bdafc7c7 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sun, 3 Mar 2024 15:25:03 +0100
Subject: [PATCH 15/65] add log loss + test, make brier loss more robust

---
 hiclass/HierarchicalClassifier.py           | 11 +++++-
 hiclass/LocalClassifierPerNode.py           |  5 +--
 hiclass/_calibration/Calibrator.py          |  1 -
 hiclass/_calibration/VennAbersCalibrator.py | 41 ++++-----------------
 hiclass/metrics.py                          | 36 +++++++++++++++---
 tests/test_metrics.py                       | 29 +++++++++++----
 6 files changed, 70 insertions(+), 53 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index a53b3f0f..c4cde731 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -155,6 +155,9 @@ def _pre_fit(self, X, y, sample_weight):
             self.sample_weight_ = None
 
         self.y_ = make_leveled(self.y_)
+        # Avoids creating more columns in prediction if edges are a->b and b->c,
+        # which would generate the prediction a->b->c
+        self.y_ = self._disambiguate(self.y_)
 
         if self.y_.ndim > 1:
             self.max_level_dimensions_ = np.array([len(np.unique(self.y_[:, level])) for level in range(self.y_.shape[1])])
@@ -170,7 +173,7 @@ def _pre_fit(self, X, y, sample_weight):
 
         # Avoids creating more columns in prediction if edges are a->b and b->c,
         # which would generate the prediction a->b->c
-        self.y_ = self._disambiguate(self.y_)
+        #self.y_ = self._disambiguate(self.y_)
 
         # Create DAG from self.y_ and store to self.hierarchy_
         self._create_digraph()
@@ -416,7 +419,11 @@ def _fit_node_classifier(
                 )
 
         else:
-            classifiers = [self._fit_classifier(self, node) for node in nodes]
+            classifiers = []
+            for idx, node in enumerate(nodes):
+                self.logger_.info(f"fitting node {idx+1}/{len(nodes)}")
+                classifiers.append(self._fit_classifier(self, node))
+            #classifiers = [self._fit_classifier(self, node) for node in nodes]
         for classifier, node in zip(classifiers, nodes):
             self.hierarchy_.nodes[node]["classifier"] = classifier
 
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index d96c8d00..f67795aa 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -264,15 +264,14 @@ def predict_proba(self, X):
             if subset_x.shape[0] > 0:
                 local_probabilities = np.zeros((subset_x.shape[0], len(successors)))
                 for i, successor in enumerate(successors):
-                    successor_name = str(successor).split(self.separator_)[-1]
-                    self.logger_.info(f"Predicting probabilities for node '{successor_name}'")
+                    self.logger_.info(f"Predicting probabilities for node '{str(successor)}'")
                     classifier = self.hierarchy_.nodes[successor]["classifier"]
                     # use classifier as a fallback if no calibrator is available
                     calibrator = self.hierarchy_.nodes[successor].get("calibrator", classifier)
                     positive_index = np.where(calibrator.classes_ == 1)[0]
                     proba = calibrator.predict_proba(subset_x)[:, positive_index][:, 0]
                     local_probabilities[:, i] = proba
-                    class_index = self.class_to_index_mapping_[level][successor_name]
+                    class_index = self.class_to_index_mapping_[level][str(successor)]
                     level_probability_list[-1][mask, class_index] = proba
 
                 highest_local_probability = np.argmax(local_probabilities, axis=1)
diff --git a/hiclass/_calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
index f362dd3f..ad9b87ed 100644
--- a/hiclass/_calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -62,7 +62,6 @@ def fit(self, X, y):
         else:
             self.label_encoder = LabelEncoder()
             encoded_y = self.label_encoder.fit_transform(y)
-            print(f"after encoding: {np.unique(encoded_y)}")
             calibrator = self._create_calibrator(self.method, self.method_params)
             calibrator.fit(encoded_y, calibration_scores[:, 1], X)
             self.calibrators.append(calibrator)
diff --git a/hiclass/_calibration/VennAbersCalibrator.py b/hiclass/_calibration/VennAbersCalibrator.py
index 0cfa3665..3f178cb9 100644
--- a/hiclass/_calibration/VennAbersCalibrator.py
+++ b/hiclass/_calibration/VennAbersCalibrator.py
@@ -191,9 +191,14 @@ def fit(self, y, scores, X):
         self.ivaps = []
 
         try:
-            splits_x, splits_y = create_n_splits(X, y, self.n_folds)
+            splitter = StratifiedKFold(n_splits=self.n_folds)
+            splits_x = []
+            splits_y = []
+            for train_index, cal_index in splitter.split(X, y):
+                splits_x.append((X[train_index], X[cal_index]))
+                splits_y.append((y[train_index], y[cal_index]))
         except ValueError:
-            splits_x, splits_y = [], []
+                splits_x, splits_y = [], []
 
         if len(splits_x) == 0 or any([(len(np.unique(y_train)) < 2 or len(np.unique(y_cal)) < 2) for y_train, y_cal in splits_y]):
             print("skip cv split due to lack of positive samples!")
@@ -238,35 +243,3 @@ def predict_proba(self, scores):
 
         p1_gm = gmean(p1)
         return p1_gm / (gmean(1 - p0) + p1_gm)
-
-def split_csr_matrix(matrix, n_folds):
-    n_rows = matrix.shape[0]
-    rows_per_fold = n_rows // n_folds
-    remainder = n_rows % n_folds
-
-    folds = []
-    start_idx = 0
-    for i in range(n_folds):
-        # Determine the number of rows for this chunk
-        # Add an extra row to some chunks to account for the remainder
-        extra_row = 1 if i < remainder else 0
-        end_idx = start_idx + rows_per_fold + extra_row
-        
-        # Slice the matrix to create the chunk
-        fold = matrix[start_idx:end_idx]
-        folds.append(fold)
-        
-        # Update the start index for the next chunk
-        start_idx = end_idx
-
-    return folds
-
-def create_n_splits(X, y, n_folds):
-    splitter = StratifiedKFold(n_splits=n_folds)
-    splits_x = []
-    splits_y = []
-    for train_index, cal_index in splitter.split(X, y):
-        splits_x.append((X[train_index], X[cal_index]))
-        splits_y.append((y[train_index], y[cal_index]))
-    
-    return splits_x, splits_y
diff --git a/hiclass/metrics.py b/hiclass/metrics.py
index 996c82e0..743f2187 100644
--- a/hiclass/metrics.py
+++ b/hiclass/metrics.py
@@ -1,9 +1,11 @@
 """Helper functions to compute hierarchical evaluation metrics."""
 import numpy as np
 from sklearn.utils import check_array
+from sklearn.metrics import log_loss
 from sklearn.preprocessing import LabelEncoder
 
 from hiclass.HierarchicalClassifier import make_leveled
+from hiclass import HierarchicalClassifier
 
 
 def _validate_input(y_true, y_pred):
@@ -250,10 +252,32 @@ def _compute_macro(y_true: np.ndarray, y_pred: np.ndarray, _micro_function):
     return overall_sum / len(y_true)
 
 
-def _multiclass_brier_score(y_true: np.ndarray, y_prob: np.ndarray):
-    '''
-    Assumes that y_true is ordered
-    '''
+def _multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, level: int):
+    y_true, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level)
     label_encoder = LabelEncoder()
-    y_true = label_encoder.fit_transform(y_true)
-    return (1/y_prob.shape[0]) * np.sum(np.sum(np.square(y_prob - np.eye(y_prob.shape[1])[y_true]), axis=1))
+    label_encoder.fit(labels)
+    y_true_encoded = label_encoder.transform(y_true)
+    return (1/y_prob.shape[0]) * np.sum(np.sum(np.square(y_prob - np.eye(y_prob.shape[1])[y_true_encoded]), axis=1))
+
+
+def _log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, level: int):
+    y_true, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level)
+    return log_loss(y_true, y_prob, labels=labels)
+
+def _prepare_data(classifier, y_true, y_prob, level):
+    classifier_classes = np.array(classifier.classes_[level]).astype("str")
+    y_true = make_leveled(y_true)
+    y_true = classifier._disambiguate(y_true)
+    y_true = np.array(list(map(lambda x: x[level], y_true)))
+    unique_labels = np.unique(y_true).astype("str")
+    # add labels not seen in the training process
+    new_labels = np.sort(np.union1d(unique_labels, classifier_classes))
+
+    # add empty columns to y_prob
+    new_y_prob = np.zeros((y_prob[level].shape[0], len(new_labels)), dtype=np.float32)
+    for idx, label in enumerate(new_labels):
+        if label in classifier_classes:
+            old_idx = np.where(classifier_classes == label)[0][0]
+            new_y_prob[:, idx] = y_prob[level][:, old_idx]
+    
+    return y_true, new_labels, new_y_prob
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index 31a8d1bf..d3509217 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -3,8 +3,10 @@
 from pytest import approx
 import math
 from numpy.testing import assert_array_almost_equal, assert_array_equal
+from unittest.mock import Mock
 
-from hiclass.metrics import precision, recall, f1, _multiclass_brier_score
+from hiclass.HierarchicalClassifier import HierarchicalClassifier
+from hiclass.metrics import precision, recall, f1, _multiclass_brier_score, _log_loss
 
 
 # TODO: add tests for 3D dataframe (not sure if it's possible to have 3D dataframes)
@@ -340,7 +342,7 @@ def test_empty_levels_2d_list_2():
 
 @pytest.fixture
 def uncertainty_data():
-    prob = np.array([[0.88, 0.06, 0.06],
+    prob = [np.array([[0.88, 0.06, 0.06],
                      [0.22, 0.48, 0.30],
                      [0.33, 0.33, 0.34],
                      [0.49, 0.40, 0.11],
@@ -349,17 +351,30 @@ def uncertainty_data():
                      [0.34, 0.34, 0.32],
                      [0.02, 0.77, 0.21],
                      [0.44, 0.42, 0.14],
-                     [0.85, 0.13, 0.02]])
+                     [0.85, 0.13, 0.02]])]
 
-    assert_array_equal(np.sum(prob, axis=1), np.ones(len(prob)))
+    assert_array_equal(np.sum(prob[0], axis=1), np.ones(len(prob[0])))
 
     y_pred = np.array([0, 1, 2, 0, 2, 1, 0, 1, 0, 0])
-    y_true = np.array([0, 2, 0, 0, 2, 1, 1, 1, 0, 0])
+    y_true = np.array([[0], [2], [0], [0], [2], [1], [1], [1], [0], [0]])
 
     return prob, y_pred, y_true
 
 
 def test_local_brier_score(uncertainty_data):
     prob, _, y_true = uncertainty_data
-    brier_score = _multiclass_brier_score(y_true, prob)
-    assert math.isclose(brier_score, 0.34852)
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = [[0, 1, 2]]
+    brier_score = _multiclass_brier_score(classifier, y_true, prob, level=0)
+    assert math.isclose(brier_score, 0.34852, abs_tol=1e-4)
+
+def test_local_log_loss(uncertainty_data):
+    prob, _, y_true = uncertainty_data
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = [[0, 1, 2]]
+    log_loss = _log_loss(classifier, y_true, prob, level=0)
+    assert math.isclose(log_loss, 0.61790, abs_tol=1e-4)

From 9bc6b2eadf211449f54e70f3a5779ec10af28b7c Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Mon, 4 Mar 2024 16:35:51 +0100
Subject: [PATCH 16/65] add MultiplyCombiner + test

---
 hiclass/HierarchicalClassifier.py             |  4 +-
 hiclass/LocalClassifierPerNode.py             |  4 +-
 hiclass/_calibration/VennAbersCalibrator.py   |  3 -
 .../probability_combiner/MultiplyCombiner.py  | 18 +++++
 .../ProbabilityCombiner.py                    | 12 +++
 hiclass/probability_combiner/__init__.py      |  5 ++
 tests/test_ProbabilityCombiner.py             | 73 +++++++++++++++++++
 tests/test_metrics.py                         |  4 +-
 8 files changed, 115 insertions(+), 8 deletions(-)
 create mode 100644 hiclass/probability_combiner/MultiplyCombiner.py
 create mode 100644 hiclass/probability_combiner/ProbabilityCombiner.py
 create mode 100644 hiclass/probability_combiner/__init__.py
 create mode 100644 tests/test_ProbabilityCombiner.py

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index c4cde731..fc428a63 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -421,7 +421,7 @@ def _fit_node_classifier(
         else:
             classifiers = []
             for idx, node in enumerate(nodes):
-                self.logger_.info(f"fitting node {idx+1}/{len(nodes)}")
+                self.logger_.info(f"fitting node {idx+1}/{len(nodes)}: {str(node)}")
                 classifiers.append(self._fit_classifier(self, node))
             #classifiers = [self._fit_classifier(self, node) for node in nodes]
         for classifier, node in zip(classifiers, nodes):
@@ -432,7 +432,7 @@ def _fit_node_calibrator(self, nodes, local_mode: bool = False, use_joblib: bool
         #calibrators = [self._fit_calibrator(self, node) for node in nodes]
         calibrators = []
         for idx, node in enumerate(nodes):
-            self.logger_.info(f"calibrating node {idx+1}/{len(nodes)}")
+            self.logger_.info(f"calibrating node {idx+1}/{len(nodes)}: {str(node)}")
             calibrators.append(self._fit_calibrator(self, node))
 
         for calibrator, node in zip(calibrators, nodes):
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index f67795aa..913bdf79 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -360,7 +360,7 @@ def _calibrate_digraph(self, local_mode: bool = False, use_joblib: bool = False)
     def _fit_classifier(self, node):
         classifier = self.hierarchy_.nodes[node]["classifier"]
         X, y, sample_weight = self.binary_policy_.get_binary_examples(node)
-        self.logger_.info("fitting model " + "node: " + str(node) + " " + str(X.shape) + " : " + str(y.shape) + " unique labels: " + str(len(np.unique(y))))
+        #self.logger_.info("fitting model " + "node: " + str(node) + " " + str(X.shape) + " : " + str(y.shape) + " unique labels: " + str(len(np.unique(y))))
         unique_y = np.unique(y)
         if len(unique_y) == 1 and self.replace_classifiers:
             self.logger_.info("adding constant classifier")
@@ -382,7 +382,7 @@ def _fit_calibrator(self, node):
             self.logger_.info("no calibrator for " + "node: " + str(node))
             return None
         X, y, _ = self.cal_binary_policy_.get_binary_examples(node)
-        self.logger_.info("fitting calibrator " + "node: " + str(node) + " " + str(X.shape) + " : " + str(y.shape))
+        #self.logger_.info("fitting calibrator " + "node: " + str(node) + " " + str(X.shape) + " : " + str(y.shape))
         if len(y) == 0 or len(np.unique(y)) < 2:
             self.logger_.info(f"No calibration samples to fit calibrator for node: {str(node)}")
             self.hierarchy_.nodes[node].pop('calibrator', None)
diff --git a/hiclass/_calibration/VennAbersCalibrator.py b/hiclass/_calibration/VennAbersCalibrator.py
index 3f178cb9..c79ce60b 100644
--- a/hiclass/_calibration/VennAbersCalibrator.py
+++ b/hiclass/_calibration/VennAbersCalibrator.py
@@ -211,9 +211,6 @@ def fit(self, y, scores, X):
                 X_train, X_cal = splits_x[i][0], splits_x[i][1]
                 y_train, y_cal = splits_y[i][0], splits_y[i][1]
 
-                elements, _, counts = np.unique(y_train, return_index=True, return_counts=True)
-                print(f"y_train : {np.unique(elements)} : {np.unique(counts)}")
-
                 # train underlying model with x_train and y_train
                 model = self.estimator_type()
                 model.set_params(**self.estimator_params)
diff --git a/hiclass/probability_combiner/MultiplyCombiner.py b/hiclass/probability_combiner/MultiplyCombiner.py
new file mode 100644
index 00000000..0a04c7ee
--- /dev/null
+++ b/hiclass/probability_combiner/MultiplyCombiner.py
@@ -0,0 +1,18 @@
+import numpy as np
+from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
+
+class MultiplyCombiner(ProbabilityCombiner):
+    def combine(self, proba):
+        res = [proba[0]]
+        for level in range(1, self.classifier.max_levels_):
+            
+            level_probs = np.zeros_like(proba[level])
+            for node in self.classifier.classes_[level]:
+                predecessor = list(self.classifier.hierarchy_.predecessors(node))[0]
+                predecessor_index = self.classifier.class_to_index_mapping_[level-1][predecessor]
+                index = self.classifier.class_to_index_mapping_[level][node]
+
+                level_probs[:, index] = res[level-1][:, predecessor_index] * proba[level][:, index]
+
+            res.append(level_probs)
+        return res
diff --git a/hiclass/probability_combiner/ProbabilityCombiner.py b/hiclass/probability_combiner/ProbabilityCombiner.py
new file mode 100644
index 00000000..5e3f3772
--- /dev/null
+++ b/hiclass/probability_combiner/ProbabilityCombiner.py
@@ -0,0 +1,12 @@
+import abc
+import numpy as np
+from typing import List
+
+class ProbabilityCombiner(abc.ABC):
+
+    def __init__(self, classifier) -> None:
+        self.classifier = classifier
+
+    @abc.abstractmethod
+    def combine(self, proba: List[np.ndarray]) -> List[np.ndarray]:
+        ...
diff --git a/hiclass/probability_combiner/__init__.py b/hiclass/probability_combiner/__init__.py
new file mode 100644
index 00000000..3109cca9
--- /dev/null
+++ b/hiclass/probability_combiner/__init__.py
@@ -0,0 +1,5 @@
+from .MultiplyCombiner import MultiplyCombiner
+
+__all__ = [
+    "MultiplyCombiner",
+]
\ No newline at end of file
diff --git a/tests/test_ProbabilityCombiner.py b/tests/test_ProbabilityCombiner.py
new file mode 100644
index 00000000..9bc5a3b8
--- /dev/null
+++ b/tests/test_ProbabilityCombiner.py
@@ -0,0 +1,73 @@
+import numpy as np
+import pytest
+from unittest.mock import Mock
+import networkx as nx
+from numpy.testing import assert_array_almost_equal, assert_array_equal
+import math
+
+from hiclass.HierarchicalClassifier import HierarchicalClassifier
+from hiclass.probability_combiner import MultiplyCombiner
+
+
+@pytest.fixture
+def one_sample_probs_with_hierarchy():
+    hierarchy = nx.DiGraph()
+    root_node = "hiclass::root"
+    hierarchy.add_node(root_node)
+    classes_ = [[], [], []]
+
+    for i in range(3):
+        # first level
+        first_level_node = f'level_0_node_{i}'
+        classes_[0].append(first_level_node)
+        hierarchy.add_node(first_level_node)
+        hierarchy.add_edge(root_node, first_level_node)
+        for j in range(3):
+            # second level
+            second_level_node = f'level_1_node_{i}:{j}'
+            classes_[1].append(second_level_node)
+            hierarchy.add_node(second_level_node)
+            hierarchy.add_edge(first_level_node, second_level_node)
+            for k in range(2):
+                # third level
+                third_level_node = f'level_2_node_{i}:{j}:{k}'
+                classes_[2].append(third_level_node)
+                hierarchy.add_node(third_level_node)
+                hierarchy.add_edge(second_level_node, third_level_node)
+
+    probs = [
+        np.array([[0.3, 0.5, 0.2]]), # level 0
+        np.array([[0.19, 0.10, 0.16, 0.14, 0.07, 0.08, 0.19, 0.03, 0.04]]), # level 1
+        np.array([[0.03, 0.17, 0.07, 0.01, 0.00, 0.06, 0.17, 0.02, 0.01, 0.07, 0.20, 0.01, 0.10, 0.00, 0.05, 0.02, 0.00, 0.01]]) # level 2
+    ]
+    assert all([np.sum(probs[level], axis=1) == 1 for level in range(3)])
+
+    return hierarchy, probs, classes_
+
+def test_multiply_combiner(one_sample_probs_with_hierarchy):
+
+    hierarchy, probs, classes = one_sample_probs_with_hierarchy
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = classes
+    classifier.max_levels_ = len(classes)
+    classifier.class_to_index_mapping_ = [{classifier.classes_[level][index]: index for index in range(len(classifier.classes_[level]))} for level in range(classifier.max_levels_)]
+    classifier.hierarchy_ = hierarchy
+
+    combiner = MultiplyCombiner(classifier=classifier)
+    combined_probs = combiner.combine(probs)
+
+    # check combined probability of first node for both levels
+    assert math.isclose(combined_probs[1][0][0], 0.0569, abs_tol=1e-4)
+    assert math.isclose(combined_probs[1][0][0], probs[0][0][0] * probs[1][0][0], abs_tol=1e-4)
+    
+    assert math.isclose(combined_probs[2][0][0], 0.0017, abs_tol=1e-4)
+    assert math.isclose(combined_probs[2][0][0], probs[0][0][0] * probs[1][0][0] * probs[2][0][0], abs_tol=1e-4)
+
+    # check combined probability of last node for both levels
+    assert math.isclose(combined_probs[1][0][-1], 0.008, abs_tol=1e-4)
+    assert math.isclose(combined_probs[1][0][-1], probs[0][0][-1] * probs[1][0][-1], abs_tol=1e-4)
+    
+    assert math.isclose(combined_probs[2][0][-1], 8e-5, abs_tol=1e-4)
+    assert math.isclose(combined_probs[2][0][-1], probs[0][0][-1] * probs[1][0][-1] * probs[2][0][-1], abs_tol=1e-4)
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index d3509217..331787e9 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -4,6 +4,7 @@
 import math
 from numpy.testing import assert_array_almost_equal, assert_array_equal
 from unittest.mock import Mock
+import networkx as nx
 
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
 from hiclass.metrics import precision, recall, f1, _multiclass_brier_score, _log_loss
@@ -360,13 +361,13 @@ def uncertainty_data():
 
     return prob, y_pred, y_true
 
-
 def test_local_brier_score(uncertainty_data):
     prob, _, y_true = uncertainty_data
     obj = HierarchicalClassifier()
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
     classifier.classes_ = [[0, 1, 2]]
+    classifier.hierarchy_ = nx.DiGraph()
     brier_score = _multiclass_brier_score(classifier, y_true, prob, level=0)
     assert math.isclose(brier_score, 0.34852, abs_tol=1e-4)
 
@@ -378,3 +379,4 @@ def test_local_log_loss(uncertainty_data):
     classifier.classes_ = [[0, 1, 2]]
     log_loss = _log_loss(classifier, y_true, prob, level=0)
     assert math.isclose(log_loss, 0.61790, abs_tol=1e-4)
+

From eb2c044c87a65fefd0b88271826736e1c80d2676 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Mon, 4 Mar 2024 17:44:46 +0100
Subject: [PATCH 17/65] add ArithmeticMeanCombiner + test

---
 .../ArithmeticMeanCombiner.py                 | 22 ++++++++++++++++++
 hiclass/probability_combiner/__init__.py      |  2 ++
 tests/test_ProbabilityCombiner.py             | 23 +++++++++++++++++--
 3 files changed, 45 insertions(+), 2 deletions(-)
 create mode 100644 hiclass/probability_combiner/ArithmeticMeanCombiner.py

diff --git a/hiclass/probability_combiner/ArithmeticMeanCombiner.py b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
new file mode 100644
index 00000000..d579a4af
--- /dev/null
+++ b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
@@ -0,0 +1,22 @@
+import numpy as np
+from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
+
+class ArithmeticMeanCombiner(ProbabilityCombiner):
+    def combine(self, proba):
+        res = [proba[0]]
+        sums = [proba[0]]
+        for level in range(1, self.classifier.max_levels_):
+            print(level)
+            level_probs = np.zeros_like(proba[level])
+            level_sum = np.zeros_like(proba[level])
+            for node in self.classifier.classes_[level]:
+                predecessor = list(self.classifier.hierarchy_.predecessors(node))[0]
+                predecessor_index = self.classifier.class_to_index_mapping_[level-1][predecessor]
+                index = self.classifier.class_to_index_mapping_[level][node]
+
+                level_sum[:, index] += proba[level][:, index] + sums[level-1][:, predecessor_index]
+                level_probs[:, index] = level_sum[:, index] / (level+1)
+
+            res.append(level_probs)
+            sums.append(level_sum)
+        return res
diff --git a/hiclass/probability_combiner/__init__.py b/hiclass/probability_combiner/__init__.py
index 3109cca9..808f4fbd 100644
--- a/hiclass/probability_combiner/__init__.py
+++ b/hiclass/probability_combiner/__init__.py
@@ -1,5 +1,7 @@
 from .MultiplyCombiner import MultiplyCombiner
+from .ArithmeticMeanCombiner import ArithmeticMeanCombiner
 
 __all__ = [
     "MultiplyCombiner",
+    "ArithmeticMeanCombiner",
 ]
\ No newline at end of file
diff --git a/tests/test_ProbabilityCombiner.py b/tests/test_ProbabilityCombiner.py
index 9bc5a3b8..1531b8db 100644
--- a/tests/test_ProbabilityCombiner.py
+++ b/tests/test_ProbabilityCombiner.py
@@ -6,7 +6,7 @@
 import math
 
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
-from hiclass.probability_combiner import MultiplyCombiner
+from hiclass.probability_combiner import MultiplyCombiner, ArithmeticMeanCombiner
 
 
 @pytest.fixture
@@ -45,7 +45,6 @@ def one_sample_probs_with_hierarchy():
     return hierarchy, probs, classes_
 
 def test_multiply_combiner(one_sample_probs_with_hierarchy):
-
     hierarchy, probs, classes = one_sample_probs_with_hierarchy
     obj = HierarchicalClassifier()
     classifier = Mock(spec=obj)
@@ -71,3 +70,23 @@ def test_multiply_combiner(one_sample_probs_with_hierarchy):
     
     assert math.isclose(combined_probs[2][0][-1], 8e-5, abs_tol=1e-4)
     assert math.isclose(combined_probs[2][0][-1], probs[0][0][-1] * probs[1][0][-1] * probs[2][0][-1], abs_tol=1e-4)
+
+def test_arithmetic_mean_combiner(one_sample_probs_with_hierarchy):
+    hierarchy, probs, classes = one_sample_probs_with_hierarchy
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = classes
+    classifier.max_levels_ = len(classes)
+    classifier.class_to_index_mapping_ = [{classifier.classes_[level][index]: index for index in range(len(classifier.classes_[level]))} for level in range(classifier.max_levels_)]
+    classifier.hierarchy_ = hierarchy
+
+    combiner = ArithmeticMeanCombiner(classifier=classifier)
+    combined_probs = combiner.combine(probs)
+
+    # check combined probability of first node for both levels
+    assert math.isclose(combined_probs[1][0][0], 0.245, abs_tol=1e-4)
+    assert math.isclose(combined_probs[1][0][0], (probs[0][0][0] + probs[1][0][0]) / 2, abs_tol=1e-4)
+
+    assert math.isclose(combined_probs[2][0][0], 0.1733, abs_tol=1e-4)
+    assert math.isclose(combined_probs[2][0][0], (probs[0][0][0] + probs[1][0][0] + probs[2][0][0]) / 3, abs_tol=1e-4)

From dacf667a8e3de67d01cf4be870b1c0f15820947b Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Mon, 4 Mar 2024 18:30:36 +0100
Subject: [PATCH 18/65] add GeometricMeanCombiner + test

---
 .../GeometricMeanCombiner.py                  | 22 +++++++++++++++++
 hiclass/probability_combiner/__init__.py      |  2 ++
 tests/test_ProbabilityCombiner.py             | 24 ++++++++++++++++++-
 3 files changed, 47 insertions(+), 1 deletion(-)
 create mode 100644 hiclass/probability_combiner/GeometricMeanCombiner.py

diff --git a/hiclass/probability_combiner/GeometricMeanCombiner.py b/hiclass/probability_combiner/GeometricMeanCombiner.py
new file mode 100644
index 00000000..bdf27063
--- /dev/null
+++ b/hiclass/probability_combiner/GeometricMeanCombiner.py
@@ -0,0 +1,22 @@
+import numpy as np
+from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
+
+class GeometricMeanCombiner(ProbabilityCombiner):
+    def combine(self, proba):
+        res = [proba[0]]
+        log_sum = [np.log(proba[0])]
+        for level in range(1, self.classifier.max_levels_):
+            
+            level_probs = np.zeros_like(proba[level])
+            level_log_sum = np.zeros_like(proba[level])
+            for node in self.classifier.classes_[level]:
+                predecessor = list(self.classifier.hierarchy_.predecessors(node))[0]
+                predecessor_index = self.classifier.class_to_index_mapping_[level-1][predecessor]
+                index = self.classifier.class_to_index_mapping_[level][node]
+
+                level_log_sum[:, index] += (np.log(proba[level][:, index]) + log_sum[level-1][:, predecessor_index])
+                level_probs[:, index] = np.exp(level_log_sum[:, index] / (level+1))
+
+            log_sum.append(level_log_sum)
+            res.append(level_probs)
+        return res
diff --git a/hiclass/probability_combiner/__init__.py b/hiclass/probability_combiner/__init__.py
index 808f4fbd..ca58dd3e 100644
--- a/hiclass/probability_combiner/__init__.py
+++ b/hiclass/probability_combiner/__init__.py
@@ -1,7 +1,9 @@
 from .MultiplyCombiner import MultiplyCombiner
 from .ArithmeticMeanCombiner import ArithmeticMeanCombiner
+from .GeometricMeanCombiner import GeometricMeanCombiner
 
 __all__ = [
     "MultiplyCombiner",
     "ArithmeticMeanCombiner",
+    "GeometricMeanCombiner",
 ]
\ No newline at end of file
diff --git a/tests/test_ProbabilityCombiner.py b/tests/test_ProbabilityCombiner.py
index 1531b8db..5a748127 100644
--- a/tests/test_ProbabilityCombiner.py
+++ b/tests/test_ProbabilityCombiner.py
@@ -4,9 +4,10 @@
 import networkx as nx
 from numpy.testing import assert_array_almost_equal, assert_array_equal
 import math
+from scipy.stats import gmean
 
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
-from hiclass.probability_combiner import MultiplyCombiner, ArithmeticMeanCombiner
+from hiclass.probability_combiner import MultiplyCombiner, ArithmeticMeanCombiner, GeometricMeanCombiner
 
 
 @pytest.fixture
@@ -90,3 +91,24 @@ def test_arithmetic_mean_combiner(one_sample_probs_with_hierarchy):
 
     assert math.isclose(combined_probs[2][0][0], 0.1733, abs_tol=1e-4)
     assert math.isclose(combined_probs[2][0][0], (probs[0][0][0] + probs[1][0][0] + probs[2][0][0]) / 3, abs_tol=1e-4)
+
+def test_geometric_mean_combiner(one_sample_probs_with_hierarchy):
+    hierarchy, probs, classes = one_sample_probs_with_hierarchy
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = classes
+    classifier.max_levels_ = len(classes)
+    classifier.class_to_index_mapping_ = [{classifier.classes_[level][index]: index for index in range(len(classifier.classes_[level]))} for level in range(classifier.max_levels_)]
+    classifier.hierarchy_ = hierarchy
+
+    combiner = GeometricMeanCombiner(classifier=classifier)
+    combined_probs = combiner.combine(probs)
+
+    # check combined probability of first node for both levels
+    assert math.isclose(combined_probs[1][0][0], 0.2387, abs_tol=1e-4)
+    assert math.isclose(combined_probs[1][0][0], gmean([probs[0][0][0], probs[1][0][0]]), abs_tol=1e-4)
+
+    assert math.isclose(combined_probs[2][0][0], 0.1195, abs_tol=1e-4)
+    assert math.isclose(combined_probs[2][0][0], gmean([probs[0][0][0], probs[1][0][0], probs[2][0][0]]), abs_tol=1e-4)
+

From 3be6b5576577b300c6b51209fb3c7ac480a38cd7 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Mon, 4 Mar 2024 18:42:38 +0100
Subject: [PATCH 19/65] add more test cases for ArithmeticMeanCombiner and
 GeometricMeanCombiner

---
 tests/test_ProbabilityCombiner.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/test_ProbabilityCombiner.py b/tests/test_ProbabilityCombiner.py
index 5a748127..c1525895 100644
--- a/tests/test_ProbabilityCombiner.py
+++ b/tests/test_ProbabilityCombiner.py
@@ -92,6 +92,13 @@ def test_arithmetic_mean_combiner(one_sample_probs_with_hierarchy):
     assert math.isclose(combined_probs[2][0][0], 0.1733, abs_tol=1e-4)
     assert math.isclose(combined_probs[2][0][0], (probs[0][0][0] + probs[1][0][0] + probs[2][0][0]) / 3, abs_tol=1e-4)
 
+    # check combined probability of last node for both levels
+    assert math.isclose(combined_probs[1][0][-1], 0.12, abs_tol=1e-4)
+    assert math.isclose(combined_probs[1][0][-1], (probs[0][0][-1] + probs[1][0][-1]) / 2, abs_tol=1e-4)
+    
+    assert math.isclose(combined_probs[2][0][-1], 0.0833, abs_tol=1e-4)
+    assert math.isclose(combined_probs[2][0][-1], (probs[0][0][-1] + probs[1][0][-1] + probs[2][0][-1]) / 3, abs_tol=1e-4)
+
 def test_geometric_mean_combiner(one_sample_probs_with_hierarchy):
     hierarchy, probs, classes = one_sample_probs_with_hierarchy
     obj = HierarchicalClassifier()
@@ -112,3 +119,9 @@ def test_geometric_mean_combiner(one_sample_probs_with_hierarchy):
     assert math.isclose(combined_probs[2][0][0], 0.1195, abs_tol=1e-4)
     assert math.isclose(combined_probs[2][0][0], gmean([probs[0][0][0], probs[1][0][0], probs[2][0][0]]), abs_tol=1e-4)
 
+    # check combined probability of last node for both levels
+    assert math.isclose(combined_probs[1][0][-1], 0.0894, abs_tol=1e-4)
+    assert math.isclose(combined_probs[1][0][-1], gmean([probs[0][0][-1], probs[1][0][-1]]), abs_tol=1e-4)
+    
+    assert math.isclose(combined_probs[2][0][-1], 0.0430, abs_tol=1e-4)
+    assert math.isclose(combined_probs[2][0][-1], gmean([probs[0][0][-1], probs[1][0][-1], probs[2][0][-1]]), abs_tol=1e-4)

From 8f2fa73757fdd379d509bf25ab7813f632096db8 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Tue, 5 Mar 2024 14:46:23 +0100
Subject: [PATCH 20/65] enable multithreaded calibration

---
 hiclass/HierarchicalClassifier.py             | 65 ++++++++++++-------
 hiclass/LocalClassifierPerNode.py             |  6 +-
 .../ArithmeticMeanCombiner.py                 |  1 -
 tests/test_metrics.py                         |  1 -
 4 files changed, 42 insertions(+), 31 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index fc428a63..dd2348fe 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -171,10 +171,6 @@ def _pre_fit(self, X, y, sample_weight):
         # Create and configure logger
         self._create_logger()
 
-        # Avoids creating more columns in prediction if edges are a->b and b->c,
-        # which would generate the prediction a->b->c
-        #self.y_ = self._disambiguate(self.y_)
-
         # Create DAG from self.y_ and store to self.hierarchy_
         self._create_digraph()
 
@@ -402,43 +398,62 @@ def _remove_separator(self, y):
     def _fit_node_classifier(
         self, nodes, local_mode: bool = False, use_joblib: bool = False
     ):
+        def logging_wrapper(func, idx, node, node_length):
+            self.logger_.info(f"fitting node {idx+1}/{node_length}: {str(node)}")
+            return func(self, node)
+        
         if self.n_jobs > 1:
             if _has_ray and not use_joblib:
-                ray.init(
-                    num_cpus=self.n_jobs,
-                    local_mode=local_mode,
-                    ignore_reinit_error=True,
-                )
+                if not ray.is_initialized:
+                    ray.init(
+                        num_cpus=self.n_jobs,
+                        local_mode=local_mode,
+                        ignore_reinit_error=True,
+                    )
                 lcppn = ray.put(self)
-                _parallel_fit = ray.remote(self._fit_classifier)
+                _parallel_fit = ray.remote(self._fit_classifier) # TODO: use logging wrapper
                 results = [_parallel_fit.remote(lcppn, node) for node in nodes]
                 classifiers = ray.get(results)
             else:
                 classifiers = Parallel(n_jobs=self.n_jobs)(
-                    delayed(self._fit_classifier)(self, node) for node in nodes
+                    delayed(logging_wrapper)(self._fit_classifier, idx, node, len(nodes)) for idx, node in enumerate(nodes)
                 )
 
         else:
-            classifiers = []
-            for idx, node in enumerate(nodes):
-                self.logger_.info(f"fitting node {idx+1}/{len(nodes)}: {str(node)}")
-                classifiers.append(self._fit_classifier(self, node))
-            #classifiers = [self._fit_classifier(self, node) for node in nodes]
+            classifiers = [logging_wrapper(self._fit_classifier, idx, node, len(nodes)) for idx, node in enumerate(nodes)]
+
         for classifier, node in zip(classifiers, nodes):
             self.hierarchy_.nodes[node]["classifier"] = classifier
 
     def _fit_node_calibrator(self, nodes, local_mode: bool = False, use_joblib: bool = False):
-        # TODO: add support for multithreading
-        #calibrators = [self._fit_calibrator(self, node) for node in nodes]
-        calibrators = []
-        for idx, node in enumerate(nodes):
-            self.logger_.info(f"calibrating node {idx+1}/{len(nodes)}: {str(node)}")
-            calibrators.append(self._fit_calibrator(self, node))
+        def logging_wrapper(func, idx, node, node_length):
+            self.logger_.info(f"calibrating node {idx+1}/{node_length}: {str(node)}")
+            return func(self, node)
 
-        for calibrator, node in zip(calibrators, nodes):
-            if calibrator:
-                self.hierarchy_.nodes[node]["calibrator"] = calibrator
+        if self.n_jobs > 1:
+            if _has_ray and not use_joblib:
+                if not ray.is_initialized:
+                    ray.init(
+                        num_cpus=self.n_jobs,
+                        local_mode=local_mode,
+                        ignore_reinit_error=True,
+                    )
+                lcppn = ray.put(self)
+                _parallel_fit = ray.remote(self._fit_calibrator)
+                results = [_parallel_fit.remote(lcppn, node) for idx, node in enumerate(nodes)] # TODO: use logging wrapper
+                calibrators = ray.get(results)
+                ray.shutdown()
+            else:
+                calibrators = Parallel(n_jobs=self.n_jobs)(
+                    delayed(logging_wrapper)(self._fit_calibrator, idx, node, len(nodes)) for idx, node in enumerate(nodes)
+                )
 
+        else:
+            calibrators = [logging_wrapper(self._fit_calibrator, idx, node, len(nodes)) for idx, node in enumerate(nodes)]
+
+        for calibrator, node in zip(calibrators, nodes):
+            self.hierarchy_.nodes[node]["calibrator"] = calibrator
+        
     @staticmethod
     def _fit_classifier(self, node):
         raise NotImplementedError("Method should be implemented in the LCPN and LCPPN")
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index 913bdf79..86114f30 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -267,7 +267,7 @@ def predict_proba(self, X):
                     self.logger_.info(f"Predicting probabilities for node '{str(successor)}'")
                     classifier = self.hierarchy_.nodes[successor]["classifier"]
                     # use classifier as a fallback if no calibrator is available
-                    calibrator = self.hierarchy_.nodes[successor].get("calibrator", classifier)
+                    calibrator = self.hierarchy_.nodes[successor].get("calibrator", classifier) or classifier
                     positive_index = np.where(calibrator.classes_ == 1)[0]
                     proba = calibrator.predict_proba(subset_x)[:, positive_index][:, 0]
                     local_probabilities[:, i] = proba
@@ -360,7 +360,6 @@ def _calibrate_digraph(self, local_mode: bool = False, use_joblib: bool = False)
     def _fit_classifier(self, node):
         classifier = self.hierarchy_.nodes[node]["classifier"]
         X, y, sample_weight = self.binary_policy_.get_binary_examples(node)
-        #self.logger_.info("fitting model " + "node: " + str(node) + " " + str(X.shape) + " : " + str(y.shape) + " unique labels: " + str(len(np.unique(y))))
         unique_y = np.unique(y)
         if len(unique_y) == 1 and self.replace_classifiers:
             self.logger_.info("adding constant classifier")
@@ -382,10 +381,9 @@ def _fit_calibrator(self, node):
             self.logger_.info("no calibrator for " + "node: " + str(node))
             return None
         X, y, _ = self.cal_binary_policy_.get_binary_examples(node)
-        #self.logger_.info("fitting calibrator " + "node: " + str(node) + " " + str(X.shape) + " : " + str(y.shape))
         if len(y) == 0 or len(np.unique(y)) < 2:
             self.logger_.info(f"No calibration samples to fit calibrator for node: {str(node)}")
-            self.hierarchy_.nodes[node].pop('calibrator', None)
+            #self.hierarchy_.nodes[node].pop('calibrator', None)
             return None
         calibrator.fit(X, y)
         return calibrator
diff --git a/hiclass/probability_combiner/ArithmeticMeanCombiner.py b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
index d579a4af..b6a9ef4e 100644
--- a/hiclass/probability_combiner/ArithmeticMeanCombiner.py
+++ b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
@@ -6,7 +6,6 @@ def combine(self, proba):
         res = [proba[0]]
         sums = [proba[0]]
         for level in range(1, self.classifier.max_levels_):
-            print(level)
             level_probs = np.zeros_like(proba[level])
             level_sum = np.zeros_like(proba[level])
             for node in self.classifier.classes_[level]:
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index 331787e9..5e6380f7 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -367,7 +367,6 @@ def test_local_brier_score(uncertainty_data):
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
     classifier.classes_ = [[0, 1, 2]]
-    classifier.hierarchy_ = nx.DiGraph()
     brier_score = _multiclass_brier_score(classifier, y_true, prob, level=0)
     assert math.isclose(brier_score, 0.34852, abs_tol=1e-4)
 

From d59fea4c5d9a9d1f5ff8c83588cb089c901c5dc7 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Tue, 5 Mar 2024 19:07:58 +0100
Subject: [PATCH 21/65] add custom Pipeline to support calibration step

---
 hiclass/Pipeline.py | 14 ++++++++++++++
 hiclass/__init__.py |  2 ++
 2 files changed, 16 insertions(+)
 create mode 100644 hiclass/Pipeline.py

diff --git a/hiclass/Pipeline.py b/hiclass/Pipeline.py
new file mode 100644
index 00000000..e347c4c6
--- /dev/null
+++ b/hiclass/Pipeline.py
@@ -0,0 +1,14 @@
+from sklearn.pipeline import Pipeline as skPipeline
+
+class Pipeline(skPipeline):
+    def __init__(self, steps, **kwargs):
+        super().__init__(steps, **kwargs)
+
+    def calibrate(self, X, y, **params):
+        """Transform the data and apply `calibrate` with the final estimator.
+
+        """
+        Xt = X
+        for _, name, transform in self._iter(with_final=False):
+            Xt = transform.transform(Xt)
+        return self.steps[-1][1].calibrate(Xt, y)
diff --git a/hiclass/__init__.py b/hiclass/__init__.py
index 6b199516..09f482ae 100644
--- a/hiclass/__init__.py
+++ b/hiclass/__init__.py
@@ -4,6 +4,7 @@
 from .LocalClassifierPerNode import LocalClassifierPerNode
 from .LocalClassifierPerParentNode import LocalClassifierPerParentNode
 from .LocalClassifierPerLevel import LocalClassifierPerLevel
+from .Pipeline import Pipeline
 
 __version__ = get_versions()["version"]
 del get_versions
@@ -12,4 +13,5 @@
     "LocalClassifierPerNode",
     "LocalClassifierPerParentNode",
     "LocalClassifierPerLevel",
+    "Pileline",
 ]

From 85f85540d48ddde2fa2e4c5176b4cb435d71285f Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Tue, 5 Mar 2024 20:17:41 +0100
Subject: [PATCH 22/65] add ECE, SCE and ACE calibration metrics + tests

---
 hiclass/metrics.py    | 159 ++++++++++++++++++++++++++++++++++++++----
 tests/test_metrics.py |  45 +++++++++++-
 2 files changed, 187 insertions(+), 17 deletions(-)

diff --git a/hiclass/metrics.py b/hiclass/metrics.py
index 743f2187..94b7b3b7 100644
--- a/hiclass/metrics.py
+++ b/hiclass/metrics.py
@@ -252,23 +252,17 @@ def _compute_macro(y_true: np.ndarray, y_pred: np.ndarray, _micro_function):
     return overall_sum / len(y_true)
 
 
-def _multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, level: int):
-    y_true, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level)
-    label_encoder = LabelEncoder()
-    label_encoder.fit(labels)
-    y_true_encoded = label_encoder.transform(y_true)
-    return (1/y_prob.shape[0]) * np.sum(np.sum(np.square(y_prob - np.eye(y_prob.shape[1])[y_true_encoded]), axis=1))
-
-
-def _log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, level: int):
-    y_true, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level)
-    return log_loss(y_true, y_prob, labels=labels)
-
-def _prepare_data(classifier, y_true, y_prob, level):
+def _prepare_data(classifier, y_true, y_prob, level, y_pred=None):
     classifier_classes = np.array(classifier.classes_[level]).astype("str")
     y_true = make_leveled(y_true)
     y_true = classifier._disambiguate(y_true)
     y_true = np.array(list(map(lambda x: x[level], y_true)))
+
+    if y_pred is not None:
+        y_pred = make_leveled(y_pred)
+        y_pred = classifier._disambiguate(y_pred)
+        y_pred = np.array(list(map(lambda x: x[level], y_pred)))
+
     unique_labels = np.unique(y_true).astype("str")
     # add labels not seen in the training process
     new_labels = np.sort(np.union1d(unique_labels, classifier_classes))
@@ -280,4 +274,141 @@ def _prepare_data(classifier, y_true, y_prob, level):
             old_idx = np.where(classifier_classes == label)[0][0]
             new_y_prob[:, idx] = y_prob[level][:, old_idx]
     
-    return y_true, new_labels, new_y_prob
+    return y_true, y_pred, new_labels, new_y_prob
+
+def _multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, level: int):
+    y_true, _, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level)
+    label_encoder = LabelEncoder()
+    label_encoder.fit(labels)
+    y_true_encoded = label_encoder.transform(y_true)
+    return (1/y_prob.shape[0]) * np.sum(np.sum(np.square(y_prob - np.eye(y_prob.shape[1])[y_true_encoded]), axis=1))
+
+def _log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, level: int):
+    y_true, _, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level)
+    return log_loss(y_true, y_prob, labels=labels)
+
+def _expected_calibration_error(classifier: HierarchicalClassifier, y_true, y_prob, y_pred, level, n_bins=10):
+    y_true, y_pred, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level, y_pred)
+
+    n = len(y_true)
+    label_encoder = LabelEncoder()
+    label_encoder.fit(labels)
+
+    y_true_encoded = label_encoder.transform(y_true)
+    y_pred_encoded = label_encoder.transform(y_pred)
+    
+    y_prob = np.max(y_prob, axis=1)
+    stacked = np.column_stack([y_prob, y_pred_encoded, y_true_encoded])
+
+    # calculate equally sized bins
+    _, bin_edges = np.histogram(stacked, bins=n_bins, range=(0,1))
+    bin_indices = np.digitize(stacked, bin_edges)[:, 0]
+
+    # add bin index to each data point
+    data = np.column_stack([stacked, bin_indices])
+
+    # create bin mask
+    masks = (data[:, -1, None] == range(1, n_bins+1)).T
+
+    # create actual bins
+    bins = [data[masks[i]] for i in range(n_bins)]
+
+    # calculate ECE
+    acc = np.zeros(n_bins)
+    conf = np.zeros(n_bins)
+    ece = 0
+    for i in range(n_bins):
+        acc[i] = 1 / (bins[i].shape[0]) * np.sum((bins[i][:, 1] == bins[i][:, 2])) if bins[i].shape[0] != 0 else 0
+        conf[i] = 1 / (bins[i].shape[0]) * np.sum(bins[i][:, 0]) if bins[i].shape[0] != 0 else 0
+        ece += (bins[i].shape[0] / n) * abs(acc[i] - conf[i]) if bins[i].shape[0] != 0 else 0
+    return ece
+
+def _statistical_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins=10):
+    y_true, y_pred, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level, y_pred)
+
+    n_samples, n_classes = y_prob.shape
+    assert n_classes > 2
+
+    label_encoder = LabelEncoder()
+    label_encoder.fit(labels)
+
+    y_true_encoded = label_encoder.transform(y_true)
+    y_pred_encoded = label_encoder.transform(y_pred)
+
+    class_error = np.zeros(n_classes)
+
+    for k in range(n_classes):
+        class_scores = y_prob[:, k]
+        stacked = np.column_stack([class_scores, y_pred_encoded, y_true_encoded])
+
+        # create bins
+        _, bin_edges = np.histogram(stacked, bins=n_bins, range=(0,1))
+        bin_indices = np.digitize(stacked, bin_edges)[:, 0]
+
+        # add bin index to each data point
+        data = np.column_stack([stacked, bin_indices])
+
+        # create bin mask
+        masks = (data[:, -1, None] == range(1, n_bins+1)).T
+        
+        # create actual bins
+        bins = [data[masks[i]] for i in range(n_bins)]
+
+        # calculate per class calibration error
+        acc = np.zeros(n_bins)
+        conf = np.zeros(n_bins)
+        error = 0
+        for i in range(n_bins):
+            acc[i] = 1 / (bins[i].shape[0]) * np.sum((bins[i][:, 1] == bins[i][:, 2])) if bins[i].shape[0] != 0 else 0
+            conf[i] = 1 / (bins[i].shape[0]) * np.sum(bins[i][:, 0]) if bins[i].shape[0] != 0 else 0
+            error += (bins[i].shape[0] / n_samples) * abs(acc[i] - conf[i]) if bins[i].shape[0] != 0 else 0
+        
+        class_error[k] = error
+            
+    return np.mean(class_error)
+
+def _adaptive_calibration_error(classifier, y_true, y_prob, y_pred, level, n_ranges=10):
+    y_true, y_pred, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level, y_pred)
+
+    _, n_classes = y_prob.shape
+    label_encoder = LabelEncoder()
+    label_encoder.fit(labels)
+
+    y_true_encoded = label_encoder.transform(y_true)
+    y_pred_encoded = label_encoder.transform(y_pred)
+
+    class_error = np.zeros(n_classes)
+
+    for k in range(n_classes):
+        class_scores = y_prob[:, k]
+
+        # sort by score probability
+        idx = np.argsort([class_scores])[0]
+        class_scores, ordered_y_pred_labels, ordered_y_true = class_scores[idx], y_pred_encoded[idx], y_true_encoded[idx]
+        stacked = np.column_stack([np.array(range(len(class_scores))), class_scores, ordered_y_pred_labels, ordered_y_true])
+
+        bin_edges = np.floor(np.linspace(0, len(class_scores), n_ranges+1, endpoint=True)).astype(int)
+        _, bin_edges = np.histogram(stacked, bins=bin_edges, range=(0,len(class_scores)))
+        bin_indices = np.digitize(stacked, bin_edges)[:, 0]
+
+        # add bin index to each data point
+        data = np.column_stack([stacked, bin_indices])
+
+        # create bin mask
+        masks = (data[:, -1, None] == range(1, n_ranges+1)).T
+
+        # create actual bins
+        bins = [data[masks[i]] for i in range(n_ranges)]
+
+        # calculate per class calibration error
+        acc = np.zeros(n_ranges)
+        conf = np.zeros(n_ranges)
+        error = 0
+        for i in range(n_ranges):
+            acc[i] = 1 / (bins[i].shape[0]) * np.sum((bins[i][:, 2] == bins[i][:, 3])) if bins[i].shape[0] != 0 else 0
+            conf[i] = 1 / (bins[i].shape[0]) * np.sum(bins[i][:, 1]) if bins[i].shape[0] != 0 else 0
+            error += abs(acc[i] - conf[i]) if bins[i].shape[0] != 0 else 0
+        
+        class_error[k] = error
+
+    return (1/(n_classes*n_ranges)) * np.sum(class_error)
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index 5e6380f7..b8d69b00 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -4,10 +4,18 @@
 import math
 from numpy.testing import assert_array_almost_equal, assert_array_equal
 from unittest.mock import Mock
-import networkx as nx
 
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
-from hiclass.metrics import precision, recall, f1, _multiclass_brier_score, _log_loss
+from hiclass.metrics import (
+    precision, 
+    recall, 
+    f1, 
+    _multiclass_brier_score, 
+    _log_loss,
+    _expected_calibration_error,
+    _statistical_calibration_error,
+    _adaptive_calibration_error
+)
 
 
 # TODO: add tests for 3D dataframe (not sure if it's possible to have 3D dataframes)
@@ -356,7 +364,7 @@ def uncertainty_data():
 
     assert_array_equal(np.sum(prob[0], axis=1), np.ones(len(prob[0])))
 
-    y_pred = np.array([0, 1, 2, 0, 2, 1, 0, 1, 0, 0])
+    y_pred = np.array([[0], [1], [2], [0], [2], [1], [0], [1], [0], [0]])
     y_true = np.array([[0], [2], [0], [0], [2], [1], [1], [1], [0], [0]])
 
     return prob, y_pred, y_true
@@ -367,6 +375,7 @@ def test_local_brier_score(uncertainty_data):
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
     classifier.classes_ = [[0, 1, 2]]
+
     brier_score = _multiclass_brier_score(classifier, y_true, prob, level=0)
     assert math.isclose(brier_score, 0.34852, abs_tol=1e-4)
 
@@ -376,6 +385,36 @@ def test_local_log_loss(uncertainty_data):
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
     classifier.classes_ = [[0, 1, 2]]
+
     log_loss = _log_loss(classifier, y_true, prob, level=0)
     assert math.isclose(log_loss, 0.61790, abs_tol=1e-4)
 
+def test_expected_calibration_error(uncertainty_data):
+    prob, y_pred, y_true = uncertainty_data
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = [[0, 1, 2]]
+
+    ece = _expected_calibration_error(classifier, y_true, prob, y_pred, level=0, n_bins=3)
+    assert math.isclose(ece, 0.118, abs_tol=1e-4)
+
+def test_statistical_calibration_error(uncertainty_data):
+    prob, y_pred, y_true = uncertainty_data
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = [[0, 1, 2]]
+
+    sce = _statistical_calibration_error(classifier, y_true, prob, y_pred, level=0, n_bins=3)
+    assert math.isclose(sce, 0.3889, abs_tol=1e-3)
+
+def test_adaptive_calibration_error(uncertainty_data):
+    prob, y_pred, y_true = uncertainty_data
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = [[0, 1, 2]]
+
+    ace = _adaptive_calibration_error(classifier, y_true, prob, y_pred, level=0, n_ranges=3)
+    assert math.isclose(ace, 0.44, abs_tol=1e-3)

From 918f7343bb3cc0c27bbc1379e7ec27049e4e529c Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Tue, 5 Mar 2024 21:24:07 +0100
Subject: [PATCH 23/65] fix misspelling

---
 hiclass/__init__.py                      | 2 +-
 hiclass/probability_combiner/__init__.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/hiclass/__init__.py b/hiclass/__init__.py
index 09f482ae..f9f4cf1e 100644
--- a/hiclass/__init__.py
+++ b/hiclass/__init__.py
@@ -13,5 +13,5 @@
     "LocalClassifierPerNode",
     "LocalClassifierPerParentNode",
     "LocalClassifierPerLevel",
-    "Pileline",
+    "Pipeline",
 ]
diff --git a/hiclass/probability_combiner/__init__.py b/hiclass/probability_combiner/__init__.py
index ca58dd3e..c10dfa32 100644
--- a/hiclass/probability_combiner/__init__.py
+++ b/hiclass/probability_combiner/__init__.py
@@ -6,4 +6,4 @@
     "MultiplyCombiner",
     "ArithmeticMeanCombiner",
     "GeometricMeanCombiner",
-]
\ No newline at end of file
+]

From 33e07647491d7e3ebf336e191582b374e26f51b2 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Wed, 6 Mar 2024 20:19:31 +0100
Subject: [PATCH 24/65] add support for LocalClassifierPerParentNode

---
 hiclass/HierarchicalClassifier.py       |   8 +-
 hiclass/LocalClassifierPerNode.py       |   5 +-
 hiclass/LocalClassifierPerParentNode.py | 105 ++++++++++++++++++++++--
 hiclass/_calibration/Calibrator.py      |   3 +-
 4 files changed, 109 insertions(+), 12 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index 92a1d747..2e6ee9ec 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -232,7 +232,7 @@ def calibrate(self, X, y):
             self.y_cross_val = np.vstack([self.y_, y])
             self.y_cross_val = make_leveled(self.y_cross_val)
             self.y_cross_val = self._disambiguate(self.y_cross_val)
-            self.y_cross_val = self._convert_1d_y_to_2d(self.y_cross_val)
+            self.y_cross_val = self._convert_1d_y_to_2d(self.y_cross_val) # TODO: rename to y_cal?
         else:
             self.X_cal = X
             self.y_cal = y
@@ -241,7 +241,6 @@ def calibrate(self, X, y):
             self.y_cal = self._disambiguate(self.y_cal)
             self.y_cal = self._convert_1d_y_to_2d(self.y_cal)
 
-        self.cal_binary_policy_ = self._initialize_binary_policy(calibration=True)
         self.logger_.info("Calibrating")
 
         # Create a calibrator for each local classifier
@@ -379,9 +378,8 @@ def _initialize_local_classifiers(self):
         else:
             self.local_classifier_ = self.local_classifier
 
-        @abc.abstractmethod
-        def _initialize_local_calibrators(self):
-            raise NotImplementedError("Method should be implemented in the LCPN and LCPPN")
+    def _initialize_local_calibrators(self):
+        self.logger_.info("Initializing local calibrators")
 
     def _convert_to_1d(self, y):
         # Convert predictions to 1D if there is only 1 column
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index 46c03cc8..5bd746f2 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -46,7 +46,7 @@ def __init__(
         n_jobs: int = 1,
         bert: bool = False,
         calibration_method: str = None,
-        return_all_probabilities: bool = False,
+        return_all_probabilities: bool = False
     ):
         """
         Initialize a local classifier per node.
@@ -333,6 +333,7 @@ def _initialize_local_classifiers(self):
         nx.set_node_attributes(self.hierarchy_, local_classifiers)
 
     def _initialize_local_calibrators(self):
+        super()._initialize_local_calibrators()
         local_calibrators = {}
         for node in self.hierarchy_.nodes:
             # Skip only root node
@@ -352,6 +353,8 @@ def _fit_digraph(self, local_mode: bool = False, use_joblib: bool = False):
         self._fit_node_classifier(nodes, local_mode, use_joblib)
 
     def _calibrate_digraph(self, local_mode: bool = False, use_joblib: bool = False):
+        self.logger_.info("Fitting local calibrators")
+        self.cal_binary_policy_ = self._initialize_binary_policy(calibration=True)
         nodes = list(self.hierarchy_.nodes)
         # Remove root because it does not need to be fitted
         nodes.remove(self.root_)
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index c7c0a62f..bb0b36f2 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -13,6 +13,7 @@
 
 from hiclass.ConstantClassifier import ConstantClassifier
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
+from hiclass._calibration.Calibrator import _Calibrator
 
 
 class LocalClassifierPerParentNode(BaseEstimator, HierarchicalClassifier):
@@ -43,6 +44,7 @@ def __init__(
         n_jobs: int = 1,
         bert: bool = False,
         calibration_method: str = None,
+        return_all_probabilities: bool = False
     ):
         """
         Initialize a local classifier per parent node.
@@ -68,6 +70,8 @@ def __init__(
             If True, skip scikit-learn's checks and sample_weight passing for BERT.
         calibration_method : {"ivap", "cvap", "platt", "isotonic"}, str, default=None
             If set, use the desired method to calibrate probabilities returned by predict_proba().
+        return_all_probabilities : bool, default=False
+            If True, return probabilities for all levels. Otherwise, return only probabilities for the last level.
         """
         super().__init__(
             local_classifier=local_classifier,
@@ -79,6 +83,7 @@ def __init__(
             bert=bert,
             calibration_method=calibration_method,
         )
+        self.return_all_probabilities = return_all_probabilities
 
     def fit(self, X, y, sample_weight=None):
         """
@@ -160,7 +165,66 @@ def predict(self, X):
         self._remove_separator(y)
 
         return y
+    
+    def predict_proba(self, X):
+        # Check if fit has been called
+        check_is_fitted(self)
+
+        # Input validation
+        if not self.bert:
+            X = check_array(X, accept_sparse="csr", allow_nd=True, ensure_2d=False)
+        else:
+            X = np.array(X)
+
+        if not self.calibration_method:
+            self.logger_.info("It is not recommended to use predict_proba() without calibration")
+        
+        self.logger_.info("Predicting Probability")
+
+        # Initialize array that holds predictions
+        y = np.empty((X.shape[0], self.max_levels_), dtype=self.dtype_)
 
+        # Predict first level
+        classifier = self.hierarchy_.nodes[self.root_]["classifier"]
+        # use classifier as a fallback if no calibrator is available
+        calibrator = self.hierarchy_.nodes[self.root_].get("calibrator", classifier) or classifier
+        proba = calibrator.predict_proba(X)
+
+        y[:, 0] = calibrator.classes_[np.argmax(proba, axis=1)]
+        level_probability_list = [proba] + self._predict_proba_remaining_levels(X, y)
+
+        return level_probability_list if self.return_all_probabilities else level_probability_list[-1]
+
+    def _predict_proba_remaining_levels(self, X, y):
+        level_probability_list = []
+        for level in range(1, y.shape[1]):
+                predecessors = set(y[:, level - 1])
+                predecessors.discard("")
+                level_dimension = self.max_level_dimensions_[level]
+                cur_level_probabilities = np.zeros((X.shape[0], level_dimension))
+
+                for predecessor in predecessors:
+                    mask = np.isin(y[:, level - 1], predecessor)
+                    predecessor_x = X[mask]
+                    if predecessor_x.shape[0] > 0:
+                        successors = list(self.hierarchy_.successors(predecessor))
+                        if len(successors) > 0:
+                            classifier = self.hierarchy_.nodes[predecessor]["classifier"]
+                            # use classifier as a fallback if no calibrator is available
+                            calibrator = self.hierarchy_.nodes[predecessor].get("calibrator", classifier) or classifier
+
+                            proba = calibrator.predict_proba(predecessor_x)
+                            y[mask, level] = calibrator.classes_[np.argmax(proba, axis=1)]
+
+                            for successor in successors:
+                                class_index = self.class_to_index_mapping_[level][str(successor)]
+
+                                proba_index = np.where(calibrator.classes_ == successor)[0][0]
+                                cur_level_probabilities[mask, class_index] = proba[:, proba_index]
+
+                level_probability_list.append(cur_level_probabilities)
+        return level_probability_list
+        
     def _predict_remaining_levels(self, X, y):
         for level in range(1, y.shape[1]):
             predecessors = set(y[:, level - 1])
@@ -182,6 +246,17 @@ def _initialize_local_classifiers(self):
             local_classifiers[node] = {"classifier": deepcopy(self.local_classifier_)}
         nx.set_node_attributes(self.hierarchy_, local_classifiers)
 
+    def _initialize_local_calibrators(self):
+        super()._initialize_local_calibrators()
+        local_calibrators = {}
+        nodes = self._get_parents()
+        for node in nodes:
+            local_classifier = self.hierarchy_.nodes[node]["classifier"]
+            local_calibrators[node] = {
+                "calibrator": _Calibrator(estimator=local_classifier, method=self.calibration_method)
+            }
+        nx.set_node_attributes(self.hierarchy_, local_calibrators)
+
     def _get_parents(self):
         nodes = []
         for node in self.hierarchy_.nodes:
@@ -191,18 +266,19 @@ def _get_parents(self):
                 nodes.append(node)
         return nodes
 
-    def _get_successors(self, node):
+    def _get_successors(self, node, calibration=False):
         successors = list(self.hierarchy_.successors(node))
-        mask = np.isin(self.y_, successors).any(axis=1)
-        X = self.X_[mask]
+        mask = np.isin(self.y_cal, successors).any(axis=1) if calibration else np.isin(self.y_, successors).any(axis=1)
+        X = self.X_cal[mask] if calibration else self.X_[mask]
         y = []
-        for row in self.y_[mask]:
+        masked_labels = self.y_cal[mask] if calibration else self.y_[mask]
+        for row in masked_labels:
             if node == self.root_:
                 y.append(row[0])
             else:
                 y.append(row[np.where(row == node)[0][0] + 1])
         y = np.array(y)
-        sample_weight = (
+        sample_weight = None if calibration else (
             self.sample_weight_[mask] if self.sample_weight_ is not None else None
         )
         return X, y, sample_weight
@@ -224,7 +300,26 @@ def _fit_classifier(self, node):
             classifier.fit(X, y)
         return classifier
 
+    @staticmethod
+    def _fit_calibrator(self, node):
+        try:
+            calibrator = self.hierarchy_.nodes[node]["calibrator"]
+        except KeyError:
+            self.logger_.info("no calibrator for " + "node: " + str(node))
+            return None
+        X, y, _ = self._get_successors(node, calibration=True)
+        if len(y) == 0 or len(np.unique(y)) < 2:
+            self.logger_.info(f"No calibration samples to fit calibrator for node: {str(node)}")
+            return None
+        calibrator.fit(X, y)
+        return calibrator
+        
     def _fit_digraph(self, local_mode: bool = False, use_joblib: bool = False):
         self.logger_.info("Fitting local classifiers")
         nodes = self._get_parents()
         self._fit_node_classifier(nodes, local_mode, use_joblib)
+    
+    def _calibrate_digraph(self, local_mode: bool = False, use_joblib: bool = False):
+        self.logger_.info("Fitting local calibrators")
+        nodes = self._get_parents()
+        self._fit_node_calibrator(nodes, local_mode, use_joblib)
diff --git a/hiclass/_calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
index ad9b87ed..de9c010d 100644
--- a/hiclass/_calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -48,7 +48,8 @@ def fit(self, X, y):
         if self.multiclass:
             # binarize multiclass labels
             label_binarizer = LabelBinarizer(sparse_output=False)
-            binary_labels = label_binarizer.fit_transform(y).T
+            label_binarizer.fit(self.estimator.classes_)
+            binary_labels = label_binarizer.transform(y).T
 
             # split scores into k one vs rest splits
             score_splits = [calibration_scores[:, i] for i in range(calibration_scores.shape[1])]

From 8ff021f0e61b8eb40dde81bd66bcea5cd6736f0a Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sat, 9 Mar 2024 18:04:51 +0100
Subject: [PATCH 25/65] add predict_proba to all model types, change output to
 full probability distribution

---
 hiclass/LocalClassifierPerLevel.py      | 86 +++++++++++++++++++++++++
 hiclass/LocalClassifierPerNode.py       |  3 +-
 hiclass/LocalClassifierPerParentNode.py | 10 ++-
 3 files changed, 97 insertions(+), 2 deletions(-)

diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index 34b5aa4f..74b711e5 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -13,6 +13,7 @@
 
 from hiclass.ConstantClassifier import ConstantClassifier
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
+from hiclass._calibration.Calibrator import _Calibrator
 
 try:
     import ray
@@ -50,6 +51,7 @@ def __init__(
         n_jobs: int = 1,
         bert: bool = False,
         calibration_method: str = None,
+        return_all_probabilities: bool = False
     ):
         """
         Initialize a local classifier per level.
@@ -75,6 +77,8 @@ def __init__(
             If True, skip scikit-learn's checks and sample_weight passing for BERT.
         calibration_method : {"ivap", "cvap", "platt", "isotonic"}, str, default=None
             If set, use the desired method to calibrate probabilities returned by predict_proba().
+        return_all_probabilities : bool, default=False
+            If True, return probabilities for all levels. Otherwise, return only probabilities for the last level.
         """
         super().__init__(
             local_classifier=local_classifier,
@@ -86,6 +90,7 @@ def __init__(
             bert=bert,
             calibration_method=calibration_method,
         )
+        self.return_all_probabilities = return_all_probabilities
 
     def fit(self, X, y, sample_weight=None):
         """
@@ -167,6 +172,49 @@ def predict(self, X):
         self._remove_separator(y)
 
         return y
+    
+    def predict_proba(self, X):
+        # Check if fit has been called
+        check_is_fitted(self)
+
+        if not self.bert:
+            X = check_array(X, accept_sparse="csr", allow_nd=True, ensure_2d=False)
+        else:
+            X = np.array(X)
+
+        if not self.calibration_method:
+            self.logger_.info("It is not recommended to use predict_proba() without calibration")
+
+        # Initialize array that holds predictions
+        y = np.empty((X.shape[0], self.max_levels_), dtype=self.dtype_)
+
+        self.logger_.info("Predicting Probability")
+
+        # Predict first level
+        classifier = self.local_classifiers_[0]
+        calibrator = self.local_calibrators_[0]
+
+        # use classifier as a fallback if no calibrator is available
+        calibrator = calibrator or classifier
+        proba = calibrator.predict_proba(X)
+        y[:, 0] = calibrator.classes_[np.argmax(proba, axis=1)]
+
+        level_probability_list = [proba] + self._predict_proba_remaining_levels(X, y)
+
+        return level_probability_list if self.return_all_probabilities else level_probability_list[-1]
+
+    def _predict_proba_remaining_levels(self, X, y):
+        level_probability_list = []
+        for level in range(1, y.shape[1]):
+            classifier = self.local_classifiers_[level]
+            calibrator = self.local_calibrators_[level]
+            # use classifier as a fallback if no calibrator is available
+            calibrator = calibrator or classifier
+            probabilities = calibrator.predict_proba(X)
+            level_probability_list.append(probabilities)
+            # TODO: test with empty nodes, etc.
+        return level_probability_list
+
 
     def _predict_remaining_levels(self, X, y):
         for level in range(1, y.shape[1]):
@@ -217,6 +265,12 @@ def _initialize_local_classifiers(self):
             deepcopy(self.local_classifier_) for _ in range(self.y_.shape[1])
         ]
         self.masks_ = [None for _ in range(self.y_.shape[1])]
+    
+    def _initialize_local_calibrators(self):
+        super()._initialize_local_calibrators()
+        self.local_calibrators_ = [
+                _Calibrator(estimator=local_classifier, method=self.calibration_method) for local_classifier in self.local_classifiers_
+        ]
 
     def _fit_digraph(self, local_mode: bool = False, use_joblib: bool = False):
         self.logger_.info("Fitting local classifiers")
@@ -246,6 +300,20 @@ def _fit_digraph(self, local_mode: bool = False, use_joblib: bool = False):
             ]
         for level, classifier in enumerate(classifiers):
             self.local_classifiers_[level] = classifier
+    
+    def _calibrate_digraph(self, local_mode: bool = False, use_joblib: bool = False):
+        self.logger_.info("Fitting local calibrators")
+        if self.n_jobs > 1:
+            # TODO
+            pass
+        else:
+            calibrators = [
+                self._fit_calibrator(self, level, self.separator_)
+                for level in range(len(self.local_calibrators_))
+            ]
+        for level, calibrator in enumerate(calibrators):
+            self.local_calibrators_[level] = calibrator
+        
 
     @staticmethod
     def _fit_classifier(self, level, separator):
@@ -267,6 +335,24 @@ def _fit_classifier(self, level, separator):
             classifier.fit(X, y)
         return classifier
 
+    @staticmethod
+    def _fit_calibrator(self, level, separator):
+        try:
+            calibrator = self.local_calibrators_[level]
+        except:
+            self.logger_.info("no calibrator for " + "level: " + str(level))
+            return None
+        
+        X, y, _ = self._remove_empty_leaves(
+            separator, self.X_cal, self.y_cal[:, level], None
+        )
+        if len(y) == 0 or len(np.unique(y)) < 2:
+            self.logger_.info(f"No calibration samples to fit calibrator for level: {str(level)}")
+            return None
+
+        calibrator.fit(X, y)
+        return calibrator
+
     @staticmethod
     def _remove_empty_leaves(separator, X, y, sample_weight):
         # Detect rows where leaves are not empty
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index 5bd746f2..20c8285c 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -259,7 +259,8 @@ def predict_proba(self, X):
                 mask = [True] * X.shape[0]
                 subset_x = X[mask]
             else:
-                mask = np.isin(y, predecessor).any(axis=1)
+                #mask = np.isin(y, predecessor).any(axis=1)
+                mask = np.isin(y, self.classes_[level-1]).any(axis=1)
                 subset_x = X[mask]
 
             if subset_x.shape[0] > 0:
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index bb0b36f2..4c4d684c 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -204,7 +204,8 @@ def _predict_proba_remaining_levels(self, X, y):
                 cur_level_probabilities = np.zeros((X.shape[0], level_dimension))
 
                 for predecessor in predecessors:
-                    mask = np.isin(y[:, level - 1], predecessor)
+                    #mask = np.isin(y[:, level - 1], predecessor)
+                    mask = np.isin(y[:, level - 1], self.classes_[level-1])
                     predecessor_x = X[mask]
                     if predecessor_x.shape[0] > 0:
                         successors = list(self.hierarchy_.successors(predecessor))
@@ -223,6 +224,13 @@ def _predict_proba_remaining_levels(self, X, y):
                                 cur_level_probabilities[mask, class_index] = proba[:, proba_index]
 
                 level_probability_list.append(cur_level_probabilities)
+        
+        # normalize probabilities
+        
+        for level_probabilities in level_probability_list:
+            level_probabilities /= level_probabilities.sum(axis=1, keepdims=True)
+        
+
         return level_probability_list
         
     def _predict_remaining_levels(self, X, y):

From e7bbbf58c26333dc1c36288cf3420cb0f31ca873 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sat, 9 Mar 2024 22:27:57 +0100
Subject: [PATCH 26/65] fix test - make LocalClassifierPerLevel compatible with
 scikit learn

---
 hiclass/LocalClassifierPerLevel.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index 74b711e5..5bf6d4d8 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -115,6 +115,7 @@ def fit(self, X, y, sample_weight=None):
         """
         # Execute common methods necessary before fitting
         super()._pre_fit(X, y, sample_weight)
+        self.local_calibrators_ = None
 
         # Fit local classifiers in DAG
         super().fit(X, y)
@@ -192,7 +193,7 @@ def predict_proba(self, X):
 
         # Predict first level
         classifier = self.local_classifiers_[0]
-        calibrator = self.local_calibrators_[0]
+        calibrator = self.local_calibrators_[0] if self.local_calibrators_ else None
 
         # use classifier as a fallback if no calibrator is available
         calibrator = calibrator or classifier

From d124318be2eb1744ae6f460af90be03f95b7b2fd Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sat, 9 Mar 2024 22:58:28 +0100
Subject: [PATCH 27/65] add multithreading to LocalClassifierPerLevel

---
 hiclass/LocalClassifierPerLevel.py | 49 ++++++++++++++++++++++++------
 1 file changed, 39 insertions(+), 10 deletions(-)

diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index 5bf6d4d8..9f455ade 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -275,13 +275,19 @@ def _initialize_local_calibrators(self):
 
     def _fit_digraph(self, local_mode: bool = False, use_joblib: bool = False):
         self.logger_.info("Fitting local classifiers")
+
+        def logging_wrapper(func, level, separator, max_level):
+            self.logger_.info(f"fitting level {level+1}/{max_level}")
+            return func(self, level, separator)
+
         if self.n_jobs > 1:
             if _has_ray and not use_joblib:
-                ray.init(
-                    num_cpus=self.n_jobs,
-                    local_mode=local_mode,
-                    ignore_reinit_error=True,
-                )
+                if not ray.is_initialized:
+                    ray.init(
+                        num_cpus=self.n_jobs,
+                        local_mode=local_mode,
+                        ignore_reinit_error=True,
+                    )
                 lcpl = ray.put(self)
                 _parallel_fit = ray.remote(self._fit_classifier)
                 results = [
@@ -291,12 +297,12 @@ def _fit_digraph(self, local_mode: bool = False, use_joblib: bool = False):
                 classifiers = ray.get(results)
             else:
                 classifiers = Parallel(n_jobs=self.n_jobs)(
-                    delayed(self._fit_classifier)(self, level, self.separator_)
+                    delayed(logging_wrapper)(self._fit_classifier, level, self.separator_, len(self.local_classifiers_))
                     for level in range(len(self.local_classifiers_))
                 )
         else:
             classifiers = [
-                self._fit_classifier(self, level, self.separator_)
+                logging_wrapper(self._fit_classifier, level, self.separator_, len(self.local_classifiers_))
                 for level in range(len(self.local_classifiers_))
             ]
         for level, classifier in enumerate(classifiers):
@@ -304,14 +310,37 @@ def _fit_digraph(self, local_mode: bool = False, use_joblib: bool = False):
     
     def _calibrate_digraph(self, local_mode: bool = False, use_joblib: bool = False):
         self.logger_.info("Fitting local calibrators")
+
+        def logging_wrapper(func, level, separator, max_level):
+            self.logger_.info(f"calibrating level {level+1}/{max_level}")
+            return func(self, level, separator)
+
         if self.n_jobs > 1:
-            # TODO
-            pass
+            if _has_ray and not use_joblib:
+                if not ray.is_initialized:
+                    ray.init(
+                        num_cpus=self.n_jobs,
+                        local_mode=local_mode,
+                        ignore_reinit_error=True,
+                    )
+                lcpl = ray.put(self)
+                _parallel_fit = ray.remote(self._fit_calibrator)
+                results = [
+                    _parallel_fit.remote(lcpl, level, self.separator_)
+                    for level in range(len(self.local_calibrators_))
+                ]
+                calibrators = ray.get(results)
+                
+            else:
+                calibrators = Parallel(n_jobs=self.n_jobs)(
+                    delayed(logging_wrapper)(self._fit_calibrator, level, self.separator_, len(self.local_calibrators_)) for level in range(len(self.local_calibrators_))
+                )  
         else:
             calibrators = [
-                self._fit_calibrator(self, level, self.separator_)
+                logging_wrapper(self._fit_calibrator, level, self.separator_, len(self.local_calibrators_))
                 for level in range(len(self.local_calibrators_))
             ]
+        
         for level, calibrator in enumerate(calibrators):
             self.local_calibrators_[level] = calibrator
         

From f95981ee0b6c43b23be20c0f87f148dd63e8abc9 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sun, 10 Mar 2024 22:14:47 +0100
Subject: [PATCH 28/65] fix bug

---
 hiclass/LocalClassifierPerLevel.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index 9f455ade..3b6003eb 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -208,7 +208,7 @@ def _predict_proba_remaining_levels(self, X, y):
         level_probability_list = []
         for level in range(1, y.shape[1]):
             classifier = self.local_classifiers_[level]
-            calibrator = self.local_calibrators_[level]
+            calibrator = self.local_calibrators_[level] if self.local_calibrators_ else None
             # use classifier as a fallback if no calibrator is available
             calibrator = calibrator or classifier
             probabilities = calibrator.predict_proba(X)

From aad04a6405b4477bca3608192781e21e68c3227e Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Thu, 14 Mar 2024 16:47:46 +0100
Subject: [PATCH 29/65] fix error with different number of levels

---
 hiclass/probability_combiner/ArithmeticMeanCombiner.py | 7 ++++++-
 hiclass/probability_combiner/GeometricMeanCombiner.py  | 7 ++++++-
 hiclass/probability_combiner/MultiplyCombiner.py       | 7 ++++++-
 3 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/hiclass/probability_combiner/ArithmeticMeanCombiner.py b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
index b6a9ef4e..7c51b0a7 100644
--- a/hiclass/probability_combiner/ArithmeticMeanCombiner.py
+++ b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
@@ -1,4 +1,5 @@
 import numpy as np
+from networkx.exception import NetworkXError
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
 
 class ArithmeticMeanCombiner(ProbabilityCombiner):
@@ -9,7 +10,11 @@ def combine(self, proba):
             level_probs = np.zeros_like(proba[level])
             level_sum = np.zeros_like(proba[level])
             for node in self.classifier.classes_[level]:
-                predecessor = list(self.classifier.hierarchy_.predecessors(node))[0]
+                try:
+                    predecessor = list(self.classifier.hierarchy_.predecessors(node))[0]
+                except NetworkXError:
+                    # skip empty levels
+                    continue
                 predecessor_index = self.classifier.class_to_index_mapping_[level-1][predecessor]
                 index = self.classifier.class_to_index_mapping_[level][node]
 
diff --git a/hiclass/probability_combiner/GeometricMeanCombiner.py b/hiclass/probability_combiner/GeometricMeanCombiner.py
index bdf27063..d26b3ae3 100644
--- a/hiclass/probability_combiner/GeometricMeanCombiner.py
+++ b/hiclass/probability_combiner/GeometricMeanCombiner.py
@@ -1,4 +1,5 @@
 import numpy as np
+from networkx.exception import NetworkXError
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
 
 class GeometricMeanCombiner(ProbabilityCombiner):
@@ -10,7 +11,11 @@ def combine(self, proba):
             level_probs = np.zeros_like(proba[level])
             level_log_sum = np.zeros_like(proba[level])
             for node in self.classifier.classes_[level]:
-                predecessor = list(self.classifier.hierarchy_.predecessors(node))[0]
+                try:
+                    predecessor = list(self.classifier.hierarchy_.predecessors(node))[0]
+                except NetworkXError:
+                    # skip empty levels
+                    continue
                 predecessor_index = self.classifier.class_to_index_mapping_[level-1][predecessor]
                 index = self.classifier.class_to_index_mapping_[level][node]
 
diff --git a/hiclass/probability_combiner/MultiplyCombiner.py b/hiclass/probability_combiner/MultiplyCombiner.py
index 0a04c7ee..6204eca3 100644
--- a/hiclass/probability_combiner/MultiplyCombiner.py
+++ b/hiclass/probability_combiner/MultiplyCombiner.py
@@ -1,4 +1,5 @@
 import numpy as np
+from networkx.exception import NetworkXError
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
 
 class MultiplyCombiner(ProbabilityCombiner):
@@ -8,7 +9,11 @@ def combine(self, proba):
             
             level_probs = np.zeros_like(proba[level])
             for node in self.classifier.classes_[level]:
-                predecessor = list(self.classifier.hierarchy_.predecessors(node))[0]
+                try:
+                    predecessor = list(self.classifier.hierarchy_.predecessors(node))[0]
+                except NetworkXError:
+                    # skip empty levels
+                    continue
                 predecessor_index = self.classifier.class_to_index_mapping_[level-1][predecessor]
                 index = self.classifier.class_to_index_mapping_[level][node]
 

From 420c10ddfce273687633f4af7c67642ed83fca38 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sun, 17 Mar 2024 18:05:45 +0100
Subject: [PATCH 30/65] integrate GeometricMeanCombiner into predict_proba, fix
 bugs

---
 hiclass/HierarchicalClassifier.py             | 27 ++++++
 hiclass/LocalClassifierPerLevel.py            | 25 +++++-
 hiclass/LocalClassifierPerNode.py             | 27 +++++-
 hiclass/LocalClassifierPerParentNode.py       | 82 +++++++++++--------
 .../GeometricMeanCombiner.py                  |  3 +-
 hiclass/probability_combiner/__init__.py      |  6 ++
 6 files changed, 131 insertions(+), 39 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index 2e6ee9ec..f251f10d 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -12,6 +12,12 @@
 from sklearn.utils.validation import _check_sample_weight
 from sklearn.utils.validation import check_array, check_is_fitted
 
+from hiclass.probability_combiner import (
+    GeometricMeanCombiner,
+    ArithmeticMeanCombiner,
+    MultiplyCombiner
+)
+
 try:
     import ray
 except ImportError:
@@ -460,6 +466,14 @@ def _fit_classifier(self, node):
     @staticmethod
     def _fit_calibrator(self, node):
         raise NotImplementedError("Method should be implemented in the LCPN and LCPPN")
+    
+    def _create_probability_combiner(self, name):
+        if name == 'geometric':
+            return GeometricMeanCombiner(self)
+        elif name == 'arithmetic':
+            return ArithmeticMeanCombiner(self)
+        elif name == 'multiply':
+            return MultiplyCombiner(self)
 
     def _clean_up(self):
         self.logger_.info("Cleaning up variables that can take a lot of disk space")
@@ -477,3 +491,16 @@ def _clean_up(self):
             del self.y_cross_val
         if hasattr(self, 'X_cross_val'):
             del self.X_cross_val
+    
+    def _reorder_local_probabilities(self, probabilities, local_labels, level):
+        n_samples, n_labels = probabilities.shape[0], self.max_level_dimensions_[level]
+        sorted_probabilities = np.zeros(shape=(n_samples, n_labels))
+
+        for idx, label in enumerate(local_labels):
+            new_idx = self.class_to_index_mapping_[level][label]
+            sorted_probabilities[:, new_idx] = probabilities[:, idx]
+        
+        return sorted_probabilities
+
+
+
diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index 3b6003eb..2738a920 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -15,6 +15,8 @@
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
 from hiclass._calibration.Calibrator import _Calibrator
 
+from hiclass.probability_combiner import init_strings as probability_combiner_init_strings
+
 try:
     import ray
 except ImportError:
@@ -51,7 +53,8 @@ def __init__(
         n_jobs: int = 1,
         bert: bool = False,
         calibration_method: str = None,
-        return_all_probabilities: bool = False
+        return_all_probabilities: bool = False,
+        probability_combiner: str = "geometric"
     ):
         """
         Initialize a local classifier per level.
@@ -79,6 +82,12 @@ def __init__(
             If set, use the desired method to calibrate probabilities returned by predict_proba().
         return_all_probabilities : bool, default=False
             If True, return probabilities for all levels. Otherwise, return only probabilities for the last level.
+        probability_combiner: {"geometric", "arithmetic", "multiply"}, str, default="geometric"
+            Specify the rule for combining probabilities over multiple levels:
+
+            - `geometric`: Each levels probabilities are calculated by taking the geometric mean of itself and its predecessors;
+            - `arithmetic`: Each levels probabilities are calculated by taking the arithmetic mean of itself and its predecessors;
+            - `multiply`: Each levels probabilities are calculated by multiplying itself with its predecessors.
         """
         super().__init__(
             local_classifier=local_classifier,
@@ -91,6 +100,10 @@ def __init__(
             calibration_method=calibration_method,
         )
         self.return_all_probabilities = return_all_probabilities
+        self.probability_combiner = probability_combiner
+
+        if self.probability_combiner and self.probability_combiner not in probability_combiner_init_strings:
+            raise ValueError(f"probability_combiner must be one of {', '.join(probability_combiner_init_strings)} or None.")
 
     def fit(self, X, y, sample_weight=None):
         """
@@ -201,6 +214,11 @@ def predict_proba(self, X):
         y[:, 0] = calibrator.classes_[np.argmax(proba, axis=1)]
 
         level_probability_list = [proba] + self._predict_proba_remaining_levels(X, y)
+        # combine probabilities
+        if self.probability_combiner:
+            probability_combiner_ = self._create_probability_combiner(self.probability_combiner)
+            self.logger_.info(f"Combining probabilities using {type(probability_combiner_).__name__}")
+            level_probability_list = probability_combiner_.combine(level_probability_list)
 
         return level_probability_list if self.return_all_probabilities else level_probability_list[-1]
 
@@ -212,8 +230,9 @@ def _predict_proba_remaining_levels(self, X, y):
             # use classifier as a fallback if no calibrator is available
             calibrator = calibrator or classifier
             probabilities = calibrator.predict_proba(X)
+            # sort probabilities
+            probabilities = self._reorder_local_probabilities(probabilities, calibrator.classes_, level)
             level_probability_list.append(probabilities)
-            # TODO: test with empty nodes, etc.
         return level_probability_list
 
 
@@ -381,6 +400,8 @@ def _fit_calibrator(self, level, separator):
             return None
 
         calibrator.fit(X, y)
+        print("calibrator classes:")
+        print(calibrator.classes_)
         return calibrator
 
     @staticmethod
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index 20c8285c..b330c691 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -16,6 +16,8 @@
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
 from hiclass._calibration.Calibrator import _Calibrator
 
+from hiclass.probability_combiner import init_strings as probability_combiner_init_strings
+
 
 class LocalClassifierPerNode(BaseEstimator, HierarchicalClassifier):
     """
@@ -46,7 +48,8 @@ def __init__(
         n_jobs: int = 1,
         bert: bool = False,
         calibration_method: str = None,
-        return_all_probabilities: bool = False
+        return_all_probabilities: bool = False,
+        probability_combiner: str = "geometric"
     ):
         """
         Initialize a local classifier per node.
@@ -85,6 +88,12 @@ def __init__(
             If set, use the desired method to calibrate probabilities returned by predict_proba().
         return_all_probabilities : bool, default=False
             If True, return probabilities for all levels. Otherwise, return only probabilities for the last level.
+        probability_combiner: {"geometric", "arithmetic", "multiply"}, str, default="geometric"
+            Specify the rule for combining probabilities over multiple levels:
+
+            - `geometric`: Each levels probabilities are calculated by taking the geometric mean of itself and its predecessors;
+            - `arithmetic`: Each levels probabilities are calculated by taking the arithmetic mean of itself and its predecessors;
+            - `multiply`: Each levels probabilities are calculated by multiplying itself with its predecessors.
         """
         super().__init__(
             local_classifier=local_classifier,
@@ -98,6 +107,10 @@ def __init__(
         )
         self.binary_policy = binary_policy
         self.return_all_probabilities = return_all_probabilities
+        self.probability_combiner = probability_combiner
+
+        if self.probability_combiner and self.probability_combiner not in probability_combiner_init_strings:
+                raise ValueError(f"probability_combiner must be one of {', '.join(probability_combiner_init_strings)} or None.")
 
     def fit(self, X, y, sample_weight=None):
         """
@@ -288,8 +301,16 @@ def predict_proba(self, X):
         self._remove_separator(y)
 
         # normalize probabilities
-        for level_probabilities in level_probability_list:
-            level_probabilities /= level_probabilities.sum(axis=1, keepdims=True)
+        level_probability_list = [
+            np.nan_to_num(level_probabilities / level_probabilities.sum(axis=1, keepdims=True)) 
+                for level_probabilities in level_probability_list
+        ]
+        
+        # combine probabilities
+        if self.probability_combiner:
+            probability_combiner_ = self._create_probability_combiner(self.probability_combiner)
+            self.logger_.info(f"Combining probabilities using {type(probability_combiner_).__name__}")
+            level_probability_list = probability_combiner_.combine(level_probability_list)
 
         return level_probability_list if self.return_all_probabilities else level_probability_list[-1]
 
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index 4c4d684c..56469eb5 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -15,6 +15,7 @@
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
 from hiclass._calibration.Calibrator import _Calibrator
 
+from hiclass.probability_combiner import init_strings as probability_combiner_init_strings
 
 class LocalClassifierPerParentNode(BaseEstimator, HierarchicalClassifier):
     """
@@ -44,7 +45,8 @@ def __init__(
         n_jobs: int = 1,
         bert: bool = False,
         calibration_method: str = None,
-        return_all_probabilities: bool = False
+        return_all_probabilities: bool = False,
+        probability_combiner: str = "geometric"
     ):
         """
         Initialize a local classifier per parent node.
@@ -72,6 +74,12 @@ def __init__(
             If set, use the desired method to calibrate probabilities returned by predict_proba().
         return_all_probabilities : bool, default=False
             If True, return probabilities for all levels. Otherwise, return only probabilities for the last level.
+        probability_combiner: {"geometric", "arithmetic", "multiply"}, str, default="geometric"
+            Specify the rule for combining probabilities over multiple levels:
+
+            - `geometric`: Each levels probabilities are calculated by taking the geometric mean of itself and its predecessors;
+            - `arithmetic`: Each levels probabilities are calculated by taking the arithmetic mean of itself and its predecessors;
+            - `multiply`: Each levels probabilities are calculated by multiplying itself with its predecessors.
         """
         super().__init__(
             local_classifier=local_classifier,
@@ -84,6 +92,10 @@ def __init__(
             calibration_method=calibration_method,
         )
         self.return_all_probabilities = return_all_probabilities
+        self.probability_combiner = probability_combiner
+
+        if self.probability_combiner and self.probability_combiner not in probability_combiner_init_strings:
+            raise ValueError(f"probability_combiner must be one of {', '.join(probability_combiner_init_strings)} or None.")
 
     def fit(self, X, y, sample_weight=None):
         """
@@ -192,44 +204,50 @@ def predict_proba(self, X):
 
         y[:, 0] = calibrator.classes_[np.argmax(proba, axis=1)]
         level_probability_list = [proba] + self._predict_proba_remaining_levels(X, y)
-
+    
+        # combine probabilities
+        if self.probability_combiner:
+            probability_combiner_ = self._create_probability_combiner(self.probability_combiner)
+            self.logger_.info(f"Combining probabilities using {type(probability_combiner_).__name__}")
+            level_probability_list = probability_combiner_.combine(level_probability_list)
+        
         return level_probability_list if self.return_all_probabilities else level_probability_list[-1]
 
     def _predict_proba_remaining_levels(self, X, y):
         level_probability_list = []
         for level in range(1, y.shape[1]):
-                predecessors = set(y[:, level - 1])
-                predecessors.discard("")
-                level_dimension = self.max_level_dimensions_[level]
-                cur_level_probabilities = np.zeros((X.shape[0], level_dimension))
-
-                for predecessor in predecessors:
-                    #mask = np.isin(y[:, level - 1], predecessor)
-                    mask = np.isin(y[:, level - 1], self.classes_[level-1])
-                    predecessor_x = X[mask]
-                    if predecessor_x.shape[0] > 0:
-                        successors = list(self.hierarchy_.successors(predecessor))
-                        if len(successors) > 0:
-                            classifier = self.hierarchy_.nodes[predecessor]["classifier"]
-                            # use classifier as a fallback if no calibrator is available
-                            calibrator = self.hierarchy_.nodes[predecessor].get("calibrator", classifier) or classifier
-
-                            proba = calibrator.predict_proba(predecessor_x)
-                            y[mask, level] = calibrator.classes_[np.argmax(proba, axis=1)]
-
-                            for successor in successors:
-                                class_index = self.class_to_index_mapping_[level][str(successor)]
-
-                                proba_index = np.where(calibrator.classes_ == successor)[0][0]
-                                cur_level_probabilities[mask, class_index] = proba[:, proba_index]
-
-                level_probability_list.append(cur_level_probabilities)
+            predecessors = set(y[:, level - 1])
+            predecessors.discard("")
+            level_dimension = self.max_level_dimensions_[level]
+            cur_level_probabilities = np.zeros((X.shape[0], level_dimension))
+
+            for predecessor in predecessors:
+                #mask = np.isin(y[:, level - 1], predecessor)
+                mask = np.isin(y[:, level - 1], self.classes_[level-1])
+                predecessor_x = X[mask]
+                if predecessor_x.shape[0] > 0:
+                    successors = list(self.hierarchy_.successors(predecessor))
+                    if len(successors) > 0:
+                        classifier = self.hierarchy_.nodes[predecessor]["classifier"]
+                        # use classifier as a fallback if no calibrator is available
+                        calibrator = self.hierarchy_.nodes[predecessor].get("calibrator", classifier) or classifier
+
+                        proba = calibrator.predict_proba(predecessor_x)
+                        y[mask, level] = calibrator.classes_[np.argmax(proba, axis=1)]
+
+                        for successor in successors:
+                            class_index = self.class_to_index_mapping_[level][str(successor)]
+
+                            proba_index = np.where(calibrator.classes_ == successor)[0][0]
+                            cur_level_probabilities[mask, class_index] = proba[:, proba_index]
+
+            level_probability_list.append(cur_level_probabilities)
         
         # normalize probabilities
-        
-        for level_probabilities in level_probability_list:
-            level_probabilities /= level_probabilities.sum(axis=1, keepdims=True)
-        
+        level_probability_list = [
+            np.nan_to_num(level_probabilities / level_probabilities.sum(axis=1, keepdims=True)) 
+                for level_probabilities in level_probability_list
+        ]
 
         return level_probability_list
         
diff --git a/hiclass/probability_combiner/GeometricMeanCombiner.py b/hiclass/probability_combiner/GeometricMeanCombiner.py
index d26b3ae3..2ee29759 100644
--- a/hiclass/probability_combiner/GeometricMeanCombiner.py
+++ b/hiclass/probability_combiner/GeometricMeanCombiner.py
@@ -7,7 +7,6 @@ def combine(self, proba):
         res = [proba[0]]
         log_sum = [np.log(proba[0])]
         for level in range(1, self.classifier.max_levels_):
-            
             level_probs = np.zeros_like(proba[level])
             level_log_sum = np.zeros_like(proba[level])
             for node in self.classifier.classes_[level]:
@@ -16,9 +15,9 @@ def combine(self, proba):
                 except NetworkXError:
                     # skip empty levels
                     continue
+
                 predecessor_index = self.classifier.class_to_index_mapping_[level-1][predecessor]
                 index = self.classifier.class_to_index_mapping_[level][node]
-
                 level_log_sum[:, index] += (np.log(proba[level][:, index]) + log_sum[level-1][:, predecessor_index])
                 level_probs[:, index] = np.exp(level_log_sum[:, index] / (level+1))
 
diff --git a/hiclass/probability_combiner/__init__.py b/hiclass/probability_combiner/__init__.py
index c10dfa32..540d35ba 100644
--- a/hiclass/probability_combiner/__init__.py
+++ b/hiclass/probability_combiner/__init__.py
@@ -7,3 +7,9 @@
     "ArithmeticMeanCombiner",
     "GeometricMeanCombiner",
 ]
+
+init_strings = [
+    "multiply",
+    "geometric",
+    "arithmetic",
+]
\ No newline at end of file

From 6dae86d4abd54bcfb2b7c369dfbd4342cacadcfd Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sun, 17 Mar 2024 21:37:16 +0100
Subject: [PATCH 31/65] make label_binarizer in calibrator class output sparse
 labels

---
 hiclass/LocalClassifierPerLevel.py | 2 --
 hiclass/_calibration/Calibrator.py | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index 2738a920..b672638f 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -400,8 +400,6 @@ def _fit_calibrator(self, level, separator):
             return None
 
         calibrator.fit(X, y)
-        print("calibrator classes:")
-        print(calibrator.classes_)
         return calibrator
 
     @staticmethod
diff --git a/hiclass/_calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
index de9c010d..43a2ac3a 100644
--- a/hiclass/_calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -47,7 +47,7 @@ def fit(self, X, y):
 
         if self.multiclass:
             # binarize multiclass labels
-            label_binarizer = LabelBinarizer(sparse_output=False)
+            label_binarizer = LabelBinarizer()
             label_binarizer.fit(self.estimator.classes_)
             binary_labels = label_binarizer.transform(y).T
 

From 3ce635355e1b15a1123103e433be243facd63ff3 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sun, 17 Mar 2024 22:15:03 +0100
Subject: [PATCH 32/65] make calibration metrics public

---
 hiclass/metrics.py | 42 ++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/hiclass/metrics.py b/hiclass/metrics.py
index aa39de27..51ba4785 100644
--- a/hiclass/metrics.py
+++ b/hiclass/metrics.py
@@ -2,7 +2,7 @@
 
 import numpy as np
 from sklearn.utils import check_array
-from sklearn.metrics import log_loss
+from sklearn.metrics import log_loss as sk_log_loss
 from sklearn.preprocessing import LabelEncoder
 
 from hiclass.HierarchicalClassifier import make_leveled
@@ -277,6 +277,44 @@ def _prepare_data(classifier, y_true, y_prob, level, y_pred=None):
     
     return y_true, y_pred, new_labels, new_y_prob
 
+def _aggregate_scores(scores, agg):
+        if agg == 'average':
+            return np.mean(scores)
+        if agg == 'sum':
+            return np.sum(scores)
+        if agg == None or agg == 'None':
+            return scores
+
+def multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, agg='average'):
+    scores = []
+    for level in range(make_leveled(y_true).shape[1]):
+        scores.append(_multiclass_brier_score(classifier, y_true, y_prob, level))
+    return _aggregate_scores(scores, agg)
+
+def log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, agg='average'):
+    scores = []
+    for level in range(make_leveled(y_true).shape[1]):
+        scores.append(_log_loss(classifier, y_true, y_prob, level))
+    return _aggregate_scores(scores, agg)
+
+def expected_calibration_error(classifier: HierarchicalClassifier, y_true, y_prob, y_pred, n_bins=10, agg='average'):
+    scores = []
+    for level in range(make_leveled(y_true).shape[1]):
+        scores.append(_expected_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins))
+    return _aggregate_scores(scores, agg)
+
+def statistical_calibration_error(classifier, y_true, y_prob, y_pred, n_bins=10, agg='average'):
+    scores = []
+    for level in range(make_leveled(y_true).shape[1]):
+        scores.append(_statistical_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins=n_bins))
+    return _aggregate_scores(scores, agg)
+
+def adaptive_calibration_error(classifier, y_true, y_prob, y_pred, n_ranges=3, agg='average'):
+    scores = []
+    for level in range(make_leveled(y_true).shape[1]):
+        scores.append(_adaptive_calibration_error(classifier, y_true, y_prob, y_pred, level, n_ranges=n_ranges))
+    return _aggregate_scores(scores, agg)
+
 def _multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, level: int):
     y_true, _, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level)
     label_encoder = LabelEncoder()
@@ -286,7 +324,7 @@ def _multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarr
 
 def _log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, level: int):
     y_true, _, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level)
-    return log_loss(y_true, y_prob, labels=labels)
+    return sk_log_loss(y_true, y_prob, labels=labels)
 
 def _expected_calibration_error(classifier: HierarchicalClassifier, y_true, y_prob, y_pred, level, n_bins=10):
     y_true, y_pred, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level, y_pred)

From ef6b7d2ff613faab0bb67c43b43bfb8e6b4ab213 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sat, 30 Mar 2024 14:42:00 +0100
Subject: [PATCH 33/65] refactoring

---
 hiclass/HierarchicalClassifier.py  | 27 ++++++++++++++++++++++-----
 hiclass/LocalClassifierPerLevel.py |  3 +--
 hiclass/LocalClassifierPerNode.py  | 10 +++++-----
 hiclass/metrics.py                 |  2 +-
 4 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index f251f10d..3a0fa7b0 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -228,17 +228,34 @@ def calibrate(self, X, y):
             X = np.array(X)
         
         if self.calibration_method == "cvap":
+            '''
             # combine train and calibration dataset for cross validation
             if isinstance(self.X_, scipy.sparse._csr.csr_matrix):
+                self.logger_.info(f"Sparse Calibration size: {X.shape} train size: {self.X_.shape}")
                 self.X_cross_val = scipy.sparse.vstack([self.X_, X])
                 self.logger_.info(f"CV Dataset X: {str(type(self.X_cross_val))} {str(self.X_cross_val.shape)}")
             else:
-                self.X_cross_val = np.hstack([self.X_, X])
+                self.logger_.info(f"Not sparse Calibration size: {X.shape} train size: {self.X_.shape}")
+                self.X_cross_val = np.vstack([self.X_, X])
                 self.logger_.info(f"CV Dataset X: {str(type(self.X_cross_val))} {str(self.X_cross_val.shape)}")
             self.y_cross_val = np.vstack([self.y_, y])
             self.y_cross_val = make_leveled(self.y_cross_val)
             self.y_cross_val = self._disambiguate(self.y_cross_val)
             self.y_cross_val = self._convert_1d_y_to_2d(self.y_cross_val) # TODO: rename to y_cal?
+            '''
+            # combine train and calibration dataset for cross validation
+            if isinstance(self.X_, scipy.sparse._csr.csr_matrix):
+                self.logger_.info(f"Sparse Calibration size: {X.shape} train size: {self.X_.shape}")
+                self.X_cal = scipy.sparse.vstack([self.X_, X])
+                self.logger_.info(f"CV Dataset X: {str(type(self.X_cal))} {str(self.X_cal.shape)}")
+            else:
+                self.logger_.info(f"Not sparse Calibration size: {X.shape} train size: {self.X_.shape}")
+                self.X_cal = np.vstack([self.X_, X])
+                self.logger_.info(f"CV Dataset X: {str(type(self.X_cal))} {str(self.X_cal.shape)}")
+            self.y_cal = np.vstack([self.y_, y])
+            self.y_cal = make_leveled(self.y_cal)
+            self.y_cal = self._disambiguate(self.y_cal)
+            self.y_cal = self._convert_1d_y_to_2d(self.y_cal) # TODO: rename to y_cal?
         else:
             self.X_cal = X
             self.y_cal = y
@@ -487,10 +504,10 @@ def _clean_up(self):
             del self.X_cal
         if hasattr(self, 'y_cal'):
             del self.y_cal
-        if hasattr(self, 'y_cross_val'):
-            del self.y_cross_val
-        if hasattr(self, 'X_cross_val'):
-            del self.X_cross_val
+        #if hasattr(self, 'y_cross_val'):
+        #    del self.y_cross_val
+        #if hasattr(self, 'X_cross_val'):
+        #    del self.X_cross_val
     
     def _reorder_local_probabilities(self, probabilities, local_labels, level):
         n_samples, n_labels = probabilities.shape[0], self.max_level_dimensions_[level]
diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index b672638f..3961a22d 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -393,12 +393,11 @@ def _fit_calibrator(self, level, separator):
             return None
         
         X, y, _ = self._remove_empty_leaves(
-            separator, self.X_cal, self.y_cal[:, level], None
+            separator, self.X_cal, self.y_cal[:, level], None #X_cross_val, y_cross_val
         )
         if len(y) == 0 or len(np.unique(y)) < 2:
             self.logger_.info(f"No calibration samples to fit calibrator for level: {str(level)}")
             return None
-
         calibrator.fit(X, y)
         return calibrator
 
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index b330c691..5c4b6d20 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -318,14 +318,14 @@ def _initialize_binary_policy(self, calibration=False):
         if isinstance(self.binary_policy, str):
             self.logger_.info(f"Initializing {self.binary_policy} binary policy")
             try:
-                if calibration and self.calibration_method != "cvap":
+                if calibration: # and self.calibration_method != "cvap":
                     binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
                         self.binary_policy.lower()
                     ](self.hierarchy_, self.X_cal, self.y_cal, None)
-                elif calibration and self.calibration_method == "cvap":
-                    binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
-                        self.binary_policy.lower()
-                    ](self.hierarchy_, self.X_cross_val, self.y_cross_val, None)
+                #elif calibration and self.calibration_method == "cvap":
+                #    binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
+                #        self.binary_policy.lower()
+                #    ](self.hierarchy_, self.X_cross_val, self.y_cross_val, None)
                 else:
                     binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
                         self.binary_policy.lower()
diff --git a/hiclass/metrics.py b/hiclass/metrics.py
index 51ba4785..65300880 100644
--- a/hiclass/metrics.py
+++ b/hiclass/metrics.py
@@ -309,7 +309,7 @@ def statistical_calibration_error(classifier, y_true, y_prob, y_pred, n_bins=10,
         scores.append(_statistical_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins=n_bins))
     return _aggregate_scores(scores, agg)
 
-def adaptive_calibration_error(classifier, y_true, y_prob, y_pred, n_ranges=3, agg='average'):
+def adaptive_calibration_error(classifier, y_true, y_prob, y_pred, n_ranges=10, agg='average'):
     scores = []
     for level in range(make_leveled(y_true).shape[1]):
         scores.append(_adaptive_calibration_error(classifier, y_true, y_prob, y_pred, level, n_ranges=n_ranges))

From 4b2bf99831c500b3ea1e69a8389dfc3b9f76633d Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Wed, 3 Apr 2024 13:58:01 +0200
Subject: [PATCH 34/65] refactor probability_combiners

---
 .../ArithmeticMeanCombiner.py                 | 22 ++++++++++--------
 .../GeometricMeanCombiner.py                  | 22 ++++++++++--------
 .../probability_combiner/MultiplyCombiner.py  | 22 +++++++++---------
 .../ProbabilityCombiner.py                    | 23 +++++++++++++++++++
 4 files changed, 58 insertions(+), 31 deletions(-)

diff --git a/hiclass/probability_combiner/ArithmeticMeanCombiner.py b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
index 7c51b0a7..90d1570d 100644
--- a/hiclass/probability_combiner/ArithmeticMeanCombiner.py
+++ b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
@@ -1,6 +1,7 @@
 import numpy as np
 from networkx.exception import NetworkXError
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
+from collections import defaultdict
 
 class ArithmeticMeanCombiner(ProbabilityCombiner):
     def combine(self, proba):
@@ -9,18 +10,19 @@ def combine(self, proba):
         for level in range(1, self.classifier.max_levels_):
             level_probs = np.zeros_like(proba[level])
             level_sum = np.zeros_like(proba[level])
-            for node in self.classifier.classes_[level]:
-                try:
-                    predecessor = list(self.classifier.hierarchy_.predecessors(node))[0]
-                except NetworkXError:
-                    # skip empty levels
-                    continue
-                predecessor_index = self.classifier.class_to_index_mapping_[level-1][predecessor]
-                index = self.classifier.class_to_index_mapping_[level][node]
+            # find all predecessors of a node
+            predecessors = self._find_predecessors(level)
 
-                level_sum[:, index] += proba[level][:, index] + sums[level-1][:, predecessor_index]
+            for node in predecessors.keys():
+                index = self.classifier.class_to_index_mapping_[level][node]
+                # find indices of all predecessors
+                predecessor_indices = [self.classifier.class_to_index_mapping_[level-1][predecessor] for predecessor in predecessors[node]]
+                # combine probabilities of all predecessors
+                predecessors_combined_prob = np.sum([sums[level-1][:, pre_index] for pre_index in predecessor_indices], axis=0)
+                level_sum[:, index] += proba[level][:, index] + predecessors_combined_prob
                 level_probs[:, index] = level_sum[:, index] / (level+1)
 
+
             res.append(level_probs)
             sums.append(level_sum)
-        return res
+        return self._normalize(res)
diff --git a/hiclass/probability_combiner/GeometricMeanCombiner.py b/hiclass/probability_combiner/GeometricMeanCombiner.py
index 2ee29759..6326dedd 100644
--- a/hiclass/probability_combiner/GeometricMeanCombiner.py
+++ b/hiclass/probability_combiner/GeometricMeanCombiner.py
@@ -1,6 +1,7 @@
 import numpy as np
 from networkx.exception import NetworkXError
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
+from collections import defaultdict
 
 class GeometricMeanCombiner(ProbabilityCombiner):
     def combine(self, proba):
@@ -9,18 +10,19 @@ def combine(self, proba):
         for level in range(1, self.classifier.max_levels_):
             level_probs = np.zeros_like(proba[level])
             level_log_sum = np.zeros_like(proba[level])
-            for node in self.classifier.classes_[level]:
-                try:
-                    predecessor = list(self.classifier.hierarchy_.predecessors(node))[0]
-                except NetworkXError:
-                    # skip empty levels
-                    continue
+            # find all predecessors of a node
+            predecessors = self._find_predecessors(level)
 
-                predecessor_index = self.classifier.class_to_index_mapping_[level-1][predecessor]
+            for node in predecessors.keys():
                 index = self.classifier.class_to_index_mapping_[level][node]
-                level_log_sum[:, index] += (np.log(proba[level][:, index]) + log_sum[level-1][:, predecessor_index])
-                level_probs[:, index] = np.exp(level_log_sum[:, index] / (level+1))
+                # find indices of all predecessors
+                predecessor_indices = [self.classifier.class_to_index_mapping_[level-1][predecessor] for predecessor in predecessors[node]]
+                # combine probabilities of all predecessors
+                predecessors_combined_log_prob = np.log(np.sum([np.exp(log_sum[level-1][:, pre_index]) for pre_index in predecessor_indices], axis=0))
 
+                level_log_sum[:, index] += (np.log(proba[level][:, index]) + predecessors_combined_log_prob)
+                level_probs[:, index] = np.exp(level_log_sum[:, index] / (level+1))
+            
             log_sum.append(level_log_sum)
             res.append(level_probs)
-        return res
+        return self._normalize(res)
diff --git a/hiclass/probability_combiner/MultiplyCombiner.py b/hiclass/probability_combiner/MultiplyCombiner.py
index 6204eca3..c790d8b5 100644
--- a/hiclass/probability_combiner/MultiplyCombiner.py
+++ b/hiclass/probability_combiner/MultiplyCombiner.py
@@ -1,23 +1,23 @@
 import numpy as np
 from networkx.exception import NetworkXError
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
+from collections import defaultdict
 
 class MultiplyCombiner(ProbabilityCombiner):
     def combine(self, proba):
         res = [proba[0]]
         for level in range(1, self.classifier.max_levels_):
-            
             level_probs = np.zeros_like(proba[level])
-            for node in self.classifier.classes_[level]:
-                try:
-                    predecessor = list(self.classifier.hierarchy_.predecessors(node))[0]
-                except NetworkXError:
-                    # skip empty levels
-                    continue
-                predecessor_index = self.classifier.class_to_index_mapping_[level-1][predecessor]
-                index = self.classifier.class_to_index_mapping_[level][node]
+            # find all predecessors of a node
+            predecessors = self._find_predecessors(level)
 
-                level_probs[:, index] = res[level-1][:, predecessor_index] * proba[level][:, index]
+            for node in predecessors.keys():
+                index = self.classifier.class_to_index_mapping_[level][node]
+                # find indices of all predecessors
+                predecessor_indices = [self.classifier.class_to_index_mapping_[level-1][predecessor] for predecessor in predecessors[node]]
+                # combine probabilities of all predecessors
+                predecessors_combined_prob = np.sum([res[level-1][:, pre_index] for pre_index in predecessor_indices], axis=0)
+                level_probs[:, index] = predecessors_combined_prob * proba[level][:, index]
 
             res.append(level_probs)
-        return res
+        return self._normalize(res)
diff --git a/hiclass/probability_combiner/ProbabilityCombiner.py b/hiclass/probability_combiner/ProbabilityCombiner.py
index 5e3f3772..861a79c8 100644
--- a/hiclass/probability_combiner/ProbabilityCombiner.py
+++ b/hiclass/probability_combiner/ProbabilityCombiner.py
@@ -1,6 +1,8 @@
 import abc
 import numpy as np
 from typing import List
+from collections import defaultdict
+from networkx.exception import NetworkXError
 
 class ProbabilityCombiner(abc.ABC):
 
@@ -10,3 +12,24 @@ def __init__(self, classifier) -> None:
     @abc.abstractmethod
     def combine(self, proba: List[np.ndarray]) -> List[np.ndarray]:
         ...
+    
+    def _normalize(self, proba):
+        return [
+            np.nan_to_num(level_probabilities / level_probabilities.sum(axis=1, keepdims=True)) 
+                for level_probabilities in proba
+        ]
+    
+    def _find_predecessors(self, level):
+        predecessors = defaultdict(list)
+        for node in self.classifier.global_classes_[level]:
+            try:
+                predecessor = list(self.classifier.hierarchy_.predecessors(node))[0]
+            except NetworkXError:
+                # skip empty levels
+                continue
+
+            predecessor_name = str(predecessor).split(self.classifier.separator_)[-1]
+            node_name = str(node).split(self.classifier.separator_)[-1]
+
+            predecessors[node_name].append(predecessor_name)
+        return predecessors

From 29f6a1bb4ae1355ee3030adcd704e515c220859e Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Wed, 3 Apr 2024 13:58:59 +0200
Subject: [PATCH 35/65] refactor predict_proba

---
 hiclass/HierarchicalClassifier.py           |  63 ++++++----
 hiclass/LocalClassifierPerLevel.py          |   4 +
 hiclass/LocalClassifierPerNode.py           |  17 ++-
 hiclass/LocalClassifierPerParentNode.py     |   7 +-
 hiclass/_calibration/Calibrator.py          |  46 +++++---
 hiclass/_calibration/VennAbersCalibrator.py | 123 +++++++++++++++++---
 hiclass/_calibration/calibration_utils.py   |  13 +++
 hiclass/metrics.py                          |   2 +
 8 files changed, 204 insertions(+), 71 deletions(-)
 create mode 100644 hiclass/_calibration/calibration_utils.py

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index 3a0fa7b0..4499f61a 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -168,12 +168,12 @@ def _pre_fit(self, X, y, sample_weight):
 
         if self.y_.ndim > 1:
             self.max_level_dimensions_ = np.array([len(np.unique(self.y_[:, level])) for level in range(self.y_.shape[1])])
-            self.classes_ = [np.unique(self.y_[:, level]).astype("str") for level in range(self.y_.shape[1])]
-            self.class_to_index_mapping_ = [{self.classes_[level][index]: index for index in range(len(self.classes_[level]))} for level in range(self.y_.shape[1])]
+            self.global_classes_ = [np.unique(self.y_[:, level]).astype("str") for level in range(self.y_.shape[1])]
+            self.global_class_to_index_mapping_ = [{self.global_classes_[level][index]: index for index in range(len(self.global_classes_[level]))} for level in range(self.y_.shape[1])]
         else:
             self.max_level_dimensions_ = np.array([len(np.unique(self.y_))])
-            self.classes_ = [np.unique(self.y_).astype("str")]
-            self.class_to_index_mapping_ = [{self.classes_[0][index] : index for index in range(len(self.classes_[0]))}]
+            self.global_classes_ = [np.unique(self.y_).astype("str")]
+            self.global_class_to_index_mapping_ = [{self.global_classes_[0][index] : index for index in range(len(self.global_classes_[0]))}]
 
         # Create and configure logger
         self._create_logger()
@@ -228,21 +228,6 @@ def calibrate(self, X, y):
             X = np.array(X)
         
         if self.calibration_method == "cvap":
-            '''
-            # combine train and calibration dataset for cross validation
-            if isinstance(self.X_, scipy.sparse._csr.csr_matrix):
-                self.logger_.info(f"Sparse Calibration size: {X.shape} train size: {self.X_.shape}")
-                self.X_cross_val = scipy.sparse.vstack([self.X_, X])
-                self.logger_.info(f"CV Dataset X: {str(type(self.X_cross_val))} {str(self.X_cross_val.shape)}")
-            else:
-                self.logger_.info(f"Not sparse Calibration size: {X.shape} train size: {self.X_.shape}")
-                self.X_cross_val = np.vstack([self.X_, X])
-                self.logger_.info(f"CV Dataset X: {str(type(self.X_cross_val))} {str(self.X_cross_val.shape)}")
-            self.y_cross_val = np.vstack([self.y_, y])
-            self.y_cross_val = make_leveled(self.y_cross_val)
-            self.y_cross_val = self._disambiguate(self.y_cross_val)
-            self.y_cross_val = self._convert_1d_y_to_2d(self.y_cross_val) # TODO: rename to y_cal?
-            '''
             # combine train and calibration dataset for cross validation
             if isinstance(self.X_, scipy.sparse._csr.csr_matrix):
                 self.logger_.info(f"Sparse Calibration size: {X.shape} train size: {self.X_.shape}")
@@ -255,7 +240,7 @@ def calibrate(self, X, y):
             self.y_cal = np.vstack([self.y_, y])
             self.y_cal = make_leveled(self.y_cal)
             self.y_cal = self._disambiguate(self.y_cal)
-            self.y_cal = self._convert_1d_y_to_2d(self.y_cal) # TODO: rename to y_cal?
+            self.y_cal = self._convert_1d_y_to_2d(self.y_cal)
         else:
             self.X_cal = X
             self.y_cal = y
@@ -504,20 +489,48 @@ def _clean_up(self):
             del self.X_cal
         if hasattr(self, 'y_cal'):
             del self.y_cal
-        #if hasattr(self, 'y_cross_val'):
-        #    del self.y_cross_val
-        #if hasattr(self, 'X_cross_val'):
-        #    del self.X_cross_val
     
     def _reorder_local_probabilities(self, probabilities, local_labels, level):
         n_samples, n_labels = probabilities.shape[0], self.max_level_dimensions_[level]
         sorted_probabilities = np.zeros(shape=(n_samples, n_labels))
 
         for idx, label in enumerate(local_labels):
-            new_idx = self.class_to_index_mapping_[level][label]
+            #local_label = label.split(self.separator_)[level]
+            new_idx = self.global_class_to_index_mapping_[level][label]
             sorted_probabilities[:, new_idx] = probabilities[:, idx]
         
         return sorted_probabilities
 
+    def _combine_and_reorder(self, proba):
+        res = [proba[0]]
+        classes_ = [self.global_classes_[0]]
+        for level in range(1, len(proba)):
+            # get local labels
+            local_labels = np.sort(np.unique([label.split(self.separator_)[level] for label in self.global_classes_[level]]))
+
+            oldToNew = {}
+            for label in self.global_classes_[level]:
+                # local label
+                local_label = label.split(self.separator_)[level]
+
+                # old index
+                # old_index = self.global_class_to_index_mapping_[level][label]
+                new_index = np.where(local_labels == local_label)[0][0]
+
+                oldToNew[label] = local_label, new_index
+
+            res_proba = np.zeros(shape=(proba[level].shape[0], len(local_labels)))
+
+            for old_label in self.global_classes_[level]:
+                old_idx = self.global_class_to_index_mapping_[level][old_label]
+                _, new_idx = oldToNew[old_label]
+                res_proba[:, new_idx] += proba[level][:, old_idx]
+
+            res.append(res_proba)
+            classes_.append(local_labels)
+            class_to_index_mapping_ = [{local_labels[index]: index for index in range(len(local_labels))} for local_labels in classes_]
+            
+        return classes_, class_to_index_mapping_, res
+
 
 
diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index 3961a22d..e41bdabe 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -214,6 +214,8 @@ def predict_proba(self, X):
         y[:, 0] = calibrator.classes_[np.argmax(proba, axis=1)]
 
         level_probability_list = [proba] + self._predict_proba_remaining_levels(X, y)
+        self.classes_, self.class_to_index_mapping_, level_probability_list = self._combine_and_reorder(level_probability_list)
+
         # combine probabilities
         if self.probability_combiner:
             probability_combiner_ = self._create_probability_combiner(self.probability_combiner)
@@ -288,6 +290,8 @@ def _initialize_local_classifiers(self):
     
     def _initialize_local_calibrators(self):
         super()._initialize_local_calibrators()
+        #train_length = self.X_.shape[0]
+        #cal_length = self.X_cal.shape[0]
         self.local_calibrators_ = [
                 _Calibrator(estimator=local_classifier, method=self.calibration_method) for local_classifier in self.local_classifiers_
         ]
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index 5c4b6d20..3a04a4e3 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -272,8 +272,7 @@ def predict_proba(self, X):
                 mask = [True] * X.shape[0]
                 subset_x = X[mask]
             else:
-                #mask = np.isin(y, predecessor).any(axis=1)
-                mask = np.isin(y, self.classes_[level-1]).any(axis=1)
+                mask = np.isin(y, self.global_classes_[level-1]).any(axis=1)
                 subset_x = X[mask]
 
             if subset_x.shape[0] > 0:
@@ -286,7 +285,7 @@ def predict_proba(self, X):
                     positive_index = np.where(calibrator.classes_ == 1)[0]
                     proba = calibrator.predict_proba(subset_x)[:, positive_index][:, 0]
                     local_probabilities[:, i] = proba
-                    class_index = self.class_to_index_mapping_[level][str(successor)]
+                    class_index = self.global_class_to_index_mapping_[level][str(successor)]
                     level_probability_list[-1][mask, class_index] = proba
 
                 highest_local_probability = np.argmax(local_probabilities, axis=1)
@@ -305,8 +304,11 @@ def predict_proba(self, X):
             np.nan_to_num(level_probabilities / level_probabilities.sum(axis=1, keepdims=True)) 
                 for level_probabilities in level_probability_list
         ]
+
+        # combine probabilities horizontally
+        self.classes_, self.class_to_index_mapping_, level_probability_list = self._combine_and_reorder(level_probability_list)
         
-        # combine probabilities
+        # combine probabilities vertically
         if self.probability_combiner:
             probability_combiner_ = self._create_probability_combiner(self.probability_combiner)
             self.logger_.info(f"Combining probabilities using {type(probability_combiner_).__name__}")
@@ -314,18 +316,15 @@ def predict_proba(self, X):
 
         return level_probability_list if self.return_all_probabilities else level_probability_list[-1]
 
+
     def _initialize_binary_policy(self, calibration=False):
         if isinstance(self.binary_policy, str):
             self.logger_.info(f"Initializing {self.binary_policy} binary policy")
             try:
-                if calibration: # and self.calibration_method != "cvap":
+                if calibration:
                     binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
                         self.binary_policy.lower()
                     ](self.hierarchy_, self.X_cal, self.y_cal, None)
-                #elif calibration and self.calibration_method == "cvap":
-                #    binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
-                #        self.binary_policy.lower()
-                #    ](self.hierarchy_, self.X_cross_val, self.y_cross_val, None)
                 else:
                     binary_policy_ = BinaryPolicy.IMPLEMENTED_POLICIES[
                         self.binary_policy.lower()
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index 56469eb5..7a1d5c6a 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -205,6 +205,8 @@ def predict_proba(self, X):
         y[:, 0] = calibrator.classes_[np.argmax(proba, axis=1)]
         level_probability_list = [proba] + self._predict_proba_remaining_levels(X, y)
     
+        self.classes_, self.class_to_index_mapping_, level_probability_list = self._combine_and_reorder(level_probability_list)
+
         # combine probabilities
         if self.probability_combiner:
             probability_combiner_ = self._create_probability_combiner(self.probability_combiner)
@@ -222,8 +224,7 @@ def _predict_proba_remaining_levels(self, X, y):
             cur_level_probabilities = np.zeros((X.shape[0], level_dimension))
 
             for predecessor in predecessors:
-                #mask = np.isin(y[:, level - 1], predecessor)
-                mask = np.isin(y[:, level - 1], self.classes_[level-1])
+                mask = np.isin(y[:, level - 1], self.global_classes_[level-1])
                 predecessor_x = X[mask]
                 if predecessor_x.shape[0] > 0:
                     successors = list(self.hierarchy_.successors(predecessor))
@@ -236,7 +237,7 @@ def _predict_proba_remaining_levels(self, X, y):
                         y[mask, level] = calibrator.classes_[np.argmax(proba, axis=1)]
 
                         for successor in successors:
-                            class_index = self.class_to_index_mapping_[level][str(successor)]
+                            class_index = self.global_class_to_index_mapping_[level][str(successor)]
 
                             proba_index = np.where(calibrator.classes_ == successor)[0][0]
                             cur_level_probabilities[mask, class_index] = proba[:, proba_index]
diff --git a/hiclass/_calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
index 43a2ac3a..7121fdf9 100644
--- a/hiclass/_calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -5,10 +5,12 @@
 from hiclass._calibration.VennAbersCalibrator import _InductiveVennAbersCalibrator, _CrossVennAbersCalibrator
 from hiclass._calibration.IsotonicRegression import _IsotonicRegression
 from hiclass._calibration.PlattScaling import _PlattScaling
+from hiclass._calibration.calibration_utils import _one_vs_rest_split
 
 
 class _Calibrator(BaseEstimator):
     available_methods = ["ivap", "cvap", "sigmoid", "isotonic"]
+    _multiclass_methods = ["cvap"]
 
     def __init__(self, estimator, method="ivap", **method_params) -> None:
         assert callable(getattr(estimator, 'predict_proba', None))
@@ -16,9 +18,11 @@ def __init__(self, estimator, method="ivap", **method_params) -> None:
         self.method_params = method_params
         self.classes_ = self.estimator.classes_
         self.multiclass = False
+        self.multiclass_support = (method in self._multiclass_methods)
         if method not in self.available_methods:
             raise ValueError(f"{method} is not a valid calibration method.")
         self.method = method
+    
 
     def fit(self, X, y):
         """
@@ -46,20 +50,23 @@ def fit(self, X, y):
         self.calibrators = []
 
         if self.multiclass:
-            # binarize multiclass labels
-            label_binarizer = LabelBinarizer()
-            label_binarizer.fit(self.estimator.classes_)
-            binary_labels = label_binarizer.transform(y).T
-
-            # split scores into k one vs rest splits
-            score_splits = [calibration_scores[:, i] for i in range(calibration_scores.shape[1])]
-
-            for idx, split in enumerate(score_splits):
-                # create a calibrator for each step
+            if self.multiclass_support:
+                # only cvap
+                self.label_encoder = LabelEncoder()
+                encoded_y = self.label_encoder.fit_transform(y)
                 calibrator = self._create_calibrator(self.method, self.method_params)
-                calibrator.fit(binary_labels[idx], split, X)
+                calibrator.fit(encoded_y, calibration_scores, X)
                 self.calibrators.append(calibrator)
 
+            else:
+                # do one vs rest calibration
+                score_splits, label_splits = _one_vs_rest_split(y, calibration_scores, self.estimator)
+                for i in range(len(score_splits)):
+                    # create a calibrator for each split
+                    calibrator = self._create_calibrator(self.method, self.method_params)
+                    calibrator.fit(label_splits[i], score_splits[i], X)
+                    self.calibrators.append(calibrator)
+
         else:
             self.label_encoder = LabelEncoder()
             encoded_y = self.label_encoder.fit_transform(y)
@@ -67,18 +74,25 @@ def fit(self, X, y):
             calibrator.fit(encoded_y, calibration_scores[:, 1], X)
             self.calibrators.append(calibrator)
         return self
+    
 
     def predict_proba(self, X):
         test_scores = self.estimator.predict_proba(X)
 
         if self.multiclass:
-            score_splits = [test_scores[:, i] for i in range(test_scores.shape[1])]
+            if self.multiclass_support:
+                # only cvap
+                return self.calibrators[0].predict_proba(test_scores)
+
+            else:
+                # one vs rest calibration
+                score_splits = [test_scores[:, i] for i in range(test_scores.shape[1])]
 
-            probabilities = np.zeros((X.shape[0], len(self.estimator.classes_)))
-            for idx, split in enumerate(score_splits):
-                probabilities[:, idx] = self.calibrators[idx].predict_proba(split)
+                probabilities = np.zeros((X.shape[0], len(self.estimator.classes_)))
+                for idx, split in enumerate(score_splits):
+                    probabilities[:, idx] = self.calibrators[idx].predict_proba(split)
 
-            probabilities /= probabilities.sum(axis=1, keepdims=True)
+                probabilities /= probabilities.sum(axis=1, keepdims=True)
 
         else:
             probabilities = np.zeros((X.shape[0], 2))
diff --git a/hiclass/_calibration/VennAbersCalibrator.py b/hiclass/_calibration/VennAbersCalibrator.py
index c79ce60b..45c2fc89 100644
--- a/hiclass/_calibration/VennAbersCalibrator.py
+++ b/hiclass/_calibration/VennAbersCalibrator.py
@@ -3,6 +3,8 @@
 from sklearn.model_selection import StratifiedKFold
 from hiclass._calibration.BinaryCalibrator import _BinaryCalibrator
 from scipy.stats import gmean
+from hiclass._calibration.calibration_utils import _one_vs_rest_split
+from collections import defaultdict
 
 
 class _InductiveVennAbersCalibrator(_BinaryCalibrator):
@@ -184,10 +186,14 @@ def __init__(self, estimator, n_folds=5) -> None:
         self.n_folds = n_folds
         self.estimator_type = type(estimator)
         self.estimator_params = estimator.get_params()
+        self.estimator = estimator
+        self.multiclass = False
+        self.use_estimator_fallback = False
+        self.used_cv = True
 
     def fit(self, y, scores, X):
         unique_labels = np.unique(y)
-        assert len(unique_labels) == 2
+        assert len(unique_labels) >= 2
         self.ivaps = []
 
         try:
@@ -200,13 +206,29 @@ def fit(self, y, scores, X):
         except ValueError:
                 splits_x, splits_y = [], []
 
+        # don't use cross validation
         if len(splits_x) == 0 or any([(len(np.unique(y_train)) < 2 or len(np.unique(y_cal)) < 2) for y_train, y_cal in splits_y]):
+            self.used_cv = False
             print("skip cv split due to lack of positive samples!")
-            calibrator = _InductiveVennAbersCalibrator()
-            calibrator.fit(y, scores)
-            self.ivaps.append(calibrator)
+
+            if len(unique_labels) > 2:
+                # use one vs rest
+                score_splits, label_splits = _one_vs_rest_split(y, scores, self.estimator) # TODO use only original calibration samples
+                for i in range(len(score_splits)):
+                    # create a calibrator for each split
+                    calibrator = _InductiveVennAbersCalibrator()
+                    calibrator.fit(label_splits[i], score_splits[i])
+                    self.ivaps.append(calibrator)
+            elif len(unique_labels) == 2 and scores.ndim == 1:
+                calibrator = _InductiveVennAbersCalibrator()
+                calibrator.fit(y, scores) # TODO use only original calibration samples
+                self.ivaps.append(calibrator)
+            else:
+                print("no fitted ivaps!")
+                self.use_estimator_fallback = True
 
         else:
+            self.ovr_ivaps = defaultdict(list)
             for i in range(self.n_folds):
                 X_train, X_cal = splits_x[i][0], splits_x[i][1]
                 y_train, y_cal = splits_y[i][0], splits_y[i][1]
@@ -218,10 +240,25 @@ def fit(self, y, scores, X):
 
                 # calibrate IVAP with left out dataset
                 calibration_scores = model.predict_proba(X_cal)
-
-                calibrator = _InductiveVennAbersCalibrator()
-                calibrator.fit(y_cal, calibration_scores[:, 1])
-                self.ivaps.append(calibrator)
+                
+                if calibration_scores.shape[1] > 2:
+                    self.multiclass = True
+                    # one vs rest calibration
+                    score_splits, label_splits = _one_vs_rest_split(y_cal, calibration_scores, model)
+                    for idx in range(len(score_splits)):
+                        # create a calibrator for each split
+                        calibrator = _InductiveVennAbersCalibrator()
+                        calibrator.fit(label_splits[idx], score_splits[idx])
+                        self.ovr_ivaps[idx].append(calibrator)
+
+                elif calibration_scores.shape[1] == 2 and len(np.unique(y_cal)) == 2:
+                    calibrator = _InductiveVennAbersCalibrator()
+                    calibrator.fit(y_cal, calibration_scores[:, 1])
+                    self.ivaps.append(calibrator)
+
+            if len(self.ivaps) == 0 and len(self.ovr_ivaps) == 0:
+                print("no fitted ivaps!")
+                self.use_estimator_fallback = True
 
         self.fitted = True
 
@@ -230,13 +267,63 @@ def fit(self, y, scores, X):
     def predict_proba(self, scores):
         if not self.fitted:
             raise NotFittedError(f"This {self.name} calibrator is not fitted yet. Call 'fit' with appropriate arguments before using this calibrator.")
-        res = []
-        for calibrator in self.ivaps:
-            res.append(calibrator.predict_intervall(scores))
-        
-        res = np.array(res)
-        p0 = res[:, :, 0]
-        p1 = res[:, :, 1]
-
-        p1_gm = gmean(p1)
-        return p1_gm / (gmean(1 - p0) + p1_gm)
+
+        if self.use_estimator_fallback:
+            return scores
+
+        if self.multiclass:
+
+            score_splits = [scores[:, i] for i in range(scores.shape[1])]
+            probabilities = np.zeros((scores.shape[0], scores.shape[1]))
+
+            if self.used_cv:
+
+                #score_splits = [scores[:, i] for i in range(scores.shape[1])]
+                #probabilities = np.zeros((scores.shape[0], scores.shape[1]))
+
+                for idx, scores in enumerate(score_splits):
+                    res = []
+
+                    if not self.ovr_ivaps[idx]:
+                        continue
+
+                    for calibrator in self.ovr_ivaps[idx]:
+                        res.append(calibrator.predict_intervall(scores))
+                    
+                    res = np.array(res)
+                    
+                    p0 = res[:, :, 0]
+                    p1 = res[:, :, 1]
+
+                    p1_gm = gmean(p1)
+                    probabilities[:, idx] = p1_gm / (gmean(1 - p0) + p1_gm)
+                
+                # normalize
+                #probabilities /= probabilities.sum(axis=1, keepdims=True)
+                #return probabilities
+            
+            else:
+                #score_splits = [scores[:, i] for i in range(scores.shape[1])]
+                #probabilities = np.zeros((scores.shape[0], scores.shape[1]))
+                for idx, scores in enumerate(score_splits):
+                    probabilities[:, idx] = self.ivaps[idx].predict_proba(scores)
+                
+                # normalize
+                #probabilities /= probabilities.sum(axis=1, keepdims=True)
+                #return probabilities
+                    
+            # normalize       
+            probabilities /= probabilities.sum(axis=1, keepdims=True)
+            return probabilities
+
+        else:
+            res = []
+            for calibrator in self.ivaps:
+                res.append(calibrator.predict_intervall(scores))
+            
+            res = np.array(res)
+            p0 = res[:, :, 0]
+            p1 = res[:, :, 1]
+
+            p1_gm = gmean(p1)
+            return p1_gm / (gmean(1 - p0) + p1_gm)
diff --git a/hiclass/_calibration/calibration_utils.py b/hiclass/_calibration/calibration_utils.py
new file mode 100644
index 00000000..b0a06669
--- /dev/null
+++ b/hiclass/_calibration/calibration_utils.py
@@ -0,0 +1,13 @@
+from sklearn.preprocessing import LabelBinarizer
+
+def _one_vs_rest_split(y, scores, estimator):
+        # binarize multiclass labels
+        label_binarizer = LabelBinarizer()
+        label_binarizer.fit(estimator.classes_)
+        binary_labels = label_binarizer.transform(y).T
+
+        # split scores into k one vs rest splits
+        score_splits = [scores[:, i] for i in range(scores.shape[1])]
+        label_splits = [binary_labels[i] for i in range(len(score_splits))]
+
+        return score_splits, label_splits
\ No newline at end of file
diff --git a/hiclass/metrics.py b/hiclass/metrics.py
index 65300880..c6509cc8 100644
--- a/hiclass/metrics.py
+++ b/hiclass/metrics.py
@@ -258,11 +258,13 @@ def _prepare_data(classifier, y_true, y_prob, level, y_pred=None):
     y_true = make_leveled(y_true)
     y_true = classifier._disambiguate(y_true)
     y_true = np.array(list(map(lambda x: x[level], y_true)))
+    y_true = np.array([label.split(classifier.separator_)[level] for label in y_true])
 
     if y_pred is not None:
         y_pred = make_leveled(y_pred)
         y_pred = classifier._disambiguate(y_pred)
         y_pred = np.array(list(map(lambda x: x[level], y_pred)))
+        y_pred = np.array([label.split(classifier.separator_)[level] for label in y_pred])
 
     unique_labels = np.unique(y_true).astype("str")
     # add labels not seen in the training process

From 98b204b63cd0f89889a3d26f1932f36cde43748b Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Wed, 3 Apr 2024 22:59:16 +0200
Subject: [PATCH 36/65] refactoring, fix probability combiner tests

---
 hiclass/HierarchicalClassifier.py             | 38 +++-----
 hiclass/LocalClassifierPerLevel.py            |  2 +-
 hiclass/LocalClassifierPerNode.py             |  2 +-
 hiclass/LocalClassifierPerParentNode.py       |  2 +-
 .../ArithmeticMeanCombiner.py                 |  2 +-
 .../GeometricMeanCombiner.py                  |  2 +-
 .../probability_combiner/MultiplyCombiner.py  |  2 +-
 .../ProbabilityCombiner.py                    |  3 +-
 tests/test_ProbabilityCombiner.py             | 91 +++++++++++++------
 tests/test_metrics.py                         |  7 +-
 10 files changed, 90 insertions(+), 61 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index 4499f61a..8fa274a7 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -175,6 +175,12 @@ def _pre_fit(self, X, y, sample_weight):
             self.global_classes_ = [np.unique(self.y_).astype("str")]
             self.global_class_to_index_mapping_ = [{self.global_classes_[0][index] : index for index in range(len(self.global_classes_[0]))}]
 
+        classes_ = [self.global_classes_[0]]
+        for level in range(1, len(self.max_level_dimensions_)):
+            classes_.append(np.sort(np.unique([label.split(self.separator_)[level] for label in self.global_classes_[level]])))
+        self.classes_ = classes_
+        self.class_to_index_mapping_ = [{local_labels[index]: index for index in range(len(local_labels))} for local_labels in classes_]
+
         # Create and configure logger
         self._create_logger()
 
@@ -503,34 +509,14 @@ def _reorder_local_probabilities(self, probabilities, local_labels, level):
 
     def _combine_and_reorder(self, proba):
         res = [proba[0]]
-        classes_ = [self.global_classes_[0]]
-        for level in range(1, len(proba)):
-            # get local labels
-            local_labels = np.sort(np.unique([label.split(self.separator_)[level] for label in self.global_classes_[level]]))
-
-            oldToNew = {}
-            for label in self.global_classes_[level]:
-                # local label
-                local_label = label.split(self.separator_)[level]
-
-                # old index
-                # old_index = self.global_class_to_index_mapping_[level][label]
-                new_index = np.where(local_labels == local_label)[0][0]
-
-                oldToNew[label] = local_label, new_index
-
-            res_proba = np.zeros(shape=(proba[level].shape[0], len(local_labels)))
+        for level in range(1, self.max_levels_):
+            res_proba = np.zeros(shape=(proba[level].shape[0], len(self.classes_[level])))
 
             for old_label in self.global_classes_[level]:
                 old_idx = self.global_class_to_index_mapping_[level][old_label]
-                _, new_idx = oldToNew[old_label]
+                local_label = old_label.split(self.separator_)[level]
+                new_idx = self.class_to_index_mapping_[level][local_label]
                 res_proba[:, new_idx] += proba[level][:, old_idx]
 
-            res.append(res_proba)
-            classes_.append(local_labels)
-            class_to_index_mapping_ = [{local_labels[index]: index for index in range(len(local_labels))} for local_labels in classes_]
-            
-        return classes_, class_to_index_mapping_, res
-
-
-
+            res.append(res_proba) 
+        return res
diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index e41bdabe..fa7e991e 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -214,7 +214,7 @@ def predict_proba(self, X):
         y[:, 0] = calibrator.classes_[np.argmax(proba, axis=1)]
 
         level_probability_list = [proba] + self._predict_proba_remaining_levels(X, y)
-        self.classes_, self.class_to_index_mapping_, level_probability_list = self._combine_and_reorder(level_probability_list)
+        level_probability_list = self._combine_and_reorder(level_probability_list)
 
         # combine probabilities
         if self.probability_combiner:
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index 3a04a4e3..67cf7e14 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -306,7 +306,7 @@ def predict_proba(self, X):
         ]
 
         # combine probabilities horizontally
-        self.classes_, self.class_to_index_mapping_, level_probability_list = self._combine_and_reorder(level_probability_list)
+        level_probability_list = self._combine_and_reorder(level_probability_list)
         
         # combine probabilities vertically
         if self.probability_combiner:
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index 7a1d5c6a..a206c8e7 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -205,7 +205,7 @@ def predict_proba(self, X):
         y[:, 0] = calibrator.classes_[np.argmax(proba, axis=1)]
         level_probability_list = [proba] + self._predict_proba_remaining_levels(X, y)
     
-        self.classes_, self.class_to_index_mapping_, level_probability_list = self._combine_and_reorder(level_probability_list)
+        level_probability_list = self._combine_and_reorder(level_probability_list)
 
         # combine probabilities
         if self.probability_combiner:
diff --git a/hiclass/probability_combiner/ArithmeticMeanCombiner.py b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
index 90d1570d..e042b9f1 100644
--- a/hiclass/probability_combiner/ArithmeticMeanCombiner.py
+++ b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
@@ -25,4 +25,4 @@ def combine(self, proba):
 
             res.append(level_probs)
             sums.append(level_sum)
-        return self._normalize(res)
+        return self._normalize(res) if self.normalize else res
diff --git a/hiclass/probability_combiner/GeometricMeanCombiner.py b/hiclass/probability_combiner/GeometricMeanCombiner.py
index 6326dedd..b9c125cd 100644
--- a/hiclass/probability_combiner/GeometricMeanCombiner.py
+++ b/hiclass/probability_combiner/GeometricMeanCombiner.py
@@ -25,4 +25,4 @@ def combine(self, proba):
             
             log_sum.append(level_log_sum)
             res.append(level_probs)
-        return self._normalize(res)
+        return self._normalize(res) if self.normalize else res
diff --git a/hiclass/probability_combiner/MultiplyCombiner.py b/hiclass/probability_combiner/MultiplyCombiner.py
index c790d8b5..a311b063 100644
--- a/hiclass/probability_combiner/MultiplyCombiner.py
+++ b/hiclass/probability_combiner/MultiplyCombiner.py
@@ -20,4 +20,4 @@ def combine(self, proba):
                 level_probs[:, index] = predecessors_combined_prob * proba[level][:, index]
 
             res.append(level_probs)
-        return self._normalize(res)
+        return self._normalize(res) if self.normalize else res
diff --git a/hiclass/probability_combiner/ProbabilityCombiner.py b/hiclass/probability_combiner/ProbabilityCombiner.py
index 861a79c8..59923625 100644
--- a/hiclass/probability_combiner/ProbabilityCombiner.py
+++ b/hiclass/probability_combiner/ProbabilityCombiner.py
@@ -6,8 +6,9 @@
 
 class ProbabilityCombiner(abc.ABC):
 
-    def __init__(self, classifier) -> None:
+    def __init__(self, classifier, normalize=True) -> None:
         self.classifier = classifier
+        self.normalize = normalize
 
     @abc.abstractmethod
     def combine(self, proba: List[np.ndarray]) -> List[np.ndarray]:
diff --git a/tests/test_ProbabilityCombiner.py b/tests/test_ProbabilityCombiner.py
index c1525895..f34a26e1 100644
--- a/tests/test_ProbabilityCombiner.py
+++ b/tests/test_ProbabilityCombiner.py
@@ -6,35 +6,51 @@
 import math
 from scipy.stats import gmean
 
+from hiclass.HierarchicalClassifier import make_leveled
+
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
 from hiclass.probability_combiner import MultiplyCombiner, ArithmeticMeanCombiner, GeometricMeanCombiner
 
 
+def _disambiguate(y, separator):
+    if y.ndim == 2:
+        new_y = []
+        for i in range(y.shape[0]):
+            row = [str(y[i, 0])]
+            for j in range(1, y.shape[1]):
+                parent = str(row[-1])
+                child = str(y[i, j])
+                row.append(parent + separator + child)
+            new_y.append(np.asarray(row, dtype=np.str_))
+        return np.array(new_y)
+    return y
+
 @pytest.fixture
 def one_sample_probs_with_hierarchy():
     hierarchy = nx.DiGraph()
     root_node = "hiclass::root"
     hierarchy.add_node(root_node)
-    classes_ = [[], [], []]
+    separator = "::HiClass::Separator::"
+    y_ = []
 
     for i in range(3):
         # first level
-        first_level_node = f'level_0_node_{i}'
-        classes_[0].append(first_level_node)
+        first_level_node = f'node_{i}'
         hierarchy.add_node(first_level_node)
         hierarchy.add_edge(root_node, first_level_node)
         for j in range(3):
             # second level
-            second_level_node = f'level_1_node_{i}:{j}'
-            classes_[1].append(second_level_node)
-            hierarchy.add_node(second_level_node)
-            hierarchy.add_edge(first_level_node, second_level_node)
+            second_level_node = f'node_{i}:{j}'
+            second_level_label = _disambiguate(np.array([[first_level_node, second_level_node]]), separator)[0][1]
+            hierarchy.add_node(second_level_label)
+            hierarchy.add_edge(first_level_node, second_level_label)
             for k in range(2):
                 # third level
-                third_level_node = f'level_2_node_{i}:{j}:{k}'
-                classes_[2].append(third_level_node)
-                hierarchy.add_node(third_level_node)
-                hierarchy.add_edge(second_level_node, third_level_node)
+                third_level_node = f'node_{i}:{j}:{k}'
+                third_level_label = _disambiguate(np.array([[first_level_node, second_level_node, third_level_node]]), separator)[0][2]
+                y_.append([first_level_node, second_level_node, third_level_node])
+                hierarchy.add_node(third_level_label)
+                hierarchy.add_edge(second_level_label, third_level_label)
 
     probs = [
         np.array([[0.3, 0.5, 0.2]]), # level 0
@@ -43,19 +59,34 @@ def one_sample_probs_with_hierarchy():
     ]
     assert all([np.sum(probs[level], axis=1) == 1 for level in range(3)])
 
-    return hierarchy, probs, classes_
+    y_ = np.array(y_)
+    y_ = make_leveled(y_)
+    y_ = _disambiguate(y_, separator)
+
+    global_classes_ = [np.unique(y_[:, level]).astype("str") for level in range(y_.shape[1])]
+
+    classes_ = [global_classes_[0]]
+    for level in range(1, len(y_[1])):
+        classes_.append(np.sort(np.unique([label.split(separator)[level] for label in global_classes_[level]])))
+
+    class_to_index_mapping_ = [{local_labels[index]: index for index in range(len(local_labels))} for local_labels in classes_]
+
+    return hierarchy, probs, global_classes_, classes_, class_to_index_mapping_
 
 def test_multiply_combiner(one_sample_probs_with_hierarchy):
-    hierarchy, probs, classes = one_sample_probs_with_hierarchy
+    hierarchy, probs, global_classes, classes_, class_to_index_mapping_ = one_sample_probs_with_hierarchy
     obj = HierarchicalClassifier()
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
-    classifier.classes_ = classes
-    classifier.max_levels_ = len(classes)
-    classifier.class_to_index_mapping_ = [{classifier.classes_[level][index]: index for index in range(len(classifier.classes_[level]))} for level in range(classifier.max_levels_)]
+    classifier.global_classes_ = global_classes
+    classifier.max_levels_ = len(global_classes)
+    classifier.classes_ = classes_
+    classifier.class_to_index_mapping_ = class_to_index_mapping_
+    
     classifier.hierarchy_ = hierarchy
+    classifier.separator_ = "::HiClass::Separator::"
 
-    combiner = MultiplyCombiner(classifier=classifier)
+    combiner = MultiplyCombiner(classifier=classifier, normalize=False)
     combined_probs = combiner.combine(probs)
 
     # check combined probability of first node for both levels
@@ -73,16 +104,19 @@ def test_multiply_combiner(one_sample_probs_with_hierarchy):
     assert math.isclose(combined_probs[2][0][-1], probs[0][0][-1] * probs[1][0][-1] * probs[2][0][-1], abs_tol=1e-4)
 
 def test_arithmetic_mean_combiner(one_sample_probs_with_hierarchy):
-    hierarchy, probs, classes = one_sample_probs_with_hierarchy
+    hierarchy, probs, global_classes, classes_, class_to_index_mapping_ = one_sample_probs_with_hierarchy
     obj = HierarchicalClassifier()
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
-    classifier.classes_ = classes
-    classifier.max_levels_ = len(classes)
-    classifier.class_to_index_mapping_ = [{classifier.classes_[level][index]: index for index in range(len(classifier.classes_[level]))} for level in range(classifier.max_levels_)]
+    
+    classifier.global_classes_ = global_classes
+    classifier.max_levels_ = len(global_classes)
+    classifier.classes_ = classes_
+    classifier.class_to_index_mapping_ = class_to_index_mapping_
     classifier.hierarchy_ = hierarchy
+    classifier.separator_ = "::HiClass::Separator::"
 
-    combiner = ArithmeticMeanCombiner(classifier=classifier)
+    combiner = ArithmeticMeanCombiner(classifier=classifier, normalize=False)
     combined_probs = combiner.combine(probs)
 
     # check combined probability of first node for both levels
@@ -100,16 +134,19 @@ def test_arithmetic_mean_combiner(one_sample_probs_with_hierarchy):
     assert math.isclose(combined_probs[2][0][-1], (probs[0][0][-1] + probs[1][0][-1] + probs[2][0][-1]) / 3, abs_tol=1e-4)
 
 def test_geometric_mean_combiner(one_sample_probs_with_hierarchy):
-    hierarchy, probs, classes = one_sample_probs_with_hierarchy
+    hierarchy, probs, global_classes, classes_, class_to_index_mapping_ = one_sample_probs_with_hierarchy
     obj = HierarchicalClassifier()
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
-    classifier.classes_ = classes
-    classifier.max_levels_ = len(classes)
-    classifier.class_to_index_mapping_ = [{classifier.classes_[level][index]: index for index in range(len(classifier.classes_[level]))} for level in range(classifier.max_levels_)]
+    classifier.separator_ = "::HiClass::Separator::"
+    classifier.global_classes_ = global_classes
+    classifier.classes_ = classes_
+
+    classifier.max_levels_ = len(global_classes)
+    classifier.class_to_index_mapping_ = class_to_index_mapping_
     classifier.hierarchy_ = hierarchy
 
-    combiner = GeometricMeanCombiner(classifier=classifier)
+    combiner = GeometricMeanCombiner(classifier=classifier, normalize=False)
     combined_probs = combiner.combine(probs)
 
     # check combined probability of first node for both levels
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index b8d69b00..fd3d0686 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -375,7 +375,8 @@ def test_local_brier_score(uncertainty_data):
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
     classifier.classes_ = [[0, 1, 2]]
-
+    classifier.separator_ = "::HiClass::Separator::"
+    
     brier_score = _multiclass_brier_score(classifier, y_true, prob, level=0)
     assert math.isclose(brier_score, 0.34852, abs_tol=1e-4)
 
@@ -385,6 +386,7 @@ def test_local_log_loss(uncertainty_data):
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
     classifier.classes_ = [[0, 1, 2]]
+    classifier.separator_ = "::HiClass::Separator::"
 
     log_loss = _log_loss(classifier, y_true, prob, level=0)
     assert math.isclose(log_loss, 0.61790, abs_tol=1e-4)
@@ -395,6 +397,7 @@ def test_expected_calibration_error(uncertainty_data):
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
     classifier.classes_ = [[0, 1, 2]]
+    classifier.separator_ = "::HiClass::Separator::"
 
     ece = _expected_calibration_error(classifier, y_true, prob, y_pred, level=0, n_bins=3)
     assert math.isclose(ece, 0.118, abs_tol=1e-4)
@@ -405,6 +408,7 @@ def test_statistical_calibration_error(uncertainty_data):
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
     classifier.classes_ = [[0, 1, 2]]
+    classifier.separator_ = "::HiClass::Separator::"
 
     sce = _statistical_calibration_error(classifier, y_true, prob, y_pred, level=0, n_bins=3)
     assert math.isclose(sce, 0.3889, abs_tol=1e-3)
@@ -415,6 +419,7 @@ def test_adaptive_calibration_error(uncertainty_data):
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
     classifier.classes_ = [[0, 1, 2]]
+    classifier.separator_ = "::HiClass::Separator::"
 
     ace = _adaptive_calibration_error(classifier, y_true, prob, y_pred, level=0, n_ranges=3)
     assert math.isclose(ace, 0.44, abs_tol=1e-3)

From 922ea9595ac83255950bd652750b40383b2757fe Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Thu, 4 Apr 2024 18:15:50 +0200
Subject: [PATCH 37/65] make calibration methods compatible with sklearn
 check_is_fitted method

---
 hiclass/_calibration/BinaryCalibrator.py    |  6 ++++++
 hiclass/_calibration/Calibrator.py          | 10 +++++++++-
 hiclass/_calibration/IsotonicRegression.py  |  9 ++++-----
 hiclass/_calibration/PlattScaling.py        |  9 ++++-----
 hiclass/_calibration/VennAbersCalibrator.py | 16 +++++++---------
 5 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/hiclass/_calibration/BinaryCalibrator.py b/hiclass/_calibration/BinaryCalibrator.py
index d2fcd015..20038537 100644
--- a/hiclass/_calibration/BinaryCalibrator.py
+++ b/hiclass/_calibration/BinaryCalibrator.py
@@ -10,3 +10,9 @@ def fit(self, y, scores, X=None):  # pragma: no cover
     @abc.abstractmethod
     def predict_proba(self, scores, X=None):  # pragma: no cover
         ...
+        
+    def __sklearn_is_fitted__(self):
+        """
+        Check fitted status and return a Boolean value.
+        """
+        return hasattr(self, "_is_fitted") and self._is_fitted
diff --git a/hiclass/_calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
index 7121fdf9..671736bb 100644
--- a/hiclass/_calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -16,7 +16,7 @@ def __init__(self, estimator, method="ivap", **method_params) -> None:
         assert callable(getattr(estimator, 'predict_proba', None))
         self.estimator = estimator
         self.method_params = method_params
-        self.classes_ = self.estimator.classes_
+        #self.classes_ = self.estimator.classes_
         self.multiclass = False
         self.multiclass_support = (method in self._multiclass_methods)
         if method not in self.available_methods:
@@ -42,6 +42,7 @@ def fit(self, X, y):
         self : object
             Calibrated estimator.
         """
+        self.classes_ = self.estimator.classes_
         calibration_scores = self.estimator.predict_proba(X)
 
         if calibration_scores.shape[1] > 2:
@@ -73,6 +74,7 @@ def fit(self, X, y):
             calibrator = self._create_calibrator(self.method, self.method_params)
             calibrator.fit(encoded_y, calibration_scores[:, 1], X)
             self.calibrators.append(calibrator)
+        self._is_fitted = True
         return self
     
 
@@ -110,3 +112,9 @@ def _create_calibrator(self, name, params):
             return _PlattScaling()
         elif name == "isotonic":
             return _IsotonicRegression(params)
+
+    def __sklearn_is_fitted__(self):
+        """
+        Check fitted status and return a Boolean value.
+        """
+        return hasattr(self, "_is_fitted") and self._is_fitted
diff --git a/hiclass/_calibration/IsotonicRegression.py b/hiclass/_calibration/IsotonicRegression.py
index 3ec7c0d0..c68009ea 100644
--- a/hiclass/_calibration/IsotonicRegression.py
+++ b/hiclass/_calibration/IsotonicRegression.py
@@ -1,23 +1,22 @@
 from hiclass._calibration.BinaryCalibrator import _BinaryCalibrator
 from sklearn.isotonic import IsotonicRegression as SkLearnIR
-from sklearn.exceptions import NotFittedError
+from sklearn.utils.validation import check_is_fitted
 
 
 class _IsotonicRegression(_BinaryCalibrator):
     name = "IsotonicRegression"
 
     def __init__(self, params={}) -> None:
-        self.fitted = False
+        self._is_fitted = False
         if "out_of_bounds" not in params:
             params["out_of_bounds"] = "clip"
         self.isotonic_regression = SkLearnIR(**params)
 
     def fit(self, y, scores, X=None):
         self.isotonic_regression.fit(scores, y)
-        self.fitted = True
+        self._is_fitted = True
         return self
 
     def predict_proba(self, scores, X=None):
-        if not self.fitted:
-            raise NotFittedError(f"This {self.name} calibrator is not fitted yet. Call 'fit' with appropriate arguments before using this calibrator.")
+        check_is_fitted(self)
         return self.isotonic_regression.predict(scores)
diff --git a/hiclass/_calibration/PlattScaling.py b/hiclass/_calibration/PlattScaling.py
index d8a1c0f2..b3109003 100644
--- a/hiclass/_calibration/PlattScaling.py
+++ b/hiclass/_calibration/PlattScaling.py
@@ -1,21 +1,20 @@
 from hiclass._calibration.BinaryCalibrator import _BinaryCalibrator
 from sklearn.calibration import _SigmoidCalibration
-from sklearn.exceptions import NotFittedError
+from sklearn.utils.validation import check_is_fitted
 
 
 class _PlattScaling(_BinaryCalibrator):
     name = "PlattScaling"
 
     def __init__(self) -> None:
-        self.fitted = False
+        self._is_fitted = False
         self.platt_scaling = _SigmoidCalibration()
 
     def fit(self, y, scores, X=None):
         self.platt_scaling.fit(scores, y)
-        self.fitted = True
+        self._is_fitted = True
         return self
 
     def predict_proba(self, scores, X=None):
-        if not self.fitted:
-            raise NotFittedError(f"This {self.name} calibrator is not fitted yet. Call 'fit' with appropriate arguments before using this calibrator.")
+        check_is_fitted(self)
         return self.platt_scaling.predict(scores)
diff --git a/hiclass/_calibration/VennAbersCalibrator.py b/hiclass/_calibration/VennAbersCalibrator.py
index 45c2fc89..1268c232 100644
--- a/hiclass/_calibration/VennAbersCalibrator.py
+++ b/hiclass/_calibration/VennAbersCalibrator.py
@@ -1,17 +1,17 @@
 import numpy as np
-from sklearn.exceptions import NotFittedError
 from sklearn.model_selection import StratifiedKFold
 from hiclass._calibration.BinaryCalibrator import _BinaryCalibrator
 from scipy.stats import gmean
 from hiclass._calibration.calibration_utils import _one_vs_rest_split
 from collections import defaultdict
+from sklearn.utils.validation import check_is_fitted
 
 
 class _InductiveVennAbersCalibrator(_BinaryCalibrator):
     name = "InductiveVennAbersCalibrator"
 
     def __init__(self):
-        self.fitted = False
+        self._is_fitted = False
 
     def fit(self, y, scores, X=None):
         positive_label = 1
@@ -154,13 +154,12 @@ def compute_f0(prev_stack, csd):
         self.F1 = compute_f1(f1_stack, csd_1)
         self.F0 = compute_f0(f0_stack, csd_0)
         self.unique_elements = unique_elements
-        self.fitted = True
+        self._is_fitted = True
 
         return self
 
     def predict_proba(self, scores, X=None):
-        if not self.fitted:
-            raise NotFittedError(f"This {self.name} calibrator is not fitted yet. Call 'fit' with appropriate arguments before using this calibrator.")
+        check_is_fitted(self)
         lower = np.searchsorted(self.unique_elements, scores, side="left")
         upper = np.searchsorted(self.unique_elements[:-1], scores, side="right") + 1
 
@@ -182,7 +181,7 @@ class _CrossVennAbersCalibrator(_BinaryCalibrator):
     name = "CrossVennAbersCalibrator"
 
     def __init__(self, estimator, n_folds=5) -> None:
-        self.fitted = False
+        self._is_fitted = False
         self.n_folds = n_folds
         self.estimator_type = type(estimator)
         self.estimator_params = estimator.get_params()
@@ -260,13 +259,12 @@ def fit(self, y, scores, X):
                 print("no fitted ivaps!")
                 self.use_estimator_fallback = True
 
-        self.fitted = True
+        self._is_fitted = True
 
         return self
 
     def predict_proba(self, scores):
-        if not self.fitted:
-            raise NotFittedError(f"This {self.name} calibrator is not fitted yet. Call 'fit' with appropriate arguments before using this calibrator.")
+        check_is_fitted(self)
 
         if self.use_estimator_fallback:
             return scores

From 493112fb8f8c32819beca09a66473ade61905bbc Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Thu, 4 Apr 2024 18:16:56 +0200
Subject: [PATCH 38/65] small refactorings

---
 hiclass/HierarchicalClassifier.py  | 13 +++++--------
 hiclass/LocalClassifierPerLevel.py |  6 +++---
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index 8fa274a7..b1a47052 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -243,18 +243,15 @@ def calibrate(self, X, y):
                 self.logger_.info(f"Not sparse Calibration size: {X.shape} train size: {self.X_.shape}")
                 self.X_cal = np.vstack([self.X_, X])
                 self.logger_.info(f"CV Dataset X: {str(type(self.X_cal))} {str(self.X_cal.shape)}")
+            
+            y = make_leveled(y)
+            y = self._disambiguate(y)
+            y = self._convert_1d_y_to_2d(y)
             self.y_cal = np.vstack([self.y_, y])
-            self.y_cal = make_leveled(self.y_cal)
-            self.y_cal = self._disambiguate(self.y_cal)
-            self.y_cal = self._convert_1d_y_to_2d(self.y_cal)
         else:
             self.X_cal = X
             self.y_cal = y
-
-            self.y_cal = make_leveled(self.y_cal)
-            self.y_cal = self._disambiguate(self.y_cal)
-            self.y_cal = self._convert_1d_y_to_2d(self.y_cal)
-
+            
         self.logger_.info("Calibrating")
 
         # Create a calibrator for each local classifier
diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index fa7e991e..c6ce45c0 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -206,7 +206,7 @@ def predict_proba(self, X):
 
         # Predict first level
         classifier = self.local_classifiers_[0]
-        calibrator = self.local_calibrators_[0] if self.local_calibrators_ else None
+        calibrator = self.local_calibrators_[0] if hasattr(self, 'self.local_calibrators_') else None
 
         # use classifier as a fallback if no calibrator is available
         calibrator = calibrator or classifier
@@ -228,12 +228,12 @@ def _predict_proba_remaining_levels(self, X, y):
         level_probability_list = []
         for level in range(1, y.shape[1]):
             classifier = self.local_classifiers_[level]
-            calibrator = self.local_calibrators_[level] if self.local_calibrators_ else None
+            calibrator = self.local_calibrators_[level] if hasattr(self, 'self.local_calibrators_') else None
             # use classifier as a fallback if no calibrator is available
             calibrator = calibrator or classifier
             probabilities = calibrator.predict_proba(X)
             # sort probabilities
-            probabilities = self._reorder_local_probabilities(probabilities, calibrator.classes_, level)
+            #probabilities = self._reorder_local_probabilities(probabilities, calibrator.classes_, level)
             level_probability_list.append(probabilities)
         return level_probability_list
 

From 028af8605c91a28cef470f3435397c2e1f3adf4f Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Thu, 4 Apr 2024 18:17:36 +0200
Subject: [PATCH 39/65] add predict_proba tests for LocalClassifierPerLevel

---
 tests/test_LocalClassifierPerLevel.py | 127 +++++++++++++++++++++++++-
 tests/test_calibration.py             |   2 +-
 2 files changed, 124 insertions(+), 5 deletions(-)

diff --git a/tests/test_LocalClassifierPerLevel.py b/tests/test_LocalClassifierPerLevel.py
index 27312f85..064b533d 100644
--- a/tests/test_LocalClassifierPerLevel.py
+++ b/tests/test_LocalClassifierPerLevel.py
@@ -3,13 +3,15 @@
 import networkx as nx
 import numpy as np
 import pytest
-from numpy.testing import assert_array_equal
+from numpy.testing import assert_array_equal, assert_array_almost_equal
 from scipy.sparse import csr_matrix
 from sklearn.exceptions import NotFittedError
 from sklearn.linear_model import LogisticRegression
 from sklearn.utils.estimator_checks import parametrize_with_checks
 from sklearn.utils.validation import check_is_fitted
 from hiclass import LocalClassifierPerLevel
+from hiclass._calibration.Calibrator import _Calibrator
+from hiclass.HierarchicalClassifier import make_leveled
 
 
 @parametrize_with_checks([LocalClassifierPerLevel()])
@@ -19,10 +21,12 @@ def test_sklearn_compatible_estimator(estimator, check):
 
 @pytest.fixture
 def digraph_logistic_regression():
-    digraph = LocalClassifierPerLevel(local_classifier=LogisticRegression())
+    digraph = LocalClassifierPerLevel(local_classifier=LogisticRegression(), calibration_method="ivap")
     digraph.hierarchy_ = nx.DiGraph([("a", "b"), ("a", "c")])
     digraph.y_ = np.array([["a", "b"], ["a", "c"]])
     digraph.X_ = np.array([[1, 2], [3, 4]])
+    digraph.y_cal = np.array([["a", "b"], ["a", "c"]])
+    digraph.X_cal = np.array([[1, 2], [3, 4]])
     digraph.logger_ = logging.getLogger("LCPL")
     digraph.root_ = "a"
     digraph.sample_weight_ = None
@@ -42,6 +46,15 @@ def test_initialize_local_classifiers(digraph_logistic_regression):
             LogisticRegression,
         )
 
+def test_initialize_local_calibrators(digraph_logistic_regression):
+    digraph_logistic_regression._initialize_local_classifiers()
+    digraph_logistic_regression._initialize_local_calibrators()
+    for calibrator in digraph_logistic_regression.local_calibrators_:
+        assert isinstance(
+            calibrator,
+            _Calibrator
+        )
+
 
 def test_fit_digraph(digraph_logistic_regression):
     classifiers = [
@@ -58,6 +71,26 @@ def test_fit_digraph(digraph_logistic_regression):
             pytest.fail(repr(e))
     assert 1
 
+def test_calibrate_digraph(digraph_logistic_regression):
+    classifiers = [
+        LogisticRegression(),
+        LogisticRegression(),
+    ]
+    digraph_logistic_regression.n_jobs = 2
+    digraph_logistic_regression.local_classifiers_ = classifiers
+    digraph_logistic_regression._fit_digraph(local_mode=True)
+
+    calibrators = [_Calibrator(classifier) for classifier in digraph_logistic_regression.local_classifiers_]
+    digraph_logistic_regression.local_calibrators_ = calibrators
+    digraph_logistic_regression._calibrate_digraph(local_mode=True)
+
+    try:
+        check_is_fitted(digraph_logistic_regression.local_calibrators_[1])
+    except NotFittedError as e:
+        pytest.fail(repr(e))
+    assert 1
+
+
 
 def test_fit_digraph_joblib_multiprocessing(digraph_logistic_regression):
     classifiers = [
@@ -75,10 +108,36 @@ def test_fit_digraph_joblib_multiprocessing(digraph_logistic_regression):
             pytest.fail(repr(e))
     assert 1
 
+def test_calibrate_digraph_joblib_multiprocessing(digraph_logistic_regression):
+    classifiers = [
+        LogisticRegression(),
+        LogisticRegression(),
+    ]
+    digraph_logistic_regression.n_jobs = 2
+    digraph_logistic_regression.local_classifiers_ = classifiers
+    digraph_logistic_regression._fit_digraph(local_mode=True, use_joblib=True)
+
+    calibrators = [_Calibrator(classifier) for classifier in digraph_logistic_regression.local_classifiers_]
+    digraph_logistic_regression.local_calibrators_ = calibrators
+    digraph_logistic_regression._calibrate_digraph(local_mode=True, use_joblib=True)
+
+    try:
+        check_is_fitted(digraph_logistic_regression.local_calibrators_[1])
+    except NotFittedError as e:
+        pytest.fail(repr(e))
+    assert 1
+
+
 
 @pytest.fixture
 def fitted_logistic_regression():
-    digraph = LocalClassifierPerLevel(local_classifier=LogisticRegression())
+    digraph = LocalClassifierPerLevel(
+        local_classifier=LogisticRegression(),
+        return_all_probabilities=True,
+        calibration_method="ivap",
+        probability_combiner=None)
+    
+    digraph.separator_ = "::HiClass::Separator::"
     digraph.hierarchy_ = nx.DiGraph(
         [("r", "1"), ("r", "2"), ("1", "1.1"), ("1", "1.2"), ("2", "2.1"), ("2", "2.2")]
     )
@@ -86,9 +145,22 @@ def fitted_logistic_regression():
     digraph.X_ = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
     digraph.logger_ = logging.getLogger("LCPL")
     digraph.max_levels_ = 2
+
+    # for predict_proba
+    tmp_labels = digraph._disambiguate(make_leveled(digraph.y_))
+    digraph.max_level_dimensions_ = np.array([len(np.unique(tmp_labels[:, level])) for level in range(tmp_labels.shape[1])])
+    digraph.global_classes_ = [np.unique(tmp_labels[:, level]).astype("str") for level in range(tmp_labels.shape[1])]
+    digraph.global_class_to_index_mapping_ = [{digraph.global_classes_[level][index]: index for index in range(len(digraph.global_classes_[level]))} for level in range(tmp_labels.shape[1])]
+
+    classes_ = [digraph.global_classes_[0]]
+    for level in range(1, digraph.max_levels_):
+        classes_.append(np.sort(np.unique([label.split(digraph.separator_)[level] for label in digraph.global_classes_[level]])))
+    digraph.classes_ = classes_
+    digraph.class_to_index_mapping_ = [{local_labels[index]: index for index in range(len(local_labels))} for local_labels in classes_]
+
+
     digraph.dtype_ = "<U3"
     digraph.root_ = "r"
-    digraph.separator_ = "::HiClass::Separator::"
     digraph.masks_ = [
         [True, True, True, True],
         [True, True, True, True],
@@ -117,6 +189,23 @@ def test_predict_sparse(fitted_logistic_regression):
     assert_array_equal(ground_truth, prediction)
 
 
+def test_predict_proba(fitted_logistic_regression):
+    proba = fitted_logistic_regression.predict_proba([[7, 8], [5, 6], [3, 4], [1, 2]])
+    assert len(proba) == 2
+    assert proba[0].shape == (4, 2)
+    assert proba[1].shape == (4, 4)
+    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
+    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+
+
+def test_predict_proba_sparse(fitted_logistic_regression):
+    proba = fitted_logistic_regression.predict_proba(csr_matrix([[7, 8], [5, 6], [3, 4], [1, 2]]))
+    assert len(proba) == 2
+    assert proba[0].shape == (4, 2)
+    assert proba[1].shape == (4, 4)
+    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
+    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+
 def test_fit_predict():
     lcpl = LocalClassifierPerLevel(local_classifier=LogisticRegression())
     x = np.array([[1, 2], [3, 4]])
@@ -133,3 +222,33 @@ def test_fit_predict():
             pytest.fail(repr(e))
     predictions = lcpl.predict(x)
     assert_array_equal(y, predictions)
+
+def test_fit_calibrate_predict_proba():
+    lcpl = LocalClassifierPerLevel(
+        local_classifier=LogisticRegression(), 
+        return_all_probabilities=True,
+        calibration_method="ivap",
+        probability_combiner="geometric")
+    
+    x = np.array([[1, 2], [3, 4]])
+    y = np.array([["a", "b"], ["b", "c"]])
+    ground_truth = np.array(
+        [["a", "b"], ["a::HiClass::Separator::b", "b::HiClass::Separator::c"]]
+    )
+    lcpl.fit(x, y)
+    lcpl.calibrate(x, y)
+    for level, classifier in enumerate(lcpl.local_classifiers_):
+        try:
+            check_is_fitted(classifier)
+            assert_array_equal(ground_truth[level], classifier.classes_)
+        except NotFittedError as e:
+            pytest.fail(repr(e))
+    predictions = lcpl.predict(x)
+    proba = lcpl.predict_proba(x)
+    assert_array_equal(y, predictions)
+
+    assert len(proba) == 2
+    assert proba[0].shape == (2, 2)
+    assert proba[1].shape == (2, 2)
+    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
+    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
diff --git a/tests/test_calibration.py b/tests/test_calibration.py
index 469a8a7e..8283acd7 100644
--- a/tests/test_calibration.py
+++ b/tests/test_calibration.py
@@ -166,7 +166,7 @@ def test_not_fitted_calibrator_throws_error(binary_test_scores, binary_mock_esti
                        _IsotonicRegression(), 
                        _InductiveVennAbersCalibrator(), 
                        _CrossVennAbersCalibrator(binary_mock_estimator)]:
-            with pytest.raises(NotFittedError, match=f"This {calibrator.name} calibrator is not fitted yet. Call 'fit' with appropriate arguments before using this calibrator."):
+            with pytest.raises(NotFittedError):
                 calibrator.predict_proba(binary_test_scores)
 
 def test_valid_calibration(binary_calibration_data, binary_test_scores, binary_cal_X, binary_mock_estimator):

From abe667758676f370bd23d47b02a6b799e8cfff82 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sat, 6 Apr 2024 17:26:45 +0200
Subject: [PATCH 40/65] add tests for predict_proba

---
 hiclass/HierarchicalClassifier.py          |   3 +
 hiclass/LocalClassifierPerNode.py          |   4 +-
 hiclass/LocalClassifierPerParentNode.py    |   1 +
 tests/test_LocalClassifierPerLevel.py      |  18 ++
 tests/test_LocalClassifierPerNode.py       | 192 +++++++++++++++++++--
 tests/test_LocalClassifierPerParentNode.py | 183 ++++++++++++++++++--
 tests/test_ProbabilityCombiner.py          |  26 +++
 7 files changed, 388 insertions(+), 39 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index b1a47052..eae04900 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -250,6 +250,9 @@ def calibrate(self, X, y):
             self.y_cal = np.vstack([self.y_, y])
         else:
             self.X_cal = X
+            y = make_leveled(y)
+            y = self._disambiguate(y)
+            y = self._convert_1d_y_to_2d(y)
             self.y_cal = y
             
         self.logger_.info("Calibrating")
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index 67cf7e14..40b4074f 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -179,7 +179,6 @@ def predict(self, X):
 
         # Initialize array that holds predictions
         y = np.empty((X.shape[0], self.max_levels_), dtype=self.dtype_)
-
         # TODO: Add threshold to stop prediction halfway if need be
 
         bfs = nx.bfs_successors(self.hierarchy_, source=self.root_)
@@ -197,6 +196,7 @@ def predict(self, X):
                 probabilities = np.zeros((subset_x.shape[0], len(successors)))
                 for i, successor in enumerate(successors):
                     successor_name = str(successor).split(self.separator_)[-1]
+                    #self
                     self.logger_.info(f"Predicting for node '{successor_name}'")
                     # TODO: use calibrator if using calibration to predict class
                     classifier = self.hierarchy_.nodes[successor]["classifier"]
@@ -213,7 +213,7 @@ def predict(self, X):
                 )
                 prediction = np.array(prediction)
                 y[mask, level] = prediction
-
+                
         y = self._convert_to_1d(y)
 
         self._remove_separator(y)
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index a206c8e7..81e4221f 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -302,6 +302,7 @@ def _get_successors(self, node, calibration=False):
         for row in masked_labels:
             if node == self.root_:
                 y.append(row[0])
+                self.logger_.info(y)
             else:
                 y.append(row[np.where(row == node)[0][0] + 1])
         y = np.array(y)
diff --git a/tests/test_LocalClassifierPerLevel.py b/tests/test_LocalClassifierPerLevel.py
index 064b533d..6286a06d 100644
--- a/tests/test_LocalClassifierPerLevel.py
+++ b/tests/test_LocalClassifierPerLevel.py
@@ -252,3 +252,21 @@ def test_fit_calibrate_predict_proba():
     assert proba[1].shape == (2, 2)
     assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
     assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+
+def test_fit_calibrate_predict_predict_proba_bert():
+    classifier = LocalClassifierPerLevel(
+        local_classifier=LogisticRegression(), 
+        return_all_probabilities=True,
+        calibration_method="ivap",
+        probability_combiner="geometric" 
+    )
+
+    classifier.logger_ = logging.getLogger("HC")
+    classifier.bert = True
+    x = [[0, 1], [2, 3]]
+    y = [["a", "b"], ["c", "d"]]
+    sample_weight = None
+    classifier.fit(x, y, sample_weight)
+    classifier.calibrate(x, y)
+    classifier.predict(x)
+    classifier.predict_proba(x)
diff --git a/tests/test_LocalClassifierPerNode.py b/tests/test_LocalClassifierPerNode.py
index dadd04eb..dbf6b82f 100644
--- a/tests/test_LocalClassifierPerNode.py
+++ b/tests/test_LocalClassifierPerNode.py
@@ -3,12 +3,14 @@
 import networkx as nx
 import numpy as np
 import pytest
-from numpy.testing import assert_array_equal
+from numpy.testing import assert_array_equal, assert_array_almost_equal
 from scipy.sparse import csr_matrix
 from sklearn.exceptions import NotFittedError
 from sklearn.linear_model import LogisticRegression
 from sklearn.utils.estimator_checks import parametrize_with_checks
 from sklearn.utils.validation import check_is_fitted
+from hiclass._calibration.Calibrator import _Calibrator
+from hiclass.HierarchicalClassifier import make_leveled
 
 from hiclass import LocalClassifierPerNode
 from hiclass.BinaryPolicy import ExclusivePolicy
@@ -66,10 +68,12 @@ def test_initialize_object_binary_policy(digraph_with_object_policy):
 
 @pytest.fixture
 def digraph_logistic_regression():
-    digraph = LocalClassifierPerNode(local_classifier=LogisticRegression())
+    digraph = LocalClassifierPerNode(local_classifier=LogisticRegression(), calibration_method="ivap")
     digraph.hierarchy_ = nx.DiGraph([("a", "b"), ("a", "c")])
     digraph.y_ = np.array([["a", "b"], ["a", "c"]])
     digraph.X_ = np.array([[1, 2], [3, 4]])
+    digraph.y_cal = np.array([["a", "b"], ["a", "c"]])
+    digraph.X_cal = np.array([[1, 2], [3, 4]])
     digraph.logger_ = logging.getLogger("LCPN")
     digraph.root_ = "a"
     digraph.separator_ = "::HiClass::Separator::"
@@ -93,6 +97,23 @@ def test_initialize_local_classifiers(digraph_logistic_regression):
                     LogisticRegression,
                 )
 
+def test_initialize_local_calibrators(digraph_logistic_regression):
+    digraph_logistic_regression._initialize_local_classifiers()
+    digraph_logistic_regression._initialize_local_calibrators()
+
+    for node in digraph_logistic_regression.hierarchy_.nodes:
+        if node != digraph_logistic_regression.root_:
+            assert isinstance(
+                digraph_logistic_regression.hierarchy_.nodes[node]["calibrator"],
+                _Calibrator,
+            )
+        else:
+            with pytest.raises(KeyError):
+                isinstance(
+                    digraph_logistic_regression.hierarchy_.nodes[node]["classifier"],
+                    _Calibrator,
+                )
+
 
 def test_fit_digraph(digraph_logistic_regression):
     classifiers = {
@@ -113,6 +134,33 @@ def test_fit_digraph(digraph_logistic_regression):
             pytest.fail(repr(e))
     assert 1
 
+def test_calibrate_digraph(digraph_logistic_regression):
+    classifiers = {
+        "b": {"classifier": LogisticRegression()},
+        "c": {"classifier": LogisticRegression()},
+    }
+    digraph_logistic_regression.n_jobs = 2
+    nx.set_node_attributes(digraph_logistic_regression.hierarchy_, classifiers)
+    digraph_logistic_regression._fit_digraph(local_mode=True)
+
+    calibrators = {
+        "b": {"calibrator": _Calibrator(digraph_logistic_regression.hierarchy_.nodes["b"]["classifier"])},
+        "c": {"calibrator": _Calibrator(digraph_logistic_regression.hierarchy_.nodes["c"]["classifier"])}
+    }
+    nx.set_node_attributes(digraph_logistic_regression.hierarchy_, calibrators)
+    digraph_logistic_regression._calibrate_digraph(local_mode=True)
+    
+    with pytest.raises(KeyError):
+        check_is_fitted(digraph_logistic_regression.hierarchy_.nodes["a"]["calibrator"])
+    for node in ["b", "c"]:
+        try:
+            check_is_fitted(
+                digraph_logistic_regression.hierarchy_.nodes[node]["calibrator"]
+            )
+        except NotFittedError as e:
+            pytest.fail(repr(e))
+    assert 1
+
 
 def test_fit_digraph_joblib_multiprocessing(digraph_logistic_regression):
     classifiers = {
@@ -133,6 +181,33 @@ def test_fit_digraph_joblib_multiprocessing(digraph_logistic_regression):
             pytest.fail(repr(e))
     assert 1
 
+def test_calibrate_digraph_joblib_multiprocessing(digraph_logistic_regression):
+    classifiers = {
+        "b": {"classifier": LogisticRegression()},
+        "c": {"classifier": LogisticRegression()},
+    }
+    digraph_logistic_regression.n_jobs = 2
+    nx.set_node_attributes(digraph_logistic_regression.hierarchy_, classifiers)
+    digraph_logistic_regression._fit_digraph(local_mode=True, use_joblib=True)
+
+    calibrators = {
+        "b": {"calibrator": _Calibrator(digraph_logistic_regression.hierarchy_.nodes["b"]["classifier"])},
+        "c": {"calibrator": _Calibrator(digraph_logistic_regression.hierarchy_.nodes["c"]["classifier"])}
+    }
+    nx.set_node_attributes(digraph_logistic_regression.hierarchy_, calibrators)
+    digraph_logistic_regression._calibrate_digraph(local_mode=True, use_joblib=True)
+
+    with pytest.raises(KeyError):
+        check_is_fitted(digraph_logistic_regression.hierarchy_.nodes["a"]["calibrator"])
+    for node in ["b", "c"]:
+        try:
+            check_is_fitted(
+                digraph_logistic_regression.hierarchy_.nodes[node]["calibrator"]
+            )
+        except NotFittedError as e:
+            pytest.fail(repr(e))
+    assert 1
+
 
 def test_clean_up(digraph_logistic_regression):
     digraph_logistic_regression._clean_up()
@@ -142,35 +217,64 @@ def test_clean_up(digraph_logistic_regression):
         assert digraph_logistic_regression.y_ is None
     with pytest.raises(AttributeError):
         assert digraph_logistic_regression.binary_policy_ is None
+    with pytest.raises(AttributeError):
+        assert digraph_logistic_regression.cal_binary_policy_ is None
 
 
 @pytest.fixture
 def fitted_logistic_regression():
-    digraph = LocalClassifierPerNode(local_classifier=LogisticRegression())
+    digraph = LocalClassifierPerNode(
+        local_classifier=LogisticRegression(),
+        return_all_probabilities=True,
+        calibration_method="ivap",
+        probability_combiner="geometric")
+    
+    digraph.separator_ = "::HiClass::Separator::"
+    #digraph.hierarchy_ = nx.DiGraph(
+    #    [("r", "1"), ("r", "2"), ("1", "1.1"), ("1", "1.2"), ("2", "2.1"), ("2", "2.2")]
+    #)
+
     digraph.hierarchy_ = nx.DiGraph(
-        [("r", "1"), ("r", "2"), ("1", "1.1"), ("1", "1.2"), ("2", "2.1"), ("2", "2.2")]
+        [("r", "1"), 
+         ("r", "2"), 
+         ("1", "1"+digraph.separator_+"1.1"),
+         ("1", "1"+digraph.separator_+"1.2"), 
+         ("2", "2"+digraph.separator_+"2.1"), 
+         ("2", "2"+digraph.separator_+"2.2")]
     )
-    digraph.y_ = np.array([["1", "1.1"], ["1", "1.2"], ["2", "2.1"], ["2", "2.2"]])
+    digraph.y_ = np.array([
+        ["1", "1.1"], 
+        ["1", "1.2"], 
+        ["2", "2.1"], 
+        ["2", "2.2"]])
+    
     digraph.X_ = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
     digraph.logger_ = logging.getLogger("LCPN")
     digraph.max_levels_ = 2
-    digraph.dtype_ = "<U3"
+    digraph.dtype_ = "<U30"
     digraph.root_ = "r"
-    digraph.separator_ = "::HiClass::Separator::"
-    classifiers = {
-        "1": {"classifier": LogisticRegression()},
-        "1.1": {"classifier": LogisticRegression()},
-        "1.2": {"classifier": LogisticRegression()},
-        "2": {"classifier": LogisticRegression()},
-        "2.1": {"classifier": LogisticRegression()},
-        "2.2": {"classifier": LogisticRegression()},
-    }
+
+    # for predict_proba
+    tmp_labels = digraph._disambiguate(make_leveled(digraph.y_))
+    #digraph.y_ = tmp_labels
+    digraph.max_level_dimensions_ = np.array([len(np.unique(tmp_labels[:, level])) for level in range(tmp_labels.shape[1])])
+    digraph.global_classes_ = [np.unique(tmp_labels[:, level]).astype("str") for level in range(tmp_labels.shape[1])]
+    digraph.global_class_to_index_mapping_ = [{digraph.global_classes_[level][index]: index for index in range(len(digraph.global_classes_[level]))} for level in range(tmp_labels.shape[1])]
+
+    classes_ = [digraph.global_classes_[0]]
+    for level in range(1, digraph.max_levels_):
+        classes_.append(np.sort(np.unique([label.split(digraph.separator_)[level] for label in digraph.global_classes_[level]])))
+    digraph.classes_ = classes_
+    digraph.class_to_index_mapping_ = [{local_labels[index]: index for index in range(len(local_labels))} for local_labels in classes_]
+
+    classifiers = {key: {"classifier": LogisticRegression()} for key in np.hstack([digraph.global_classes_[0], digraph.global_classes_[1]])}
+
     classifiers["1"]["classifier"].fit(digraph.X_, [1, 1, 0, 0])
-    classifiers["1.1"]["classifier"].fit(digraph.X_, [1, 0, 0, 0])
-    classifiers["1.2"]["classifier"].fit(digraph.X_, [0, 1, 0, 0])
+    classifiers["1"+digraph.separator_+"1.1"]["classifier"].fit(digraph.X_, [1, 0, 0, 0])
+    classifiers["1"+digraph.separator_+"1.2"]["classifier"].fit(digraph.X_, [0, 1, 0, 0])
     classifiers["2"]["classifier"].fit(digraph.X_, [0, 0, 1, 1])
-    classifiers["2.1"]["classifier"].fit(digraph.X_, [0, 0, 1, 0])
-    classifiers["2.2"]["classifier"].fit(digraph.X_, [0, 0, 0, 1])
+    classifiers["2"+digraph.separator_+"2.1"]["classifier"].fit(digraph.X_, [0, 0, 1, 0])
+    classifiers["2"+digraph.separator_+"2.2"]["classifier"].fit(digraph.X_, [0, 0, 0, 1])
     nx.set_node_attributes(digraph.hierarchy_, classifiers)
     return digraph
 
@@ -188,6 +292,22 @@ def test_predict_sparse(fitted_logistic_regression):
     )
     assert_array_equal(ground_truth, prediction)
 
+def test_predict_proba(fitted_logistic_regression):
+    proba = fitted_logistic_regression.predict_proba([[7, 8], [5, 6], [3, 4], [1, 2]])
+    assert len(proba) == 2
+    assert proba[0].shape == (4, 2)
+    assert proba[1].shape == (4, 4)
+    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
+    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+
+def test_predict_proba_sparse(fitted_logistic_regression):
+    proba = fitted_logistic_regression.predict_proba(csr_matrix([[7, 8], [5, 6], [3, 4], [1, 2]]))
+    assert len(proba) == 2
+    assert proba[0].shape == (4, 2)
+    assert proba[1].shape == (4, 4)
+    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
+    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+
 
 def test_fit_predict():
     lcpn = LocalClassifierPerNode(local_classifier=LogisticRegression())
@@ -196,3 +316,37 @@ def test_fit_predict():
     lcpn.fit(x, y)
     predictions = lcpn.predict(x)
     assert_array_equal(y, predictions)
+
+def test_fit_calibrate_predict_proba():
+    lcpn = LocalClassifierPerNode(local_classifier=LogisticRegression(), calibration_method="ivap", return_all_probabilities=True)
+    x = np.array([[1, 2], [3, 4]])
+    y = np.array([["a", "b"], ["b", "c"]])
+    lcpn.fit(x, y)
+    lcpn.calibrate(x, y)
+    predictions = lcpn.predict(x)
+    proba = lcpn.predict_proba(x)
+    assert_array_equal(y, predictions)
+
+    assert len(proba) == 2
+    assert proba[0].shape == (2, 2)
+    assert proba[1].shape == (2, 2)
+    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
+    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+
+def test_fit_calibrate_predict_predict_proba_bert():
+    classifier = LocalClassifierPerNode(
+        local_classifier=LogisticRegression(), 
+        return_all_probabilities=True,
+        calibration_method="ivap",
+        probability_combiner="geometric" 
+    )
+
+    classifier.logger_ = logging.getLogger("HC")
+    classifier.bert = True
+    x = [[0, 1], [2, 3]]
+    y = [["a", "b"], ["c", "d"]]
+    sample_weight = None
+    classifier.fit(x, y, sample_weight)
+    classifier.calibrate(x, y)
+    classifier.predict(x)
+    classifier.predict_proba(x)
\ No newline at end of file
diff --git a/tests/test_LocalClassifierPerParentNode.py b/tests/test_LocalClassifierPerParentNode.py
index 922a03a3..91c33238 100644
--- a/tests/test_LocalClassifierPerParentNode.py
+++ b/tests/test_LocalClassifierPerParentNode.py
@@ -4,7 +4,7 @@
 import networkx as nx
 import numpy as np
 import pytest
-from numpy.testing import assert_array_equal
+from numpy.testing import assert_array_equal, assert_array_almost_equal
 from scipy.sparse import csr_matrix
 from sklearn.exceptions import NotFittedError
 from sklearn.linear_model import LogisticRegression
@@ -12,6 +12,8 @@
 from sklearn.utils.validation import check_is_fitted
 
 from hiclass import LocalClassifierPerParentNode
+from hiclass._calibration.Calibrator import _Calibrator
+from hiclass.HierarchicalClassifier import make_leveled
 
 
 @parametrize_with_checks([LocalClassifierPerParentNode()])
@@ -21,10 +23,12 @@ def test_sklearn_compatible_estimator(estimator, check):
 
 @pytest.fixture
 def digraph_logistic_regression():
-    digraph = LocalClassifierPerParentNode(local_classifier=LogisticRegression())
-    digraph.hierarchy_ = nx.DiGraph([("a", "b"), ("a", "c")])
-    digraph.y_ = np.array([["a", "b"], ["a", "c"]])
-    digraph.X_ = np.array([[1, 2], [3, 4]])
+    digraph = LocalClassifierPerParentNode(local_classifier=LogisticRegression(), calibration_method="ivap")
+    digraph.hierarchy_ = nx.DiGraph([("a", "b"), ("a", "c"), ("b", "d"), ("c", "f")])
+    digraph.y_ = np.array([["a", "b"], ["a", "c"], ["b", "d"], ["c", "f"]])
+    digraph.X_ = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
+    digraph.y_cal = np.array([["a", "b"], ["a", "c"], ["b", "d"], ["c", "f"]])
+    digraph.X_cal = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
     digraph.logger_ = logging.getLogger("LCPPN")
     digraph.root_ = "a"
     digraph.separator_ = "::HiClass::Separator::"
@@ -35,7 +39,7 @@ def digraph_logistic_regression():
 def test_initialize_local_classifiers(digraph_logistic_regression):
     digraph_logistic_regression._initialize_local_classifiers()
     for node in digraph_logistic_regression.hierarchy_.nodes:
-        if node == digraph_logistic_regression.root_:
+        if node in ["a", "b", "c"]:
             assert isinstance(
                 digraph_logistic_regression.hierarchy_.nodes[node]["classifier"],
                 LogisticRegression,
@@ -47,10 +51,29 @@ def test_initialize_local_classifiers(digraph_logistic_regression):
                     LogisticRegression,
                 )
 
+def test_initialize_local_calibrators(digraph_logistic_regression):
+    digraph_logistic_regression._initialize_local_classifiers()
+    digraph_logistic_regression._initialize_local_calibrators()
+
+    for node in digraph_logistic_regression.hierarchy_.nodes:
+        if node in ["a", "b", "c"]:
+            assert isinstance(
+                digraph_logistic_regression.hierarchy_.nodes[node]["calibrator"],
+                _Calibrator,
+            )
+        else:
+            with pytest.raises(KeyError):
+                isinstance(
+                    digraph_logistic_regression.hierarchy_.nodes[node]["classifier"],
+                    _Calibrator,
+                )
+
 
 def test_fit_digraph(digraph_logistic_regression):
     classifiers = {
         "a": {"classifier": LogisticRegression()},
+        "b": {"classifier": LogisticRegression()},
+        "c": {"classifier": LogisticRegression()},
     }
     digraph_logistic_regression.n_jobs = 2
     nx.set_node_attributes(digraph_logistic_regression.hierarchy_, classifiers)
@@ -59,17 +82,46 @@ def test_fit_digraph(digraph_logistic_regression):
         check_is_fitted(digraph_logistic_regression.hierarchy_.nodes["a"]["classifier"])
     except NotFittedError as e:
         pytest.fail(repr(e))
-    for node in ["b", "c"]:
+    for node in ["d", "f"]:
         with pytest.raises(KeyError):
             check_is_fitted(
                 digraph_logistic_regression.hierarchy_.nodes[node]["classifier"]
             )
     assert 1
 
+def test_calibrate_digraph(digraph_logistic_regression):
+    classifiers = {
+        "a": {"classifier": LogisticRegression()},
+        "b": {"classifier": LogisticRegression()},
+        "c": {"classifier": LogisticRegression()},
+    }
+    digraph_logistic_regression.n_jobs = 2
+    nx.set_node_attributes(digraph_logistic_regression.hierarchy_, classifiers)
+    digraph_logistic_regression._fit_digraph(local_mode=True)
+
+    calibrators = {
+        "a": {"calibrator": _Calibrator(digraph_logistic_regression.hierarchy_.nodes["a"]["classifier"])},
+    }
+    nx.set_node_attributes(digraph_logistic_regression.hierarchy_, calibrators)
+    digraph_logistic_regression._calibrate_digraph(local_mode=True)
+
+    try:
+        check_is_fitted(digraph_logistic_regression.hierarchy_.nodes["a"]["calibrator"])
+    except NotFittedError as e:
+        pytest.fail(repr(e))
+    for node in ["b", "c", "d", "f"]:
+        with pytest.raises((TypeError, KeyError)):
+            check_is_fitted(
+                digraph_logistic_regression.hierarchy_.nodes[node]["calibrator"]
+            )
+    assert 1
+
 
 def test_fit_digraph_joblib_multiprocessing(digraph_logistic_regression):
     classifiers = {
         "a": {"classifier": LogisticRegression()},
+        "b": {"classifier": LogisticRegression()},
+        "c": {"classifier": LogisticRegression()},
     }
     digraph_logistic_regression.n_jobs = 2
     nx.set_node_attributes(digraph_logistic_regression.hierarchy_, classifiers)
@@ -78,13 +130,39 @@ def test_fit_digraph_joblib_multiprocessing(digraph_logistic_regression):
         check_is_fitted(digraph_logistic_regression.hierarchy_.nodes["a"]["classifier"])
     except NotFittedError as e:
         pytest.fail(repr(e))
-    for node in ["b", "c"]:
+    for node in ["d", "f"]:
         with pytest.raises(KeyError):
             check_is_fitted(
                 digraph_logistic_regression.hierarchy_.nodes[node]["classifier"]
             )
     assert 1
 
+def test_calibrate_digraph_joblib_multiprocessing(digraph_logistic_regression):
+    classifiers = {
+        "a": {"classifier": LogisticRegression()},
+        "b": {"classifier": LogisticRegression()},
+        "c": {"classifier": LogisticRegression()},
+    }
+    digraph_logistic_regression.n_jobs = 2
+    nx.set_node_attributes(digraph_logistic_regression.hierarchy_, classifiers)
+    digraph_logistic_regression._fit_digraph(local_mode=True, use_joblib=True)
+
+    calibrators = {
+        "a": {"calibrator": _Calibrator(digraph_logistic_regression.hierarchy_.nodes["a"]["classifier"])},
+    }
+    nx.set_node_attributes(digraph_logistic_regression.hierarchy_, calibrators)
+    digraph_logistic_regression._calibrate_digraph(local_mode=True, use_joblib=True)
+
+    try:
+        check_is_fitted(digraph_logistic_regression.hierarchy_.nodes["a"]["calibrator"])
+    except NotFittedError as e:
+        pytest.fail(repr(e))
+    for node in ["b", "c", "d", "f"]:
+        with pytest.raises((TypeError, KeyError)):
+            check_is_fitted(
+                digraph_logistic_regression.hierarchy_.nodes[node]["calibrator"]
+            )
+    assert 1
 
 @pytest.fixture
 def digraph_2d():
@@ -133,32 +211,52 @@ def test_get_successors(x_and_y_arrays):
 
 @pytest.fixture
 def fitted_logistic_regression():
-    digraph = LocalClassifierPerParentNode(local_classifier=LogisticRegression())
+    digraph = LocalClassifierPerParentNode(
+        local_classifier=LogisticRegression(), 
+        calibration_method="ivap",
+        probability_combiner="geometric",
+        return_all_probabilities=True)
+    
+    digraph.separator_ = "::HiClass::Separator::"
     digraph.hierarchy_ = nx.DiGraph(
         [
             ("r", "1"),
             ("r", "2"),
-            ("1", "1.1"),
-            ("1", "1.2"),
-            ("2", "2.1"),
-            ("2", "2.2"),
+            ("1", "1"+digraph.separator_+"1.1"),
+            ("1", "1"+digraph.separator_+"1.2"),
+            ("2", "2"+digraph.separator_+"2.1"),
+            ("2", "2"+digraph.separator_+"2.2"),
         ]
     )
-    digraph.y_ = np.array([["1", "1.1"], ["1", "1.2"], ["2", "2.1"], ["2", "2.2"]])
+    digraph.y_ = np.array([
+        ["1", "1.1"], ["1", "1.2"], ["2", "2.1"], ["2", "2.2"]])
     digraph.X_ = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
     digraph.logger_ = logging.getLogger("LCPPN")
     digraph.max_levels_ = 2
-    digraph.dtype_ = "<U3"
+    digraph.max_level_dimensions_ = np.array([2, 4])
+    digraph.dtype_ = "<U30"
     digraph.root_ = "r"
-    digraph.separator_ = "::HiClass::Separator::"
+
+    # for predict_proba
+    tmp_labels = digraph._disambiguate(make_leveled(digraph.y_))
+    #digraph.y_ = tmp_labels
+    digraph.global_classes_ = [np.unique(tmp_labels[:, level]).astype("str") for level in range(tmp_labels.shape[1])]
+    digraph.global_class_to_index_mapping_ = [{digraph.global_classes_[level][index]: index for index in range(len(digraph.global_classes_[level]))} for level in range(tmp_labels.shape[1])]
+
+    classes_ = [digraph.global_classes_[0]]
+    for level in range(1, digraph.max_levels_):
+        classes_.append(np.sort(np.unique([label.split(digraph.separator_)[level] for label in digraph.global_classes_[level]])))
+    digraph.classes_ = classes_
+    digraph.class_to_index_mapping_ = [{local_labels[index]: index for index in range(len(local_labels))} for local_labels in classes_]
+
     classifiers = {
         "r": {"classifier": LogisticRegression()},
         "1": {"classifier": LogisticRegression()},
         "2": {"classifier": LogisticRegression()},
     }
     classifiers["r"]["classifier"].fit(digraph.X_, ["1", "1", "2", "2"])
-    classifiers["1"]["classifier"].fit(digraph.X_[:2], ["1.1", "1.2"])
-    classifiers["2"]["classifier"].fit(digraph.X_[2:], ["2.1", "2.2"])
+    classifiers["1"]["classifier"].fit(digraph.X_[:2], ["1"+digraph.separator_+"1.1", "1"+digraph.separator_+"1.2"])
+    classifiers["2"]["classifier"].fit(digraph.X_[2:], ["2"+digraph.separator_+"2.1", "2"+digraph.separator_+"2.2"])
     nx.set_node_attributes(digraph.hierarchy_, classifiers)
     return digraph
 
@@ -176,6 +274,22 @@ def test_predict_sparse(fitted_logistic_regression):
     )
     assert_array_equal(ground_truth, prediction)
 
+def test_predict_proba(fitted_logistic_regression):
+    proba = fitted_logistic_regression.predict_proba([[7, 8], [5, 6], [3, 4], [1, 2]])
+    assert len(proba) == 2
+    assert proba[0].shape == (4, 2)
+    assert proba[1].shape == (4, 4)
+    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
+    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+
+def test_predict_proba_sparse(fitted_logistic_regression):
+    proba = fitted_logistic_regression.predict_proba(csr_matrix([[7, 8], [5, 6], [3, 4], [1, 2]]))
+    assert len(proba) == 2
+    assert proba[0].shape == (4, 2)
+    assert proba[1].shape == (4, 4)
+    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
+    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+
 
 def test_fit_predict():
     lcppn = LocalClassifierPerParentNode(local_classifier=LogisticRegression())
@@ -184,3 +298,36 @@ def test_fit_predict():
     lcppn.fit(x, y)
     predictions = lcppn.predict(x)
     assert_array_equal(y, predictions)
+
+def test_fit_calibrate_predict_proba():
+    lcppn = LocalClassifierPerParentNode(local_classifier=LogisticRegression(), calibration_method="ivap", return_all_probabilities=True)
+    x = np.array([[1, 2], [3, 4]])
+    y = np.array([["a", "b"], ["b", "c"]])
+    lcppn.fit(x, y)
+    predictions = lcppn.predict(x)
+    proba = lcppn.predict_proba(x)
+    assert_array_equal(y, predictions)
+
+    assert len(proba) == 2
+    assert proba[0].shape == (2, 2)
+    assert proba[1].shape == (2, 2)
+    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
+    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+
+def test_fit_calibrate_predict_predict_proba_bert():
+    classifier = LocalClassifierPerParentNode(
+        local_classifier=LogisticRegression(), 
+        return_all_probabilities=True,
+        calibration_method="ivap",
+        probability_combiner="geometric" 
+    )
+
+    classifier.logger_ = logging.getLogger("HC")
+    classifier.bert = True
+    x = [[0, 1], [2, 3]]
+    y = [["a", "b"], ["c", "d"]]
+    sample_weight = None
+    classifier.fit(x, y, sample_weight)
+    classifier.calibrate(x, y)
+    classifier.predict(x)
+    classifier.predict_proba(x)
\ No newline at end of file
diff --git a/tests/test_ProbabilityCombiner.py b/tests/test_ProbabilityCombiner.py
index f34a26e1..db4c81b8 100644
--- a/tests/test_ProbabilityCombiner.py
+++ b/tests/test_ProbabilityCombiner.py
@@ -162,3 +162,29 @@ def test_geometric_mean_combiner(one_sample_probs_with_hierarchy):
     
     assert math.isclose(combined_probs[2][0][-1], 0.0430, abs_tol=1e-4)
     assert math.isclose(combined_probs[2][0][-1], gmean([probs[0][0][-1], probs[1][0][-1], probs[2][0][-1]]), abs_tol=1e-4)
+
+def test_mean_combiner_with_normalization(one_sample_probs_with_hierarchy):
+    hierarchy, probs, global_classes, classes_, class_to_index_mapping_ = one_sample_probs_with_hierarchy
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.separator_ = "::HiClass::Separator::"
+    classifier.global_classes_ = global_classes
+    classifier.classes_ = classes_
+
+    classifier.max_levels_ = len(global_classes)
+    classifier.class_to_index_mapping_ = class_to_index_mapping_
+    classifier.hierarchy_ = hierarchy
+
+    combiners = [MultiplyCombiner(classifier), ArithmeticMeanCombiner(classifier), GeometricMeanCombiner(classifier)]
+
+    for combiner in combiners:
+        combined_probs = combiner.combine(probs)
+
+        assert len(combined_probs) == 3
+        assert combined_probs[0].shape == (1, 3)
+        assert combined_probs[1].shape == (1, 9)
+        assert combined_probs[2].shape == (1, 18)
+        assert_array_almost_equal(np.sum(combined_probs[0], axis=1), np.ones(len(combined_probs[0])), decimal=10)
+        assert_array_almost_equal(np.sum(combined_probs[1], axis=1), np.ones(len(combined_probs[1])), decimal=10)
+        assert_array_almost_equal(np.sum(combined_probs[2], axis=1), np.ones(len(combined_probs[2])), decimal=10)

From a913ff51ec5e86eee6bf57af0855bb28de0a41ae Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sun, 7 Apr 2024 16:17:49 +0200
Subject: [PATCH 41/65] refactor InductiveVennAbersCalibrator

---
 hiclass/_calibration/VennAbersCalibrator.py | 203 +++++++++-----------
 tests/test_calibration.py                   |   4 +-
 2 files changed, 92 insertions(+), 115 deletions(-)

diff --git a/hiclass/_calibration/VennAbersCalibrator.py b/hiclass/_calibration/VennAbersCalibrator.py
index 1268c232..9548e591 100644
--- a/hiclass/_calibration/VennAbersCalibrator.py
+++ b/hiclass/_calibration/VennAbersCalibrator.py
@@ -6,12 +6,11 @@
 from collections import defaultdict
 from sklearn.utils.validation import check_is_fitted
 
-
 class _InductiveVennAbersCalibrator(_BinaryCalibrator):
     name = "InductiveVennAbersCalibrator"
 
-    def __init__(self):
-        self._is_fitted = False
+    def __init__(self) -> None:
+        super().__init__()
 
     def fit(self, y, scores, X=None):
         positive_label = 1
@@ -20,163 +19,141 @@ def fit(self, y, scores, X=None):
 
         y = np.where(y == positive_label, 1, 0)
         y = y.reshape(-1)  # make sure it's a 1D array
-        # sort all scores s1, ..., sk in increasing order
 
+        # sort all scores s1, ..., sk in increasing order
         order_idx = np.lexsort([y, scores])
         ordered_calibration_scores, ordered_calibration_labels = scores[order_idx], y[order_idx]
-        # remove duplicates
         unique_elements, unique_idx, unique_element_counts = np.unique(ordered_calibration_scores, return_index=True, return_counts=True)
         ordered_unique_calibration_scores, _ = ordered_calibration_scores[unique_idx], ordered_calibration_labels[unique_idx]
 
         self.k_distinct = len(unique_idx)
 
-        def compute_csd(un_el, un_el_counts, ocs, ocl, oucs):
-
-            # Count the frequencies of each s'j
-            w = dict(zip(un_el, un_el_counts))
-
-            y = np.zeros(self.k_distinct)
-            csd = np.zeros((self.k_distinct + 1, 2))
-
-            for j in range(self.k_distinct):
-                s_j = oucs[j]
-                matching_idx = np.where(ocs == s_j)
-                matching_labels = ocl[matching_idx]
-                y[j] = np.sum(matching_labels) / w[un_el[j]]
-
-            csd[1:, 0] = np.cumsum(un_el_counts)
-            csd[1:, 1] = np.cumsum(y * un_el_counts)
+        ### compute csd
+        # Count the frequencies of each s'j
+        w = dict(zip(unique_elements, unique_element_counts))
+        y = np.zeros(self.k_distinct)
+        csd_1 = np.zeros((self.k_distinct + 1, 2))
+
+        for j in range(self.k_distinct):
+            s_j = ordered_unique_calibration_scores[j]
+            matching_idx = np.where(ordered_calibration_scores == s_j)
+            matching_labels = ordered_calibration_labels[matching_idx]
+            y[j] = np.sum(matching_labels) / w[unique_elements[j]]
+        
+        csd_1[1:, 0] = np.cumsum(unique_element_counts)
+        csd_1[1:, 1] = np.cumsum(y * unique_element_counts)
 
-            return list(csd)
-
-        def slope(top, next_to_top):
-            return (next_to_top[1] - top[1]) / (next_to_top[0] - top[0])
-
-        def at_or_above(p, cur_slope, top, next_to_top):
-            intersection_point = (p[0], top[1] + cur_slope * (p[0] - top[0]))
-            return p[1] >= intersection_point[1]
+        csd_0 = csd_1.copy()
+        csd_0 = np.append(csd_0, [np.array([csd_0[-1][0] + 1, csd_0[-1][1] + 0])], axis=0)
+        csd_1 = np.insert(csd_1, 0, [np.array([(-1, -1)])], axis=0)
 
-        def non_left_angle_turn(next_to_top, top, p_i):
-            next_to_top = np.array(next_to_top)
-            top = np.array(top)
-            p_i = np.array(p_i)
-            res = np.cross((top - next_to_top), (p_i - top))
-            return res <= 0
+        f1_stack = self._initialize_f1_corners(csd_1)
+        f0_stack = self._initialize_f0_corners(csd_0)
 
-        def non_right_angle_turn(next_to_top, top, p_i):
-            next_to_top = np.array(next_to_top)
-            top = np.array(top)
-            p_i = np.array(p_i)
-            res = np.cross((top - next_to_top), (p_i - top))
-            return res >= 0
+        self._F1 = self._compute_f1(f1_stack, csd_1)
+        self._F0 = self._compute_f0(f0_stack, csd_0)
+        self._unique_elements = unique_elements
+        self._is_fitted = True
 
-        def initialize_f1_corners(csd):
+        return self
+    
+    def _non_left_angle_turn(self, next_to_top, top, p_i):
+        res = np.cross((top - next_to_top), (p_i - top))
+        return res <= 0
+    
+    def _non_right_angle_turn(self, next_to_top, top, p_i):
+        res = np.cross((top - next_to_top), (p_i - top))
+        return res >= 0
+    
+    def _initialize_f1_corners(self, csd):
             stack = []
             # append P_{-1} and P_0
             stack.append(csd[0])
             stack.append(csd[1])
 
             for i in range(2, len(csd)):
-                while len(stack) > 1 and non_left_angle_turn(next_to_top=stack[-2], top=stack[-1], p_i=csd[i]):
+                while len(stack) > 1 and self._non_left_angle_turn(next_to_top=stack[-2], top=stack[-1], p_i=csd[i]):
                     stack.pop()
                 stack.append(csd[i])
 
             return stack
-
-        def initialize_f0_corners(csd):
+    
+    def _initialize_f0_corners(self, csd):
             stack = []
             # append p_{k'+1}, p_{k'}
             stack.append(csd[-1])
             stack.append(csd[-2])
 
             for i in range(len(csd) - 3, -1, -1):
-                while len(stack) > 1 and non_right_angle_turn(next_to_top=stack[-2], top=stack[-1], p_i=csd[i]):
+                while len(stack) > 1 and self._non_right_angle_turn(next_to_top=stack[-2], top=stack[-1], p_i=csd[i]):
                     stack.pop()
                 stack.append(csd[i])
             return stack
 
-        point_addition = lambda p1, p2: tuple((p1[0] + p2[0], p1[1] + p2[1]))
-        point_subtraction = lambda p1, p2: tuple((p1[0] - p2[0], p1[1] - p2[1]))
+    def _slope(self, top, next_to_top):
+        return (next_to_top[1] - top[1]) / (next_to_top[0] - top[0])
 
-        def compute_f1(prev_stack, csd):
-            F1 = np.zeros(self.k_distinct + 1)
-            stack = []
-            while prev_stack:
-                stack.append(prev_stack.pop())
+    def _at_or_above(self, p, cur_slope, top):
+        intersection_point = (p[0], top[1] + cur_slope * (p[0] - top[0]))
+        return p[1] >= intersection_point[1]
 
-            for i in range(2, self.k_distinct + 2):
-                F1[i - 1] = slope(top=stack[-1], next_to_top=stack[-2])
-                # p_{i-1}
-                csd[i - 1] = point_subtraction(point_addition(csd[i - 2], csd[i]), csd[i - 1])
-                p_temp = csd[i - 1]
+    def _compute_f1(self, prev_stack, csd):
+        F1 = np.zeros(self.k_distinct + 1)
+        stack = []
+        while prev_stack:
+            stack.append(prev_stack.pop())
 
-                if at_or_above(p_temp, F1[i - 1], top=stack[-1], next_to_top=stack[-2]):
-                    continue
+        for i in range(2, self.k_distinct + 2):
+            F1[i - 1] = self._slope(top=stack[-1], next_to_top=stack[-2])
+            # p_{i-1}
+            csd[i - 1] = (csd[i - 2] + csd[i]) - csd[i - 1]
+            p_temp = csd[i - 1]
 
-                stack.pop()
-                while len(stack) > 1 and non_left_angle_turn(p_temp, stack[-1], stack[-2]):
-                    stack.pop()
-                stack.append(p_temp)
-            return F1
-
-        def compute_f0(prev_stack, csd):
-            F0 = np.zeros(self.k_distinct + 1)
-            stack = []
-            while prev_stack:
-                stack.append(prev_stack.pop())
-
-            for i in range(self.k_distinct, 0, -1):
-                F0[i] = slope(top=stack[-1], next_to_top=stack[-2])
-                csd[i] = point_subtraction(point_addition(csd[i - 1], csd[i + 1]), csd[i])
+            if self._at_or_above(p_temp, F1[i - 1], top=stack[-1]):
+                continue
 
-                if at_or_above(csd[i], F0[i], top=stack[-1], next_to_top=stack[-2]):
-                    continue
+            stack.pop()
+            while len(stack) > 1 and self._non_left_angle_turn(p_temp, stack[-1], stack[-2]):
                 stack.pop()
-                while len(stack) > 1 and non_right_angle_turn(csd[i], stack[-1], stack[-2]):
-                    stack.pop()
-                stack.append(csd[i])
-            return F0
-
-        csd_1 = compute_csd(
-            unique_elements,
-            unique_element_counts,
-            ordered_calibration_scores,
-            ordered_calibration_labels,
-            ordered_unique_calibration_scores
-        )
-        csd_0 = csd_1.copy()
-        csd_0.append((csd_0[-1][0] + 1, csd_0[-1][1] + 0))
-        csd_1.insert(0, (-1, -1))
-
-        f1_stack = initialize_f1_corners(csd_1)
-        f0_stack = initialize_f0_corners(csd_0)
-
-        self.F1 = compute_f1(f1_stack, csd_1)
-        self.F0 = compute_f0(f0_stack, csd_0)
-        self.unique_elements = unique_elements
-        self._is_fitted = True
-
-        return self
-
+            stack.append(p_temp)
+        return F1
+
+    def _compute_f0(self, prev_stack, csd):
+        F0 = np.zeros(self.k_distinct + 1)
+        stack = []
+        while prev_stack:
+            stack.append(prev_stack.pop())
+
+        for i in range(self.k_distinct, 0, -1):
+            F0[i] = self._slope(top=stack[-1], next_to_top=stack[-2])
+            csd[i] = (csd[i - 1] + csd[i + 1]) - csd[i]
+
+            if self._at_or_above(csd[i], F0[i], top=stack[-1]):
+                continue
+            stack.pop()
+            while len(stack) > 1 and self._non_right_angle_turn(csd[i], stack[-1], stack[-2]):
+                stack.pop()
+            stack.append(csd[i])
+        return F0
+    
     def predict_proba(self, scores, X=None):
         check_is_fitted(self)
-        lower = np.searchsorted(self.unique_elements, scores, side="left")
-        upper = np.searchsorted(self.unique_elements[:-1], scores, side="right") + 1
+        lower = np.searchsorted(self._unique_elements, scores, side="left")
+        upper = np.searchsorted(self._unique_elements[:-1], scores, side="right") + 1
 
-        p0 = self.F0[lower]
-        p1 = self.F1[upper]
+        p0 = self._F0[lower]
+        p1 = self._F1[upper]
 
         return p1 / (1 - p0 + p1)
 
     def predict_intervall(self, scores):
-        lower = np.searchsorted(self.unique_elements, scores, side="left")
-        upper = np.searchsorted(self.unique_elements[:-1], scores, side="right") + 1
-        p0 = self.F0[lower]
-        p1 = self.F1[upper]
+        lower = np.searchsorted(self._unique_elements, scores, side="left")
+        upper = np.searchsorted(self._unique_elements[:-1], scores, side="right") + 1
+        p0 = self._F0[lower]
+        p1 = self._F1[upper]
 
         return np.array(list(zip(p0, p1)))
 
-
 class _CrossVennAbersCalibrator(_BinaryCalibrator):
     name = "CrossVennAbersCalibrator"
 
@@ -324,4 +301,4 @@ def predict_proba(self, scores):
             p1 = res[:, :, 1]
 
             p1_gm = gmean(p1)
-            return p1_gm / (gmean(1 - p0) + p1_gm)
+            return p1_gm / (gmean(1 - p0) + p1_gm)
\ No newline at end of file
diff --git a/tests/test_calibration.py b/tests/test_calibration.py
index 8283acd7..b738d83d 100644
--- a/tests/test_calibration.py
+++ b/tests/test_calibration.py
@@ -123,8 +123,8 @@ def test_inductive_venn_abers_calibrator(binary_calibration_data, binary_test_sc
     intervalls = calibrator.predict_intervall(test_scores[:, 1])
     proba = calibrator.predict_proba(test_scores[:, 1])
 
-    assert_array_almost_equal(calibrator.F1, np.array([0, 0.33333333, 0.375, 0.375, 0.4, 0.5, 0.5, 0.66666666, 0.66666666, 1.0, 1.0]))
-    assert_array_almost_equal(calibrator.F0, np.array([0, 0, 0.2, 0.2, 0.2, 0.25, 0.25, 0.33333333, 0.33333333, 0.5, 0.66666666]))
+    assert_array_almost_equal(calibrator._F1, np.array([0, 0.33333333, 0.375, 0.375, 0.4, 0.5, 0.5, 0.66666666, 0.66666666, 1.0, 1.0]))
+    assert_array_almost_equal(calibrator._F0, np.array([0, 0, 0.2, 0.2, 0.2, 0.25, 0.25, 0.33333333, 0.33333333, 0.5, 0.66666666]))
     assert_array_almost_equal(intervalls, np.array([[0.66666666, 1.0], [0.25, 0.66666666], [0, 0.33333333], [0.66666666, 1.0], [0, 0.33333333], [0.25, 0.5], [0,0.33333333]]))
     assert_array_almost_equal(proba, np.array([0.74999999, 0.47058823, 0.24999999, 0.74999999, 0.24999999, 0.4, 0.24999999]))
 

From ba530b77f955b832189d61605b02c1daa6d34121 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sun, 7 Apr 2024 16:37:20 +0200
Subject: [PATCH 42/65] refactorings and documentation

---
 hiclass/HierarchicalClassifier.py             | 19 ++-----------------
 hiclass/LocalClassifierPerLevel.py            |  8 ++------
 hiclass/LocalClassifierPerNode.py             |  4 +---
 hiclass/LocalClassifierPerParentNode.py       |  2 +-
 hiclass/_calibration/VennAbersCalibrator.py   | 17 +----------------
 .../ArithmeticMeanCombiner.py                 |  4 ++++
 .../GeometricMeanCombiner.py                  |  4 ++++
 .../probability_combiner/MultiplyCombiner.py  |  4 ++++
 8 files changed, 19 insertions(+), 43 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index eae04900..3ee73544 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -125,7 +125,7 @@ def fit(self, X, y, sample_weight=None):
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The training input samples. Internally, its dtype will be converted
             to ``dtype=np.float32``. If a sparse matrix is provided, it will be
-            converted into a sparse ``csc_matrix``.
+            converted into a sparse ``csr_matrix``.
         y : array-like of shape (n_samples, n_levels)
             The target values, i.e., hierarchical class labels for classification.
         sample_weight : array-like of shape (n_samples,), default=None
@@ -212,7 +212,7 @@ def calibrate(self, X, y):
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The calibration input samples. Internally, its dtype will be converted
             to ``dtype=np.float32``. If a sparse matrix is provided, it will be
-            converted into a sparse ``csc_matrix``.
+            converted into a sparse ``csr_matrix``.
         y : array-like of shape (n_samples, n_levels)
             The target values, i.e., hierarchical class labels for classification.
 
@@ -236,13 +236,9 @@ def calibrate(self, X, y):
         if self.calibration_method == "cvap":
             # combine train and calibration dataset for cross validation
             if isinstance(self.X_, scipy.sparse._csr.csr_matrix):
-                self.logger_.info(f"Sparse Calibration size: {X.shape} train size: {self.X_.shape}")
                 self.X_cal = scipy.sparse.vstack([self.X_, X])
-                self.logger_.info(f"CV Dataset X: {str(type(self.X_cal))} {str(self.X_cal.shape)}")
             else:
-                self.logger_.info(f"Not sparse Calibration size: {X.shape} train size: {self.X_.shape}")
                 self.X_cal = np.vstack([self.X_, X])
-                self.logger_.info(f"CV Dataset X: {str(type(self.X_cal))} {str(self.X_cal.shape)}")
             
             y = make_leveled(y)
             y = self._disambiguate(y)
@@ -495,17 +491,6 @@ def _clean_up(self):
             del self.X_cal
         if hasattr(self, 'y_cal'):
             del self.y_cal
-    
-    def _reorder_local_probabilities(self, probabilities, local_labels, level):
-        n_samples, n_labels = probabilities.shape[0], self.max_level_dimensions_[level]
-        sorted_probabilities = np.zeros(shape=(n_samples, n_labels))
-
-        for idx, label in enumerate(local_labels):
-            #local_label = label.split(self.separator_)[level]
-            new_idx = self.global_class_to_index_mapping_[level][label]
-            sorted_probabilities[:, new_idx] = probabilities[:, idx]
-        
-        return sorted_probabilities
 
     def _combine_and_reorder(self, proba):
         res = [proba[0]]
diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index c6ce45c0..a30140a0 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -114,7 +114,7 @@ def fit(self, X, y, sample_weight=None):
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The training input samples. Internally, its dtype will be converted
             to ``dtype=np.float32``. If a sparse matrix is provided, it will be
-            converted into a sparse ``csc_matrix``.
+            converted into a sparse ``csr_matrix``.
         y : array-like of shape (n_samples, n_levels)
             The target values, i.e., hierarchical class labels for classification.
         sample_weight : array-like of shape (n_samples,), default=None
@@ -232,8 +232,6 @@ def _predict_proba_remaining_levels(self, X, y):
             # use classifier as a fallback if no calibrator is available
             calibrator = calibrator or classifier
             probabilities = calibrator.predict_proba(X)
-            # sort probabilities
-            #probabilities = self._reorder_local_probabilities(probabilities, calibrator.classes_, level)
             level_probability_list.append(probabilities)
         return level_probability_list
 
@@ -290,8 +288,6 @@ def _initialize_local_classifiers(self):
     
     def _initialize_local_calibrators(self):
         super()._initialize_local_calibrators()
-        #train_length = self.X_.shape[0]
-        #cal_length = self.X_cal.shape[0]
         self.local_calibrators_ = [
                 _Calibrator(estimator=local_classifier, method=self.calibration_method) for local_classifier in self.local_classifiers_
         ]
@@ -397,7 +393,7 @@ def _fit_calibrator(self, level, separator):
             return None
         
         X, y, _ = self._remove_empty_leaves(
-            separator, self.X_cal, self.y_cal[:, level], None #X_cross_val, y_cross_val
+            separator, self.X_cal, self.y_cal[:, level], None
         )
         if len(y) == 0 or len(np.unique(y)) < 2:
             self.logger_.info(f"No calibration samples to fit calibrator for level: {str(level)}")
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index 40b4074f..ecb0249e 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -121,7 +121,7 @@ def fit(self, X, y, sample_weight=None):
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The training input samples. Internally, its dtype will be converted
             to ``dtype=np.float32``. If a sparse matrix is provided, it will be
-            converted into a sparse ``csc_matrix``.
+            converted into a sparse ``csr_matrix``.
         y : array-like of shape (n_samples, n_levels)
             The target values, i.e., hierarchical class labels for classification.
         sample_weight : array-like of shape (n_samples,), default=None
@@ -198,7 +198,6 @@ def predict(self, X):
                     successor_name = str(successor).split(self.separator_)[-1]
                     #self
                     self.logger_.info(f"Predicting for node '{successor_name}'")
-                    # TODO: use calibrator if using calibration to predict class
                     classifier = self.hierarchy_.nodes[successor]["classifier"]
                     positive_index = np.where(classifier.classes_ == 1)[0]
                     probabilities[:, i] = classifier.predict_proba(subset_x)[
@@ -408,7 +407,6 @@ def _fit_calibrator(self, node):
         X, y, _ = self.cal_binary_policy_.get_binary_examples(node)
         if len(y) == 0 or len(np.unique(y)) < 2:
             self.logger_.info(f"No calibration samples to fit calibrator for node: {str(node)}")
-            #self.hierarchy_.nodes[node].pop('calibrator', None)
             return None
         calibrator.fit(X, y)
         return calibrator
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index 81e4221f..d8281ad2 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -106,7 +106,7 @@ def fit(self, X, y, sample_weight=None):
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The training input samples. Internally, its dtype will be converted
             to ``dtype=np.float32``. If a sparse matrix is provided, it will be
-            converted into a sparse ``csc_matrix``.
+            converted into a sparse ``csr_matrix``.
         y : array-like of shape (n_samples, n_levels)
             The target values, i.e., hierarchical class labels for classification.
         sample_weight : array-like of shape (n_samples,), default=None
diff --git a/hiclass/_calibration/VennAbersCalibrator.py b/hiclass/_calibration/VennAbersCalibrator.py
index 9548e591..35511bf2 100644
--- a/hiclass/_calibration/VennAbersCalibrator.py
+++ b/hiclass/_calibration/VennAbersCalibrator.py
@@ -247,15 +247,10 @@ def predict_proba(self, scores):
             return scores
 
         if self.multiclass:
-
             score_splits = [scores[:, i] for i in range(scores.shape[1])]
             probabilities = np.zeros((scores.shape[0], scores.shape[1]))
 
             if self.used_cv:
-
-                #score_splits = [scores[:, i] for i in range(scores.shape[1])]
-                #probabilities = np.zeros((scores.shape[0], scores.shape[1]))
-
                 for idx, scores in enumerate(score_splits):
                     res = []
 
@@ -272,20 +267,10 @@ def predict_proba(self, scores):
 
                     p1_gm = gmean(p1)
                     probabilities[:, idx] = p1_gm / (gmean(1 - p0) + p1_gm)
-                
-                # normalize
-                #probabilities /= probabilities.sum(axis=1, keepdims=True)
-                #return probabilities
-            
+
             else:
-                #score_splits = [scores[:, i] for i in range(scores.shape[1])]
-                #probabilities = np.zeros((scores.shape[0], scores.shape[1]))
                 for idx, scores in enumerate(score_splits):
                     probabilities[:, idx] = self.ivaps[idx].predict_proba(scores)
-                
-                # normalize
-                #probabilities /= probabilities.sum(axis=1, keepdims=True)
-                #return probabilities
                     
             # normalize       
             probabilities /= probabilities.sum(axis=1, keepdims=True)
diff --git a/hiclass/probability_combiner/ArithmeticMeanCombiner.py b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
index e042b9f1..546a5e91 100644
--- a/hiclass/probability_combiner/ArithmeticMeanCombiner.py
+++ b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
@@ -5,6 +5,10 @@
 
 class ArithmeticMeanCombiner(ProbabilityCombiner):
     def combine(self, proba):
+        '''Combine probabilities of each level with probabilities of previous levels.
+        
+        Calculate the arithmetic mean of node probabilities and the probabilities of its predecessors.
+        '''
         res = [proba[0]]
         sums = [proba[0]]
         for level in range(1, self.classifier.max_levels_):
diff --git a/hiclass/probability_combiner/GeometricMeanCombiner.py b/hiclass/probability_combiner/GeometricMeanCombiner.py
index b9c125cd..b22a13cf 100644
--- a/hiclass/probability_combiner/GeometricMeanCombiner.py
+++ b/hiclass/probability_combiner/GeometricMeanCombiner.py
@@ -5,6 +5,10 @@
 
 class GeometricMeanCombiner(ProbabilityCombiner):
     def combine(self, proba):
+        '''Combine probabilities of each level with probabilities of previous levels.
+        
+        Calculate the geometric mean of node probabilities and the probabilities of its predecessors.
+        '''
         res = [proba[0]]
         log_sum = [np.log(proba[0])]
         for level in range(1, self.classifier.max_levels_):
diff --git a/hiclass/probability_combiner/MultiplyCombiner.py b/hiclass/probability_combiner/MultiplyCombiner.py
index a311b063..fbcb692f 100644
--- a/hiclass/probability_combiner/MultiplyCombiner.py
+++ b/hiclass/probability_combiner/MultiplyCombiner.py
@@ -5,6 +5,10 @@
 
 class MultiplyCombiner(ProbabilityCombiner):
     def combine(self, proba):
+        '''Combine probabilities of each level with probabilities of previous levels.
+        
+        Multiply node probabilities with the probabilities of its predecessors.
+        '''
         res = [proba[0]]
         for level in range(1, self.classifier.max_levels_):
             level_probs = np.zeros_like(proba[level])

From 555b697789757f7215c6ba676a8567e67d8e317e Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sun, 7 Apr 2024 22:10:26 +0200
Subject: [PATCH 43/65] add beta calibration

---
 hiclass/LocalClassifierPerLevel.py      |  2 +-
 hiclass/LocalClassifierPerNode.py       |  2 +-
 hiclass/LocalClassifierPerParentNode.py |  3 +-
 hiclass/_calibration/BetaCalibrator.py  | 37 +++++++++++++++++++++++++
 hiclass/_calibration/Calibrator.py      |  5 +++-
 tests/test_calibration.py               | 13 ++++++++-
 6 files changed, 56 insertions(+), 6 deletions(-)
 create mode 100644 hiclass/_calibration/BetaCalibrator.py

diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index 9b9e62a3..26e8d80e 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -82,7 +82,7 @@ def __init__(
             If :code:`Ray` is installed it is used, otherwise it defaults to :code:`Joblib`.
         bert : bool, default=False
             If True, skip scikit-learn's checks and sample_weight passing for BERT.
-        calibration_method : {"ivap", "cvap", "platt", "isotonic"}, str, default=None
+        calibration_method : {"ivap", "cvap", "platt", "isotonic", "beta"}, str, default=None
             If set, use the desired method to calibrate probabilities returned by predict_proba().
         return_all_probabilities : bool, default=False
             If True, return probabilities for all levels. Otherwise, return only probabilities for the last level.
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index d6272c84..88b9ee32 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -88,7 +88,7 @@ def __init__(
             If :code:`Ray` is installed it is used, otherwise it defaults to :code:`Joblib`.
         bert : bool, default=False
             If True, skip scikit-learn's checks and sample_weight passing for BERT.
-        calibration_method : {"ivap", "cvap", "platt", "isotonic"}, str, default=None
+        calibration_method : {"ivap", "cvap", "platt", "isotonic", "beta"}, str, default=None
             If set, use the desired method to calibrate probabilities returned by predict_proba().
         return_all_probabilities : bool, default=False
             If True, return probabilities for all levels. Otherwise, return only probabilities for the last level.
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index e5ab99be..fb7b40d3 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -74,7 +74,7 @@ def __init__(
             If :code:`Ray` is installed it is used, otherwise it defaults to :code:`Joblib`.
         bert : bool, default=False
             If True, skip scikit-learn's checks and sample_weight passing for BERT.
-        calibration_method : {"ivap", "cvap", "platt", "isotonic"}, str, default=None
+        calibration_method : {"ivap", "cvap", "platt", "isotonic", "beta"}, str, default=None
             If set, use the desired method to calibrate probabilities returned by predict_proba().
         return_all_probabilities : bool, default=False
             If True, return probabilities for all levels. Otherwise, return only probabilities for the last level.
@@ -310,7 +310,6 @@ def _get_successors(self, node, calibration=False):
         for row in masked_labels:
             if node == self.root_:
                 y.append(row[0])
-                self.logger_.info(y)
             else:
                 y.append(row[np.where(row == node)[0][0] + 1])
         y = np.array(y)
diff --git a/hiclass/_calibration/BetaCalibrator.py b/hiclass/_calibration/BetaCalibrator.py
new file mode 100644
index 00000000..fc3f6c4c
--- /dev/null
+++ b/hiclass/_calibration/BetaCalibrator.py
@@ -0,0 +1,37 @@
+from hiclass._calibration.BinaryCalibrator import _BinaryCalibrator
+from sklearn.utils.validation import check_is_fitted
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+
+
+class _BetaCalibrator(_BinaryCalibrator):
+    name = "BetaCalibrator"
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.skip_calibration = False
+
+    def fit(self, y, scores, X=None):
+        unique_labels = len(np.unique(y))
+        if unique_labels < 2:
+            self.skip_calibration = True
+            self._is_fitted = True
+            return self
+
+        scores_1 = np.log(scores)
+        scores_2 = -np.log(1 - scores)
+        feature_matrix = np.column_stack((scores_1, scores_2))
+
+        lr = LogisticRegression()
+        lr.fit(feature_matrix, y)
+        self.a, self.b = lr.coef_.flatten()
+        self.c = lr.intercept_[0]
+        
+        self._is_fitted = True
+        return self
+
+    def predict_proba(self, scores, X=None):
+        check_is_fitted(self)
+        if self.skip_calibration:
+            return scores
+        return 1/(1+1/(np.exp(self.c)*(np.power(scores, self.a) / np.power((1 - scores), self.b))))
\ No newline at end of file
diff --git a/hiclass/_calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
index 671736bb..2697ae44 100644
--- a/hiclass/_calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -5,11 +5,12 @@
 from hiclass._calibration.VennAbersCalibrator import _InductiveVennAbersCalibrator, _CrossVennAbersCalibrator
 from hiclass._calibration.IsotonicRegression import _IsotonicRegression
 from hiclass._calibration.PlattScaling import _PlattScaling
+from hiclass._calibration.BetaCalibrator import _BetaCalibrator
 from hiclass._calibration.calibration_utils import _one_vs_rest_split
 
 
 class _Calibrator(BaseEstimator):
-    available_methods = ["ivap", "cvap", "sigmoid", "isotonic"]
+    available_methods = ["ivap", "cvap", "sigmoid", "isotonic", "beta"]
     _multiclass_methods = ["cvap"]
 
     def __init__(self, estimator, method="ivap", **method_params) -> None:
@@ -112,6 +113,8 @@ def _create_calibrator(self, name, params):
             return _PlattScaling()
         elif name == "isotonic":
             return _IsotonicRegression(params)
+        elif name == "beta":
+            return _BetaCalibrator()
 
     def __sklearn_is_fitted__(self):
         """
diff --git a/tests/test_calibration.py b/tests/test_calibration.py
index b738d83d..0ef04f70 100644
--- a/tests/test_calibration.py
+++ b/tests/test_calibration.py
@@ -8,6 +8,7 @@
 from hiclass._calibration.VennAbersCalibrator import _InductiveVennAbersCalibrator, _CrossVennAbersCalibrator
 from hiclass._calibration.PlattScaling import _PlattScaling
 from hiclass._calibration.IsotonicRegression import _IsotonicRegression
+from hiclass._calibration.BetaCalibrator import _BetaCalibrator
 from hiclass._calibration.Calibrator import _Calibrator
 
 @pytest.fixture
@@ -157,13 +158,23 @@ def test_isotonic_regression(binary_calibration_data, binary_test_scores):
     assert proba.shape == (len(binary_test_scores),)
     assert_array_almost_equal(proba, np.array([1.0, 0.33333333, 0.0, 1.0, 0.0, 0.33333333, 0.0]))
 
+def test_beta_calibration(binary_calibration_data, binary_test_scores):
+    cal_scores, cal_labels = binary_calibration_data
+    calibrator = _BetaCalibrator()
+    calibrator.fit(cal_labels, cal_scores[:, 1])
+    proba = calibrator.predict_proba(binary_test_scores[:, 1])
+
+    assert proba.shape == (len(binary_test_scores),)
+    assert_array_almost_equal(proba, np.array([0.526125, 0.423743, 0.363907, 0.785855, 0.323201, 0.417089, 0.0]))
+
 def test_illegal_calibration_method_raises_error(binary_mock_estimator):
     with pytest.raises(ValueError, match="abc is not a valid calibration method."):
         _Calibrator(binary_mock_estimator, method="abc")
 
 def test_not_fitted_calibrator_throws_error(binary_test_scores, binary_mock_estimator):
     for calibrator in [_PlattScaling(), 
-                       _IsotonicRegression(), 
+                       _IsotonicRegression(),
+                       _BetaCalibrator(),
                        _InductiveVennAbersCalibrator(), 
                        _CrossVennAbersCalibrator(binary_mock_estimator)]:
             with pytest.raises(NotFittedError):

From 44e5a20619eeef155f89804ccbaa31c152204eee Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Mon, 8 Apr 2024 14:35:34 +0200
Subject: [PATCH 44/65] flake8 linting

---
 hiclass/HierarchicalClassifier.py             | 18 ++---
 hiclass/LocalClassifierPerLevel.py            | 20 +++--
 hiclass/LocalClassifierPerNode.py             | 14 ++--
 hiclass/LocalClassifierPerParentNode.py       | 23 +++---
 hiclass/Pipeline.py                           |  1 +
 hiclass/_calibration/BetaCalibrator.py        |  4 +-
 hiclass/_calibration/BinaryCalibrator.py      |  2 +-
 hiclass/_calibration/Calibrator.py            |  4 +-
 hiclass/_calibration/VennAbersCalibrator.py   | 74 ++++++++++---------
 hiclass/_calibration/calibration_utils.py     | 17 +++--
 hiclass/metrics.py                            | 53 +++++++------
 .../ArithmeticMeanCombiner.py                 | 12 ++-
 .../GeometricMeanCombiner.py                  | 13 ++--
 .../probability_combiner/MultiplyCombiner.py  |  9 +--
 .../ProbabilityCombiner.py                    |  9 ++-
 15 files changed, 140 insertions(+), 133 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index 23a30233..bbf698de 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -239,14 +239,14 @@ def calibrate(self, X, y):
             X = check_array(X, accept_sparse="csr", allow_nd=True, ensure_2d=False)
         else:
             X = np.array(X)
-        
+
         if self.calibration_method == "cvap":
             # combine train and calibration dataset for cross validation
             if isinstance(self.X_, scipy.sparse._csr.csr_matrix):
                 self.X_cal = scipy.sparse.vstack([self.X_, X])
             else:
                 self.X_cal = np.vstack([self.X_, X])
-            
+
             y = make_leveled(y)
             y = self._disambiguate(y)
             y = self._convert_1d_y_to_2d(y)
@@ -257,7 +257,7 @@ def calibrate(self, X, y):
             y = self._disambiguate(y)
             y = self._convert_1d_y_to_2d(y)
             self.y_cal = y
-            
+
         self.logger_.info("Calibrating")
 
         # Create a calibrator for each local classifier
@@ -417,7 +417,7 @@ def _fit_node_classifier(
         def logging_wrapper(func, idx, node, node_length):
             self.logger_.info(f"fitting node {idx+1}/{node_length}: {str(node)}")
             return func(self, node)
-        
+
         if self.n_jobs > 1:
             if _has_ray and not use_joblib:
                 if not ray.is_initialized:
@@ -427,7 +427,7 @@ def logging_wrapper(func, idx, node, node_length):
                         ignore_reinit_error=True,
                     )
                 lcppn = ray.put(self)
-                _parallel_fit = ray.remote(self._fit_classifier) # TODO: use logging wrapper
+                _parallel_fit = ray.remote(self._fit_classifier)  # TODO: use logging wrapper
                 results = [_parallel_fit.remote(lcppn, node) for node in nodes]
                 classifiers = ray.get(results)
             else:
@@ -456,7 +456,7 @@ def logging_wrapper(func, idx, node, node_length):
                     )
                 lcppn = ray.put(self)
                 _parallel_fit = ray.remote(self._fit_calibrator)
-                results = [_parallel_fit.remote(lcppn, node) for idx, node in enumerate(nodes)] # TODO: use logging wrapper
+                results = [_parallel_fit.remote(lcppn, node) for idx, node in enumerate(nodes)]  # TODO: use logging wrapper
                 calibrators = ray.get(results)
                 ray.shutdown()
             else:
@@ -469,7 +469,7 @@ def logging_wrapper(func, idx, node, node_length):
 
         for calibrator, node in zip(calibrators, nodes):
             self.hierarchy_.nodes[node]["calibrator"] = calibrator
-        
+
     @staticmethod
     def _fit_classifier(self, node):
         raise NotImplementedError(
@@ -479,7 +479,7 @@ def _fit_classifier(self, node):
     @staticmethod
     def _fit_calibrator(self, node):
         raise NotImplementedError("Method should be implemented in the LCPN and LCPPN")
-    
+
     def _create_probability_combiner(self, name):
         if name == 'geometric':
             return GeometricMeanCombiner(self)
@@ -512,7 +512,7 @@ def _combine_and_reorder(self, proba):
                 new_idx = self.class_to_index_mapping_[level][local_label]
                 res_proba[:, new_idx] += proba[level][:, old_idx]
 
-            res.append(res_proba) 
+            res.append(res_proba)
         return res
 
     def _fit_digraph(self, local_mode: bool = False, use_joblib: bool = False):
diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index 26e8d80e..d6ec6d83 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -194,7 +194,7 @@ def predict(self, X):
         self._remove_separator(y)
 
         return y
-    
+
     def predict_proba(self, X):
         # Check if fit has been called
         check_is_fitted(self)
@@ -243,7 +243,6 @@ def _predict_proba_remaining_levels(self, X, y):
             level_probability_list.append(probabilities)
         return level_probability_list
 
-
     def _predict_remaining_levels(self, X, y):
         for level in range(1, y.shape[1]):
             classifier = self.local_classifiers_[level]
@@ -293,11 +292,11 @@ def _initialize_local_classifiers(self):
             deepcopy(self.local_classifier_) for _ in range(self.y_.shape[1])
         ]
         self.masks_ = [None for _ in range(self.y_.shape[1])]
-    
+
     def _initialize_local_calibrators(self):
         super()._initialize_local_calibrators()
         self.local_calibrators_ = [
-                _Calibrator(estimator=local_classifier, method=self.calibration_method) for local_classifier in self.local_classifiers_
+            _Calibrator(estimator=local_classifier, method=self.calibration_method) for local_classifier in self.local_classifiers_
         ]
 
     def _fit_digraph(self, local_mode: bool = False, use_joblib: bool = False):
@@ -334,7 +333,7 @@ def logging_wrapper(func, level, separator, max_level):
             ]
         for level, classifier in enumerate(classifiers):
             self.local_classifiers_[level] = classifier
-    
+
     def _calibrate_digraph(self, local_mode: bool = False, use_joblib: bool = False):
         self.logger_.info("Fitting local calibrators")
 
@@ -357,20 +356,19 @@ def logging_wrapper(func, level, separator, max_level):
                     for level in range(len(self.local_calibrators_))
                 ]
                 calibrators = ray.get(results)
-                
+
             else:
                 calibrators = Parallel(n_jobs=self.n_jobs)(
                     delayed(logging_wrapper)(self._fit_calibrator, level, self.separator_, len(self.local_calibrators_)) for level in range(len(self.local_calibrators_))
-                )  
+                )
         else:
             calibrators = [
                 logging_wrapper(self._fit_calibrator, level, self.separator_, len(self.local_calibrators_))
                 for level in range(len(self.local_calibrators_))
             ]
-        
+
         for level, calibrator in enumerate(calibrators):
             self.local_calibrators_[level] = calibrator
-        
 
     @staticmethod
     def _fit_classifier(self, level, separator):
@@ -406,10 +404,10 @@ def _fit_classifier(self, level, separator):
     def _fit_calibrator(self, level, separator):
         try:
             calibrator = self.local_calibrators_[level]
-        except:
+        except IndexError:
             self.logger_.info("no calibrator for " + "level: " + str(level))
             return None
-        
+
         X, y, _ = self._remove_empty_leaves(
             separator, self.X_cal, self.y_cal[:, level], None
         )
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index 88b9ee32..158c70ee 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -118,7 +118,7 @@ def __init__(
         self.probability_combiner = probability_combiner
 
         if self.probability_combiner and self.probability_combiner not in probability_combiner_init_strings:
-                raise ValueError(f"probability_combiner must be one of {', '.join(probability_combiner_init_strings)} or None.")
+            raise ValueError(f"probability_combiner must be one of {', '.join(probability_combiner_init_strings)} or None.")
 
     def fit(self, X, y, sample_weight=None):
         """
@@ -204,7 +204,6 @@ def predict(self, X):
                 probabilities = np.zeros((subset_x.shape[0], len(successors)))
                 for i, successor in enumerate(successors):
                     successor_name = str(successor).split(self.separator_)[-1]
-                    #self
                     self.logger_.info(f"Predicting for node '{successor_name}'")
                     classifier = self.hierarchy_.nodes[successor]["classifier"]
                     positive_index = np.where(classifier.classes_ == 1)[0]
@@ -220,7 +219,7 @@ def predict(self, X):
                 )
                 prediction = np.array(prediction)
                 y[mask, level] = prediction
-                
+
         y = self._convert_to_1d(y)
 
         self._remove_separator(y)
@@ -279,7 +278,7 @@ def predict_proba(self, X):
                 mask = [True] * X.shape[0]
                 subset_x = X[mask]
             else:
-                mask = np.isin(y, self.global_classes_[level-1]).any(axis=1)
+                mask = np.isin(y, self.global_classes_[level - 1]).any(axis=1)
                 subset_x = X[mask]
 
             if subset_x.shape[0] > 0:
@@ -308,13 +307,13 @@ def predict_proba(self, X):
 
         # normalize probabilities
         level_probability_list = [
-            np.nan_to_num(level_probabilities / level_probabilities.sum(axis=1, keepdims=True)) 
-                for level_probabilities in level_probability_list
+            np.nan_to_num(level_probabilities / level_probabilities.sum(axis=1, keepdims=True))
+            for level_probabilities in level_probability_list
         ]
 
         # combine probabilities horizontally
         level_probability_list = self._combine_and_reorder(level_probability_list)
-        
+
         # combine probabilities vertically
         if self.probability_combiner:
             probability_combiner_ = self._create_probability_combiner(self.probability_combiner)
@@ -323,7 +322,6 @@ def predict_proba(self, X):
 
         return level_probability_list if self.return_all_probabilities else level_probability_list[-1]
 
-
     def _initialize_binary_policy(self, calibration=False):
         if isinstance(self.binary_policy, str):
             self.logger_.info(f"Initializing {self.binary_policy} binary policy")
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index fb7b40d3..90931631 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -20,6 +20,7 @@
 
 from hiclass.probability_combiner import init_strings as probability_combiner_init_strings
 
+
 class LocalClassifierPerParentNode(BaseEstimator, HierarchicalClassifier):
     """
     Assign local classifiers to each parent node of the graph.
@@ -185,7 +186,7 @@ def predict(self, X):
         self._remove_separator(y)
 
         return y
-    
+
     def predict_proba(self, X):
         # Check if fit has been called
         check_is_fitted(self)
@@ -198,7 +199,7 @@ def predict_proba(self, X):
 
         if not self.calibration_method:
             self.logger_.info("It is not recommended to use predict_proba() without calibration")
-        
+
         self.logger_.info("Predicting Probability")
 
         # Initialize array that holds predictions
@@ -212,7 +213,7 @@ def predict_proba(self, X):
 
         y[:, 0] = calibrator.classes_[np.argmax(proba, axis=1)]
         level_probability_list = [proba] + self._predict_proba_remaining_levels(X, y)
-    
+
         level_probability_list = self._combine_and_reorder(level_probability_list)
 
         # combine probabilities
@@ -220,7 +221,7 @@ def predict_proba(self, X):
             probability_combiner_ = self._create_probability_combiner(self.probability_combiner)
             self.logger_.info(f"Combining probabilities using {type(probability_combiner_).__name__}")
             level_probability_list = probability_combiner_.combine(level_probability_list)
-        
+
         return level_probability_list if self.return_all_probabilities else level_probability_list[-1]
 
     def _predict_proba_remaining_levels(self, X, y):
@@ -232,7 +233,7 @@ def _predict_proba_remaining_levels(self, X, y):
             cur_level_probabilities = np.zeros((X.shape[0], level_dimension))
 
             for predecessor in predecessors:
-                mask = np.isin(y[:, level - 1], self.global_classes_[level-1])
+                mask = np.isin(y[:, level - 1], self.global_classes_[level - 1])
                 predecessor_x = X[mask]
                 if predecessor_x.shape[0] > 0:
                     successors = list(self.hierarchy_.successors(predecessor))
@@ -251,15 +252,15 @@ def _predict_proba_remaining_levels(self, X, y):
                             cur_level_probabilities[mask, class_index] = proba[:, proba_index]
 
             level_probability_list.append(cur_level_probabilities)
-        
+
         # normalize probabilities
         level_probability_list = [
-            np.nan_to_num(level_probabilities / level_probabilities.sum(axis=1, keepdims=True)) 
-                for level_probabilities in level_probability_list
+            np.nan_to_num(level_probabilities / level_probabilities.sum(axis=1, keepdims=True))
+            for level_probabilities in level_probability_list
         ]
 
         return level_probability_list
-        
+
     def _predict_remaining_levels(self, X, y):
         for level in range(1, y.shape[1]):
             predecessors = set(y[:, level - 1])
@@ -359,12 +360,12 @@ def _fit_calibrator(self, node):
             return None
         calibrator.fit(X, y)
         return calibrator
-        
+
     def _fit_digraph(self, local_mode: bool = False, use_joblib: bool = False):
         self.logger_.info("Fitting local classifiers")
         nodes = self._get_parents()
         self._fit_node_classifier(nodes, local_mode, use_joblib)
-    
+
     def _calibrate_digraph(self, local_mode: bool = False, use_joblib: bool = False):
         self.logger_.info("Fitting local calibrators")
         nodes = self._get_parents()
diff --git a/hiclass/Pipeline.py b/hiclass/Pipeline.py
index e347c4c6..828f184e 100644
--- a/hiclass/Pipeline.py
+++ b/hiclass/Pipeline.py
@@ -1,5 +1,6 @@
 from sklearn.pipeline import Pipeline as skPipeline
 
+
 class Pipeline(skPipeline):
     def __init__(self, steps, **kwargs):
         super().__init__(steps, **kwargs)
diff --git a/hiclass/_calibration/BetaCalibrator.py b/hiclass/_calibration/BetaCalibrator.py
index fc3f6c4c..e2d91469 100644
--- a/hiclass/_calibration/BetaCalibrator.py
+++ b/hiclass/_calibration/BetaCalibrator.py
@@ -26,7 +26,7 @@ def fit(self, y, scores, X=None):
         lr.fit(feature_matrix, y)
         self.a, self.b = lr.coef_.flatten()
         self.c = lr.intercept_[0]
-        
+
         self._is_fitted = True
         return self
 
@@ -34,4 +34,4 @@ def predict_proba(self, scores, X=None):
         check_is_fitted(self)
         if self.skip_calibration:
             return scores
-        return 1/(1+1/(np.exp(self.c)*(np.power(scores, self.a) / np.power((1 - scores), self.b))))
\ No newline at end of file
+        return 1 / (1 + 1 / (np.exp(self.c) * (np.power(scores, self.a) / np.power((1 - scores), self.b))))
diff --git a/hiclass/_calibration/BinaryCalibrator.py b/hiclass/_calibration/BinaryCalibrator.py
index 20038537..2f24a9e8 100644
--- a/hiclass/_calibration/BinaryCalibrator.py
+++ b/hiclass/_calibration/BinaryCalibrator.py
@@ -10,7 +10,7 @@ def fit(self, y, scores, X=None):  # pragma: no cover
     @abc.abstractmethod
     def predict_proba(self, scores, X=None):  # pragma: no cover
         ...
-        
+
     def __sklearn_is_fitted__(self):
         """
         Check fitted status and return a Boolean value.
diff --git a/hiclass/_calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
index 2697ae44..40e6440b 100644
--- a/hiclass/_calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -17,13 +17,12 @@ def __init__(self, estimator, method="ivap", **method_params) -> None:
         assert callable(getattr(estimator, 'predict_proba', None))
         self.estimator = estimator
         self.method_params = method_params
-        #self.classes_ = self.estimator.classes_
+        # self.classes_ = self.estimator.classes_
         self.multiclass = False
         self.multiclass_support = (method in self._multiclass_methods)
         if method not in self.available_methods:
             raise ValueError(f"{method} is not a valid calibration method.")
         self.method = method
-    
 
     def fit(self, X, y):
         """
@@ -77,7 +76,6 @@ def fit(self, X, y):
             self.calibrators.append(calibrator)
         self._is_fitted = True
         return self
-    
 
     def predict_proba(self, X):
         test_scores = self.estimator.predict_proba(X)
diff --git a/hiclass/_calibration/VennAbersCalibrator.py b/hiclass/_calibration/VennAbersCalibrator.py
index 35511bf2..5365981e 100644
--- a/hiclass/_calibration/VennAbersCalibrator.py
+++ b/hiclass/_calibration/VennAbersCalibrator.py
@@ -6,6 +6,7 @@
 from collections import defaultdict
 from sklearn.utils.validation import check_is_fitted
 
+
 class _InductiveVennAbersCalibrator(_BinaryCalibrator):
     name = "InductiveVennAbersCalibrator"
 
@@ -39,7 +40,7 @@ def fit(self, y, scores, X=None):
             matching_idx = np.where(ordered_calibration_scores == s_j)
             matching_labels = ordered_calibration_labels[matching_idx]
             y[j] = np.sum(matching_labels) / w[unique_elements[j]]
-        
+
         csd_1[1:, 0] = np.cumsum(unique_element_counts)
         csd_1[1:, 1] = np.cumsum(y * unique_element_counts)
 
@@ -56,39 +57,39 @@ def fit(self, y, scores, X=None):
         self._is_fitted = True
 
         return self
-    
+
     def _non_left_angle_turn(self, next_to_top, top, p_i):
         res = np.cross((top - next_to_top), (p_i - top))
         return res <= 0
-    
+
     def _non_right_angle_turn(self, next_to_top, top, p_i):
         res = np.cross((top - next_to_top), (p_i - top))
         return res >= 0
-    
+
     def _initialize_f1_corners(self, csd):
-            stack = []
-            # append P_{-1} and P_0
-            stack.append(csd[0])
-            stack.append(csd[1])
-
-            for i in range(2, len(csd)):
-                while len(stack) > 1 and self._non_left_angle_turn(next_to_top=stack[-2], top=stack[-1], p_i=csd[i]):
-                    stack.pop()
-                stack.append(csd[i])
-
-            return stack
-    
+        stack = []
+        # append P_{-1} and P_0
+        stack.append(csd[0])
+        stack.append(csd[1])
+
+        for i in range(2, len(csd)):
+            while len(stack) > 1 and self._non_left_angle_turn(next_to_top=stack[-2], top=stack[-1], p_i=csd[i]):
+                stack.pop()
+            stack.append(csd[i])
+
+        return stack
+
     def _initialize_f0_corners(self, csd):
-            stack = []
-            # append p_{k'+1}, p_{k'}
-            stack.append(csd[-1])
-            stack.append(csd[-2])
+        stack = []
+        # append p_{k'+1}, p_{k'}
+        stack.append(csd[-1])
+        stack.append(csd[-2])
 
-            for i in range(len(csd) - 3, -1, -1):
-                while len(stack) > 1 and self._non_right_angle_turn(next_to_top=stack[-2], top=stack[-1], p_i=csd[i]):
-                    stack.pop()
-                stack.append(csd[i])
-            return stack
+        for i in range(len(csd) - 3, -1, -1):
+            while len(stack) > 1 and self._non_right_angle_turn(next_to_top=stack[-2], top=stack[-1], p_i=csd[i]):
+                stack.pop()
+            stack.append(csd[i])
+        return stack
 
     def _slope(self, top, next_to_top):
         return (next_to_top[1] - top[1]) / (next_to_top[0] - top[0])
@@ -135,7 +136,7 @@ def _compute_f0(self, prev_stack, csd):
                 stack.pop()
             stack.append(csd[i])
         return F0
-    
+
     def predict_proba(self, scores, X=None):
         check_is_fitted(self)
         lower = np.searchsorted(self._unique_elements, scores, side="left")
@@ -154,6 +155,7 @@ def predict_intervall(self, scores):
 
         return np.array(list(zip(p0, p1)))
 
+
 class _CrossVennAbersCalibrator(_BinaryCalibrator):
     name = "CrossVennAbersCalibrator"
 
@@ -180,7 +182,7 @@ def fit(self, y, scores, X):
                 splits_x.append((X[train_index], X[cal_index]))
                 splits_y.append((y[train_index], y[cal_index]))
         except ValueError:
-                splits_x, splits_y = [], []
+            splits_x, splits_y = [], []
 
         # don't use cross validation
         if len(splits_x) == 0 or any([(len(np.unique(y_train)) < 2 or len(np.unique(y_cal)) < 2) for y_train, y_cal in splits_y]):
@@ -189,7 +191,7 @@ def fit(self, y, scores, X):
 
             if len(unique_labels) > 2:
                 # use one vs rest
-                score_splits, label_splits = _one_vs_rest_split(y, scores, self.estimator) # TODO use only original calibration samples
+                score_splits, label_splits = _one_vs_rest_split(y, scores, self.estimator)  # TODO use only original calibration samples
                 for i in range(len(score_splits)):
                     # create a calibrator for each split
                     calibrator = _InductiveVennAbersCalibrator()
@@ -197,7 +199,7 @@ def fit(self, y, scores, X):
                     self.ivaps.append(calibrator)
             elif len(unique_labels) == 2 and scores.ndim == 1:
                 calibrator = _InductiveVennAbersCalibrator()
-                calibrator.fit(y, scores) # TODO use only original calibration samples
+                calibrator.fit(y, scores)  # TODO use only original calibration samples
                 self.ivaps.append(calibrator)
             else:
                 print("no fitted ivaps!")
@@ -216,7 +218,7 @@ def fit(self, y, scores, X):
 
                 # calibrate IVAP with left out dataset
                 calibration_scores = model.predict_proba(X_cal)
-                
+
                 if calibration_scores.shape[1] > 2:
                     self.multiclass = True
                     # one vs rest calibration
@@ -259,9 +261,9 @@ def predict_proba(self, scores):
 
                     for calibrator in self.ovr_ivaps[idx]:
                         res.append(calibrator.predict_intervall(scores))
-                    
+
                     res = np.array(res)
-                    
+
                     p0 = res[:, :, 0]
                     p1 = res[:, :, 1]
 
@@ -271,8 +273,8 @@ def predict_proba(self, scores):
             else:
                 for idx, scores in enumerate(score_splits):
                     probabilities[:, idx] = self.ivaps[idx].predict_proba(scores)
-                    
-            # normalize       
+
+            # normalize
             probabilities /= probabilities.sum(axis=1, keepdims=True)
             return probabilities
 
@@ -280,10 +282,10 @@ def predict_proba(self, scores):
             res = []
             for calibrator in self.ivaps:
                 res.append(calibrator.predict_intervall(scores))
-            
+
             res = np.array(res)
             p0 = res[:, :, 0]
             p1 = res[:, :, 1]
 
             p1_gm = gmean(p1)
-            return p1_gm / (gmean(1 - p0) + p1_gm)
\ No newline at end of file
+            return p1_gm / (gmean(1 - p0) + p1_gm)
diff --git a/hiclass/_calibration/calibration_utils.py b/hiclass/_calibration/calibration_utils.py
index b0a06669..33672a0a 100644
--- a/hiclass/_calibration/calibration_utils.py
+++ b/hiclass/_calibration/calibration_utils.py
@@ -1,13 +1,14 @@
 from sklearn.preprocessing import LabelBinarizer
 
+
 def _one_vs_rest_split(y, scores, estimator):
-        # binarize multiclass labels
-        label_binarizer = LabelBinarizer()
-        label_binarizer.fit(estimator.classes_)
-        binary_labels = label_binarizer.transform(y).T
+    # binarize multiclass labels
+    label_binarizer = LabelBinarizer()
+    label_binarizer.fit(estimator.classes_)
+    binary_labels = label_binarizer.transform(y).T
 
-        # split scores into k one vs rest splits
-        score_splits = [scores[:, i] for i in range(scores.shape[1])]
-        label_splits = [binary_labels[i] for i in range(len(score_splits))]
+    # split scores into k one vs rest splits
+    score_splits = [scores[:, i] for i in range(scores.shape[1])]
+    label_splits = [binary_labels[i] for i in range(len(score_splits))]
 
-        return score_splits, label_splits
\ No newline at end of file
+    return score_splits, label_splits
diff --git a/hiclass/metrics.py b/hiclass/metrics.py
index c6509cc8..e2735288 100644
--- a/hiclass/metrics.py
+++ b/hiclass/metrics.py
@@ -276,16 +276,18 @@ def _prepare_data(classifier, y_true, y_prob, level, y_pred=None):
         if label in classifier_classes:
             old_idx = np.where(classifier_classes == label)[0][0]
             new_y_prob[:, idx] = y_prob[level][:, old_idx]
-    
+
     return y_true, y_pred, new_labels, new_y_prob
 
+
 def _aggregate_scores(scores, agg):
-        if agg == 'average':
-            return np.mean(scores)
-        if agg == 'sum':
-            return np.sum(scores)
-        if agg == None or agg == 'None':
-            return scores
+    if agg == 'average':
+        return np.mean(scores)
+    if agg == 'sum':
+        return np.sum(scores)
+    if agg is None or agg == 'None':
+        return scores
+
 
 def multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, agg='average'):
     scores = []
@@ -293,41 +295,48 @@ def multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarra
         scores.append(_multiclass_brier_score(classifier, y_true, y_prob, level))
     return _aggregate_scores(scores, agg)
 
+
 def log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, agg='average'):
     scores = []
     for level in range(make_leveled(y_true).shape[1]):
         scores.append(_log_loss(classifier, y_true, y_prob, level))
     return _aggregate_scores(scores, agg)
 
+
 def expected_calibration_error(classifier: HierarchicalClassifier, y_true, y_prob, y_pred, n_bins=10, agg='average'):
     scores = []
     for level in range(make_leveled(y_true).shape[1]):
         scores.append(_expected_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins))
     return _aggregate_scores(scores, agg)
 
+
 def statistical_calibration_error(classifier, y_true, y_prob, y_pred, n_bins=10, agg='average'):
     scores = []
     for level in range(make_leveled(y_true).shape[1]):
         scores.append(_statistical_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins=n_bins))
     return _aggregate_scores(scores, agg)
 
+
 def adaptive_calibration_error(classifier, y_true, y_prob, y_pred, n_ranges=10, agg='average'):
     scores = []
     for level in range(make_leveled(y_true).shape[1]):
         scores.append(_adaptive_calibration_error(classifier, y_true, y_prob, y_pred, level, n_ranges=n_ranges))
     return _aggregate_scores(scores, agg)
 
+
 def _multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, level: int):
     y_true, _, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level)
     label_encoder = LabelEncoder()
     label_encoder.fit(labels)
     y_true_encoded = label_encoder.transform(y_true)
-    return (1/y_prob.shape[0]) * np.sum(np.sum(np.square(y_prob - np.eye(y_prob.shape[1])[y_true_encoded]), axis=1))
+    return (1 / y_prob.shape[0]) * np.sum(np.sum(np.square(y_prob - np.eye(y_prob.shape[1])[y_true_encoded]), axis=1))
+
 
 def _log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, level: int):
     y_true, _, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level)
     return sk_log_loss(y_true, y_prob, labels=labels)
 
+
 def _expected_calibration_error(classifier: HierarchicalClassifier, y_true, y_prob, y_pred, level, n_bins=10):
     y_true, y_pred, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level, y_pred)
 
@@ -337,19 +346,19 @@ def _expected_calibration_error(classifier: HierarchicalClassifier, y_true, y_pr
 
     y_true_encoded = label_encoder.transform(y_true)
     y_pred_encoded = label_encoder.transform(y_pred)
-    
+
     y_prob = np.max(y_prob, axis=1)
     stacked = np.column_stack([y_prob, y_pred_encoded, y_true_encoded])
 
     # calculate equally sized bins
-    _, bin_edges = np.histogram(stacked, bins=n_bins, range=(0,1))
+    _, bin_edges = np.histogram(stacked, bins=n_bins, range=(0, 1))
     bin_indices = np.digitize(stacked, bin_edges)[:, 0]
 
     # add bin index to each data point
     data = np.column_stack([stacked, bin_indices])
 
     # create bin mask
-    masks = (data[:, -1, None] == range(1, n_bins+1)).T
+    masks = (data[:, -1, None] == range(1, n_bins + 1)).T
 
     # create actual bins
     bins = [data[masks[i]] for i in range(n_bins)]
@@ -364,6 +373,7 @@ def _expected_calibration_error(classifier: HierarchicalClassifier, y_true, y_pr
         ece += (bins[i].shape[0] / n) * abs(acc[i] - conf[i]) if bins[i].shape[0] != 0 else 0
     return ece
 
+
 def _statistical_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins=10):
     y_true, y_pred, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level, y_pred)
 
@@ -383,15 +393,15 @@ def _statistical_calibration_error(classifier, y_true, y_prob, y_pred, level, n_
         stacked = np.column_stack([class_scores, y_pred_encoded, y_true_encoded])
 
         # create bins
-        _, bin_edges = np.histogram(stacked, bins=n_bins, range=(0,1))
+        _, bin_edges = np.histogram(stacked, bins=n_bins, range=(0, 1))
         bin_indices = np.digitize(stacked, bin_edges)[:, 0]
 
         # add bin index to each data point
         data = np.column_stack([stacked, bin_indices])
 
         # create bin mask
-        masks = (data[:, -1, None] == range(1, n_bins+1)).T
-        
+        masks = (data[:, -1, None] == range(1, n_bins + 1)).T
+
         # create actual bins
         bins = [data[masks[i]] for i in range(n_bins)]
 
@@ -403,11 +413,12 @@ def _statistical_calibration_error(classifier, y_true, y_prob, y_pred, level, n_
             acc[i] = 1 / (bins[i].shape[0]) * np.sum((bins[i][:, 1] == bins[i][:, 2])) if bins[i].shape[0] != 0 else 0
             conf[i] = 1 / (bins[i].shape[0]) * np.sum(bins[i][:, 0]) if bins[i].shape[0] != 0 else 0
             error += (bins[i].shape[0] / n_samples) * abs(acc[i] - conf[i]) if bins[i].shape[0] != 0 else 0
-        
+
         class_error[k] = error
-            
+
     return np.mean(class_error)
 
+
 def _adaptive_calibration_error(classifier, y_true, y_prob, y_pred, level, n_ranges=10):
     y_true, y_pred, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level, y_pred)
 
@@ -428,15 +439,15 @@ def _adaptive_calibration_error(classifier, y_true, y_prob, y_pred, level, n_ran
         class_scores, ordered_y_pred_labels, ordered_y_true = class_scores[idx], y_pred_encoded[idx], y_true_encoded[idx]
         stacked = np.column_stack([np.array(range(len(class_scores))), class_scores, ordered_y_pred_labels, ordered_y_true])
 
-        bin_edges = np.floor(np.linspace(0, len(class_scores), n_ranges+1, endpoint=True)).astype(int)
-        _, bin_edges = np.histogram(stacked, bins=bin_edges, range=(0,len(class_scores)))
+        bin_edges = np.floor(np.linspace(0, len(class_scores), n_ranges + 1, endpoint=True)).astype(int)
+        _, bin_edges = np.histogram(stacked, bins=bin_edges, range=(0, len(class_scores)))
         bin_indices = np.digitize(stacked, bin_edges)[:, 0]
 
         # add bin index to each data point
         data = np.column_stack([stacked, bin_indices])
 
         # create bin mask
-        masks = (data[:, -1, None] == range(1, n_ranges+1)).T
+        masks = (data[:, -1, None] == range(1, n_ranges + 1)).T
 
         # create actual bins
         bins = [data[masks[i]] for i in range(n_ranges)]
@@ -449,7 +460,7 @@ def _adaptive_calibration_error(classifier, y_true, y_prob, y_pred, level, n_ran
             acc[i] = 1 / (bins[i].shape[0]) * np.sum((bins[i][:, 2] == bins[i][:, 3])) if bins[i].shape[0] != 0 else 0
             conf[i] = 1 / (bins[i].shape[0]) * np.sum(bins[i][:, 1]) if bins[i].shape[0] != 0 else 0
             error += abs(acc[i] - conf[i]) if bins[i].shape[0] != 0 else 0
-        
+
         class_error[k] = error
 
-    return (1/(n_classes*n_ranges)) * np.sum(class_error)
+    return (1 / (n_classes * n_ranges)) * np.sum(class_error)
diff --git a/hiclass/probability_combiner/ArithmeticMeanCombiner.py b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
index 546a5e91..591eaaa5 100644
--- a/hiclass/probability_combiner/ArithmeticMeanCombiner.py
+++ b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
@@ -1,12 +1,11 @@
 import numpy as np
-from networkx.exception import NetworkXError
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
-from collections import defaultdict
+
 
 class ArithmeticMeanCombiner(ProbabilityCombiner):
     def combine(self, proba):
         '''Combine probabilities of each level with probabilities of previous levels.
-        
+
         Calculate the arithmetic mean of node probabilities and the probabilities of its predecessors.
         '''
         res = [proba[0]]
@@ -20,12 +19,11 @@ def combine(self, proba):
             for node in predecessors.keys():
                 index = self.classifier.class_to_index_mapping_[level][node]
                 # find indices of all predecessors
-                predecessor_indices = [self.classifier.class_to_index_mapping_[level-1][predecessor] for predecessor in predecessors[node]]
+                predecessor_indices = [self.classifier.class_to_index_mapping_[level - 1][predecessor] for predecessor in predecessors[node]]
                 # combine probabilities of all predecessors
-                predecessors_combined_prob = np.sum([sums[level-1][:, pre_index] for pre_index in predecessor_indices], axis=0)
+                predecessors_combined_prob = np.sum([sums[level - 1][:, pre_index] for pre_index in predecessor_indices], axis=0)
                 level_sum[:, index] += proba[level][:, index] + predecessors_combined_prob
-                level_probs[:, index] = level_sum[:, index] / (level+1)
-
+                level_probs[:, index] = level_sum[:, index] / (level + 1)
 
             res.append(level_probs)
             sums.append(level_sum)
diff --git a/hiclass/probability_combiner/GeometricMeanCombiner.py b/hiclass/probability_combiner/GeometricMeanCombiner.py
index b22a13cf..69b66f18 100644
--- a/hiclass/probability_combiner/GeometricMeanCombiner.py
+++ b/hiclass/probability_combiner/GeometricMeanCombiner.py
@@ -1,12 +1,11 @@
 import numpy as np
-from networkx.exception import NetworkXError
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
-from collections import defaultdict
+
 
 class GeometricMeanCombiner(ProbabilityCombiner):
     def combine(self, proba):
         '''Combine probabilities of each level with probabilities of previous levels.
-        
+
         Calculate the geometric mean of node probabilities and the probabilities of its predecessors.
         '''
         res = [proba[0]]
@@ -20,13 +19,13 @@ def combine(self, proba):
             for node in predecessors.keys():
                 index = self.classifier.class_to_index_mapping_[level][node]
                 # find indices of all predecessors
-                predecessor_indices = [self.classifier.class_to_index_mapping_[level-1][predecessor] for predecessor in predecessors[node]]
+                predecessor_indices = [self.classifier.class_to_index_mapping_[level - 1][predecessor] for predecessor in predecessors[node]]
                 # combine probabilities of all predecessors
-                predecessors_combined_log_prob = np.log(np.sum([np.exp(log_sum[level-1][:, pre_index]) for pre_index in predecessor_indices], axis=0))
+                predecessors_combined_log_prob = np.log(np.sum([np.exp(log_sum[level - 1][:, pre_index]) for pre_index in predecessor_indices], axis=0))
 
                 level_log_sum[:, index] += (np.log(proba[level][:, index]) + predecessors_combined_log_prob)
-                level_probs[:, index] = np.exp(level_log_sum[:, index] / (level+1))
-            
+                level_probs[:, index] = np.exp(level_log_sum[:, index] / (level + 1))
+
             log_sum.append(level_log_sum)
             res.append(level_probs)
         return self._normalize(res) if self.normalize else res
diff --git a/hiclass/probability_combiner/MultiplyCombiner.py b/hiclass/probability_combiner/MultiplyCombiner.py
index fbcb692f..851363bb 100644
--- a/hiclass/probability_combiner/MultiplyCombiner.py
+++ b/hiclass/probability_combiner/MultiplyCombiner.py
@@ -1,12 +1,11 @@
 import numpy as np
-from networkx.exception import NetworkXError
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
-from collections import defaultdict
+
 
 class MultiplyCombiner(ProbabilityCombiner):
     def combine(self, proba):
         '''Combine probabilities of each level with probabilities of previous levels.
-        
+
         Multiply node probabilities with the probabilities of its predecessors.
         '''
         res = [proba[0]]
@@ -18,9 +17,9 @@ def combine(self, proba):
             for node in predecessors.keys():
                 index = self.classifier.class_to_index_mapping_[level][node]
                 # find indices of all predecessors
-                predecessor_indices = [self.classifier.class_to_index_mapping_[level-1][predecessor] for predecessor in predecessors[node]]
+                predecessor_indices = [self.classifier.class_to_index_mapping_[level - 1][predecessor] for predecessor in predecessors[node]]
                 # combine probabilities of all predecessors
-                predecessors_combined_prob = np.sum([res[level-1][:, pre_index] for pre_index in predecessor_indices], axis=0)
+                predecessors_combined_prob = np.sum([res[level - 1][:, pre_index] for pre_index in predecessor_indices], axis=0)
                 level_probs[:, index] = predecessors_combined_prob * proba[level][:, index]
 
             res.append(level_probs)
diff --git a/hiclass/probability_combiner/ProbabilityCombiner.py b/hiclass/probability_combiner/ProbabilityCombiner.py
index 59923625..3c12d8ec 100644
--- a/hiclass/probability_combiner/ProbabilityCombiner.py
+++ b/hiclass/probability_combiner/ProbabilityCombiner.py
@@ -4,6 +4,7 @@
 from collections import defaultdict
 from networkx.exception import NetworkXError
 
+
 class ProbabilityCombiner(abc.ABC):
 
     def __init__(self, classifier, normalize=True) -> None:
@@ -13,13 +14,13 @@ def __init__(self, classifier, normalize=True) -> None:
     @abc.abstractmethod
     def combine(self, proba: List[np.ndarray]) -> List[np.ndarray]:
         ...
-    
+
     def _normalize(self, proba):
         return [
-            np.nan_to_num(level_probabilities / level_probabilities.sum(axis=1, keepdims=True)) 
-                for level_probabilities in proba
+            np.nan_to_num(level_probabilities / level_probabilities.sum(axis=1, keepdims=True))
+            for level_probabilities in proba
         ]
-    
+
     def _find_predecessors(self, level):
         predecessors = defaultdict(list)
         for node in self.classifier.global_classes_[level]:

From f3272e4495ba8d36e68d69978afd8f11d3ed1334 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Tue, 9 Apr 2024 15:54:27 +0200
Subject: [PATCH 45/65] add docstrings, add tests for calibration metrics

---
 hiclass/LocalClassifierPerLevel.py            |  18 ++
 hiclass/LocalClassifierPerParentNode.py       |  18 ++
 hiclass/Pipeline.py                           |   8 +-
 hiclass/_calibration/BinaryCalibrator.py      |   4 +-
 hiclass/_calibration/Calibrator.py            |   4 +-
 hiclass/metrics.py                            | 217 +++++++++++++++---
 .../ArithmeticMeanCombiner.py                 |   7 +-
 .../GeometricMeanCombiner.py                  |   7 +-
 .../probability_combiner/MultiplyCombiner.py  |   7 +-
 .../ProbabilityCombiner.py                    |   4 +
 hiclass/probability_combiner/__init__.py      |   2 +
 tests/test_metrics.py                         | 189 ++++++++++++++-
 12 files changed, 430 insertions(+), 55 deletions(-)

diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index d6ec6d83..e72fb1d2 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -196,6 +196,24 @@ def predict(self, X):
         return y
 
     def predict_proba(self, X):
+        """
+        Predict class probabilities for the given data.
+
+        Hierarchical labels are returned.
+        If return_all_probabilities=True: Returns the probabilities for each level.
+        Else: Returns the probabilities for the lowest level.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            The input samples. Internally, its dtype will be converted
+            to ``dtype=np.float32``. If a sparse matrix is provided, it will be
+            converted into a sparse ``csr_matrix``.
+        Returns
+        -------
+        T : ndarray of shape (n_samples,n_classes) or List[ndarray(n_samples,n_classes)]
+            The predicted probabilities of the lowest levels or of all levels.
+        """
         # Check if fit has been called
         check_is_fitted(self)
 
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index 90931631..0cde2db3 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -188,6 +188,24 @@ def predict(self, X):
         return y
 
     def predict_proba(self, X):
+        """
+        Predict class probabilities for the given data.
+
+        Hierarchical labels are returned.
+        If return_all_probabilities=True: Returns the probabilities for each level.
+        Else: Returns the probabilities for the lowest level.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            The input samples. Internally, its dtype will be converted
+            to ``dtype=np.float32``. If a sparse matrix is provided, it will be
+            converted into a sparse ``csr_matrix``.
+        Returns
+        -------
+        T : ndarray of shape (n_samples,n_classes) or List[ndarray(n_samples,n_classes)]
+            The predicted probabilities of the lowest levels or of all levels.
+        """
         # Check if fit has been called
         check_is_fitted(self)
 
diff --git a/hiclass/Pipeline.py b/hiclass/Pipeline.py
index 828f184e..f1202b79 100644
--- a/hiclass/Pipeline.py
+++ b/hiclass/Pipeline.py
@@ -1,14 +1,16 @@
+"""Custom Pipeline class that supports the `calibrate` method."""
 from sklearn.pipeline import Pipeline as skPipeline
 
 
 class Pipeline(skPipeline):
+    """Custom Pipeline class that supports the `calibrate` method."""
+
     def __init__(self, steps, **kwargs):
+        """Create Pipeline object."""
         super().__init__(steps, **kwargs)
 
     def calibrate(self, X, y, **params):
-        """Transform the data and apply `calibrate` with the final estimator.
-
-        """
+        """Transform the data and apply `calibrate` with the final estimator."""
         Xt = X
         for _, name, transform in self._iter(with_final=False):
             Xt = transform.transform(Xt)
diff --git a/hiclass/_calibration/BinaryCalibrator.py b/hiclass/_calibration/BinaryCalibrator.py
index 2f24a9e8..78f4ac28 100644
--- a/hiclass/_calibration/BinaryCalibrator.py
+++ b/hiclass/_calibration/BinaryCalibrator.py
@@ -12,7 +12,5 @@ def predict_proba(self, scores, X=None):  # pragma: no cover
         ...
 
     def __sklearn_is_fitted__(self):
-        """
-        Check fitted status and return a Boolean value.
-        """
+        """Check fitted status and return a Boolean value."""
         return hasattr(self, "_is_fitted") and self._is_fitted
diff --git a/hiclass/_calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
index 40e6440b..8e82f4ea 100644
--- a/hiclass/_calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -115,7 +115,5 @@ def _create_calibrator(self, name, params):
             return _BetaCalibrator()
 
     def __sklearn_is_fitted__(self):
-        """
-        Check fitted status and return a Boolean value.
-        """
+        """Check fitted status and return a Boolean value."""
         return hasattr(self, "_is_fitted") and self._is_fitted
diff --git a/hiclass/metrics.py b/hiclass/metrics.py
index e2735288..47e45a33 100644
--- a/hiclass/metrics.py
+++ b/hiclass/metrics.py
@@ -1,5 +1,6 @@
 """Helper functions to compute hierarchical evaluation metrics."""
 
+from typing import Union, List
 import numpy as np
 from sklearn.utils import check_array
 from sklearn.metrics import log_loss as sk_log_loss
@@ -253,7 +254,7 @@ def _compute_macro(y_true: np.ndarray, y_pred: np.ndarray, _micro_function):
     return overall_sum / len(y_true)
 
 
-def _prepare_data(classifier, y_true, y_prob, level, y_pred=None):
+def _prepare_data(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, level: int, y_pred: np.ndarray = None):
     classifier_classes = np.array(classifier.classes_[level]).astype("str")
     y_true = make_leveled(y_true)
     y_true = classifier._disambiguate(y_true)
@@ -270,16 +271,18 @@ def _prepare_data(classifier, y_true, y_prob, level, y_pred=None):
     # add labels not seen in the training process
     new_labels = np.sort(np.union1d(unique_labels, classifier_classes))
 
-    # add empty columns to y_prob
-    new_y_prob = np.zeros((y_prob[level].shape[0], len(new_labels)), dtype=np.float32)
+    new_y_prob = np.zeros((y_prob.shape[0], len(new_labels)), dtype=np.float32)
     for idx, label in enumerate(new_labels):
         if label in classifier_classes:
             old_idx = np.where(classifier_classes == label)[0][0]
-            new_y_prob[:, idx] = y_prob[level][:, old_idx]
+            new_y_prob[:, idx] = y_prob[:, old_idx]
 
     return y_true, y_pred, new_labels, new_y_prob
 
 
+_calibration_aggregations = ["average", "sum", "None"]
+
+
 def _aggregate_scores(scores, agg):
     if agg == 'average':
         return np.mean(scores)
@@ -289,39 +292,193 @@ def _aggregate_scores(scores, agg):
         return scores
 
 
-def multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, agg='average'):
-    scores = []
-    for level in range(make_leveled(y_true).shape[1]):
-        scores.append(_multiclass_brier_score(classifier, y_true, y_prob, level))
-    return _aggregate_scores(scores, agg)
+def _validate_args(agg, y_prob, level):
+    if agg and agg not in _calibration_aggregations:
+        raise ValueError(f"{agg} is not a valid aggregation function.")
+    if isinstance(y_prob, list) and len(y_prob) == 0:
+        raise ValueError("y_prob is empty.")
+    if (isinstance(y_prob, list) and len(y_prob) == 1 or isinstance(y_prob, np.ndarray)) and level is None:
+        raise ValueError("If y_prob is not a list of probabilities the level must be specified.")
+    if isinstance(y_prob, list) and len(y_prob) == 1:
+        return y_prob[0]
+    return y_prob
+
+
+def multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Union[np.ndarray | List], agg='average', level=None):
+    """Compute the brier score for two or more classes.
+
+    Parameters
+    ----------
+    classifier : HierarchicalClassifier
+        The classifier used.
+    y_true : np.array of shape (n_samples, n_levels)
+        Ground truth (correct) labels.
+    y_prob : np.array of shape (n_samples, n_unique_labels_per_level) or List[np.array((n_samples, n_unique_labels_per_level))]
+        Predicted probabilities.
+    agg: {"average", "sum", None}, str, default="average"
+        This parameter determines the type of averaging performed during the computation if y_prob contains probabilities for multiple levels:
+
+        - `average`: Calculate the average brier score over all levels.
+        - `sum`: Calculate the summed brier score over all levels.
+        - None: Don't aggregate results. Returns a list of brier scores.
+    level : int, default=None
+        Specifies the level of y_prob if y_prob is not a list of numpy arrays.
+    Returns
+    -------
+    brier_score : float or List[float]
+        Brier score of predicted probabilities.
+    """
+    y_prob = _validate_args(agg, y_prob, level)
+    if isinstance(y_prob, list):
+        scores = []
+        for level in range(make_leveled(y_true).shape[1]):
+            scores.append(_multiclass_brier_score(classifier, y_true, y_prob[level], level))
+        return _aggregate_scores(scores, agg)
+    return _multiclass_brier_score(classifier, y_true, y_prob, level)
+
 
+def log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Union[np.ndarray | List], agg='average', level=None):
+    """Compute the log loss of predicted probabilities.
 
-def log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, agg='average'):
-    scores = []
-    for level in range(make_leveled(y_true).shape[1]):
-        scores.append(_log_loss(classifier, y_true, y_prob, level))
-    return _aggregate_scores(scores, agg)
+    Parameters
+    ----------
+    classifier : HierarchicalClassifier
+        The classifier used.
+    y_true : np.array of shape (n_samples, n_levels)
+        Ground truth (correct) labels.
+    y_prob : np.array of shape (n_samples, n_unique_labels_per_level) or List[np.array((n_samples, n_unique_labels_per_level))]
+        Predicted probabilities.
+    agg: {"average", "sum", None}, str, default="average"
+        This parameter determines the type of averaging performed during the computation if y_prob contains probabilities for multiple levels:
+
+        - `average`: Calculate the average brier score over all levels.
+        - `sum`: Calculate the summed brier score over all levels.
+        - None: Don't aggregate results. Returns a list of brier scores.
+    level : int, default=None
+        Specifies the level of y_prob if y_prob is not a list of numpy arrays.
+    Returns
+    -------
+    log_loss : float or List[float]
+        Log loss of predicted probabilities.
+    """
+    y_prob = _validate_args(agg, y_prob, level)
+    if isinstance(y_prob, list):
+        scores = []
+        for level in range(make_leveled(y_true).shape[1]):
+            scores.append(_log_loss(classifier, y_true, y_prob[level], level))
+        return _aggregate_scores(scores, agg)
+    return _multiclass_brier_score(classifier, y_true, y_prob, level)
+
+
+def expected_calibration_error(classifier: HierarchicalClassifier, y_true, y_prob: Union[np.ndarray | List], y_pred, n_bins=10, agg='average', level=None):
+    """Compute the expected calibration error.
+
+    Parameters
+    ----------
+    classifier : HierarchicalClassifier
+        The classifier used.
+    y_true : np.array of shape (n_samples, n_levels)
+        Ground truth (correct) labels.
+    y_prob : np.array of shape (n_samples, n_unique_labels_per_level) or List[np.array((n_samples, n_unique_labels_per_level))]
+        Predicted probabilities.
+    y_pred : np.array of shape (n_samples, n_levels)
+        Predicted labels, as returned by a classifier.
+    n_bins : int, default=10
+        Number of bins to calculate the metric.
+    agg: {"average", "sum", None}, str, default="average"
+        This parameter determines the type of averaging performed during the computation if y_prob contains probabilities for multiple levels:
+
+        - `average`: Calculate the average brier score over all levels.
+        - `sum`: Calculate the summed brier score over all levels.
+        - None: Don't aggregate results. Returns a list of brier scores.
+    level : int, default=None
+        Specifies the level of y_prob if y_prob is not a list of numpy arrays.
+    Returns
+    -------
+    expected_calibration_error : float or List[float]
+        Expected calibration error of predicted probabilities.
+    """
+    y_prob = _validate_args(agg, y_prob, level)
+    if isinstance(y_prob, list):
+        scores = []
+        for level in range(make_leveled(y_true).shape[1]):
+            scores.append(_expected_calibration_error(classifier, y_true, y_prob[level], y_pred, level, n_bins))
+        return _aggregate_scores(scores, agg)
+    return _expected_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins)
 
 
-def expected_calibration_error(classifier: HierarchicalClassifier, y_true, y_prob, y_pred, n_bins=10, agg='average'):
-    scores = []
-    for level in range(make_leveled(y_true).shape[1]):
-        scores.append(_expected_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins))
-    return _aggregate_scores(scores, agg)
+def static_calibration_error(classifier, y_true, y_prob: Union[np.ndarray | List], y_pred, n_bins=10, agg='average', level=None):
+    """Compute the static calibration error.
 
+    Parameters
+    ----------
+    classifier : HierarchicalClassifier
+        The classifier used.
+    y_true : np.array of shape (n_samples, n_levels)
+        Ground truth (correct) labels.
+    y_prob : np.array of shape (n_samples, n_unique_labels_per_level) or List[np.array((n_samples, n_unique_labels_per_level))]
+        Predicted probabilities.
+    y_pred : np.array of shape (n_samples, n_levels)
+        Predicted labels, as returned by a classifier.
+    n_bins : int, default=10
+        Number of bins to calculate the metric.
+    agg: {"average", "sum", None}, str, default="average"
+        This parameter determines the type of averaging performed during the computation if y_prob contains probabilities for multiple levels:
+
+        - `average`: Calculate the average brier score over all levels.
+        - `sum`: Calculate the summed brier score over all levels.
+        - None: Don't aggregate results. Returns a list of brier scores.
+    level : int, default=None
+        Specifies the level of y_prob if y_prob is not a list of numpy arrays.
+    Returns
+    -------
+    static_calibration_error : float or List[float]
+        Static calibration error of predicted probabilities.
+    """
+    y_prob = _validate_args(agg, y_prob, level)
+    if isinstance(y_prob, list):
+        scores = []
+        for level in range(make_leveled(y_true).shape[1]):
+            scores.append(_static_calibration_error(classifier, y_true, y_prob[level], y_pred, level, n_bins=n_bins))
+        return _aggregate_scores(scores, agg)
+    return _static_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins=n_bins)
 
-def statistical_calibration_error(classifier, y_true, y_prob, y_pred, n_bins=10, agg='average'):
-    scores = []
-    for level in range(make_leveled(y_true).shape[1]):
-        scores.append(_statistical_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins=n_bins))
-    return _aggregate_scores(scores, agg)
 
+def adaptive_calibration_error(classifier, y_true, y_prob: Union[np.ndarray | List], y_pred, n_ranges=10, agg='average', level=None):
+    """Compute the adaptive calibration error.
 
-def adaptive_calibration_error(classifier, y_true, y_prob, y_pred, n_ranges=10, agg='average'):
-    scores = []
-    for level in range(make_leveled(y_true).shape[1]):
-        scores.append(_adaptive_calibration_error(classifier, y_true, y_prob, y_pred, level, n_ranges=n_ranges))
-    return _aggregate_scores(scores, agg)
+    Parameters
+    ----------
+    classifier : HierarchicalClassifier
+        The classifier used.
+    y_true : np.array of shape (n_samples, n_levels)
+        Ground truth (correct) labels.
+    y_prob : np.array of shape (n_samples, n_unique_labels_per_level) or List[np.array((n_samples, n_unique_labels_per_level))]
+        Predicted probabilities.
+    y_pred : np.array of shape (n_samples, n_levels)
+        Predicted labels, as returned by a classifier.
+    n_ranges : int, default=10
+        Number of ranges to calculate the metric.
+    agg: {"average", "sum", None}, str, default="average"
+        This parameter determines the type of averaging performed during the computation if y_prob contains probabilities for multiple levels:
+
+        - `average`: Calculate the average brier score over all levels.
+        - `sum`: Calculate the summed brier score over all levels.
+        - None: Don't aggregate results. Returns a list of brier scores.
+    level : int, default=None
+        Specifies the level of y_prob if y_prob is not a list of numpy arrays.
+    Returns
+    -------
+    adaptive_calibration_error : float or List[float]
+        Adaptive calibration error of predicted probabilities.
+    """
+    y_prob = _validate_args(agg, y_prob, level)
+    if isinstance(y_prob, list):
+        scores = []
+        for level in range(make_leveled(y_true).shape[1]):
+            scores.append(_adaptive_calibration_error(classifier, y_true, y_prob[level], y_pred, level, n_ranges=n_ranges))
+        return _aggregate_scores(scores, agg)
+    return _adaptive_calibration_error(classifier, y_true, y_prob, y_pred, level, n_ranges=n_ranges)
 
 
 def _multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, level: int):
@@ -374,7 +531,7 @@ def _expected_calibration_error(classifier: HierarchicalClassifier, y_true, y_pr
     return ece
 
 
-def _statistical_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins=10):
+def _static_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins=10):
     y_true, y_pred, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level, y_pred)
 
     n_samples, n_classes = y_prob.shape
diff --git a/hiclass/probability_combiner/ArithmeticMeanCombiner.py b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
index 591eaaa5..7ec1795c 100644
--- a/hiclass/probability_combiner/ArithmeticMeanCombiner.py
+++ b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
@@ -1,13 +1,16 @@
+"""Defines the ArithmeticMeanCombiner."""
 import numpy as np
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
 
 
 class ArithmeticMeanCombiner(ProbabilityCombiner):
+    """Combine probabilities of multiple levels by taking their arithmetic mean."""
+
     def combine(self, proba):
-        '''Combine probabilities of each level with probabilities of previous levels.
+        """Combine probabilities of each level with probabilities of previous levels.
 
         Calculate the arithmetic mean of node probabilities and the probabilities of its predecessors.
-        '''
+        """
         res = [proba[0]]
         sums = [proba[0]]
         for level in range(1, self.classifier.max_levels_):
diff --git a/hiclass/probability_combiner/GeometricMeanCombiner.py b/hiclass/probability_combiner/GeometricMeanCombiner.py
index 69b66f18..1f449ca0 100644
--- a/hiclass/probability_combiner/GeometricMeanCombiner.py
+++ b/hiclass/probability_combiner/GeometricMeanCombiner.py
@@ -1,13 +1,16 @@
+"""Defines the GeometricMeanCombiner."""
 import numpy as np
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
 
 
 class GeometricMeanCombiner(ProbabilityCombiner):
+    """Combine probabilities of multiple levels by taking their geometric mean."""
+
     def combine(self, proba):
-        '''Combine probabilities of each level with probabilities of previous levels.
+        """Combine probabilities of each level with probabilities of previous levels.
 
         Calculate the geometric mean of node probabilities and the probabilities of its predecessors.
-        '''
+        """
         res = [proba[0]]
         log_sum = [np.log(proba[0])]
         for level in range(1, self.classifier.max_levels_):
diff --git a/hiclass/probability_combiner/MultiplyCombiner.py b/hiclass/probability_combiner/MultiplyCombiner.py
index 851363bb..a06812fc 100644
--- a/hiclass/probability_combiner/MultiplyCombiner.py
+++ b/hiclass/probability_combiner/MultiplyCombiner.py
@@ -1,13 +1,16 @@
+"""Defines the MultiplyCombiner."""
 import numpy as np
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
 
 
 class MultiplyCombiner(ProbabilityCombiner):
+    """Combine probabilities of multiple levels by multiplication."""
+
     def combine(self, proba):
-        '''Combine probabilities of each level with probabilities of previous levels.
+        """Combine probabilities of each level with probabilities of previous levels.
 
         Multiply node probabilities with the probabilities of its predecessors.
-        '''
+        """
         res = [proba[0]]
         for level in range(1, self.classifier.max_levels_):
             level_probs = np.zeros_like(proba[level])
diff --git a/hiclass/probability_combiner/ProbabilityCombiner.py b/hiclass/probability_combiner/ProbabilityCombiner.py
index 3c12d8ec..d4edbb4b 100644
--- a/hiclass/probability_combiner/ProbabilityCombiner.py
+++ b/hiclass/probability_combiner/ProbabilityCombiner.py
@@ -1,3 +1,4 @@
+"""Abstract class defining the structure of a probability combiner."""
 import abc
 import numpy as np
 from typing import List
@@ -6,13 +7,16 @@
 
 
 class ProbabilityCombiner(abc.ABC):
+    """Abstract class defining the structure of a probability combiner."""
 
     def __init__(self, classifier, normalize=True) -> None:
+        """Initialize probability combiner object."""
         self.classifier = classifier
         self.normalize = normalize
 
     @abc.abstractmethod
     def combine(self, proba: List[np.ndarray]) -> List[np.ndarray]:
+        """Combine probabilities over multiple levels."""
         ...
 
     def _normalize(self, proba):
diff --git a/hiclass/probability_combiner/__init__.py b/hiclass/probability_combiner/__init__.py
index 540d35ba..1d7e069d 100644
--- a/hiclass/probability_combiner/__init__.py
+++ b/hiclass/probability_combiner/__init__.py
@@ -1,3 +1,5 @@
+"""Init the probability combiner module."""
+
 from .MultiplyCombiner import MultiplyCombiner
 from .ArithmeticMeanCombiner import ArithmeticMeanCombiner
 from .GeometricMeanCombiner import GeometricMeanCombiner
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index fd3d0686..785adbb3 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -13,8 +13,13 @@
     _multiclass_brier_score, 
     _log_loss,
     _expected_calibration_error,
-    _statistical_calibration_error,
-    _adaptive_calibration_error
+    _static_calibration_error,
+    _adaptive_calibration_error,
+    multiclass_brier_score,
+    log_loss,
+    expected_calibration_error,
+    static_calibration_error,
+    adaptive_calibration_error
 )
 
 
@@ -369,6 +374,23 @@ def uncertainty_data():
 
     return prob, y_pred, y_true
 
+@pytest.fixture
+def uncertainty_data_multi_level():
+    prob = [np.array([[0.88, 0.06, 0.06],
+                     [0.22, 0.48, 0.30],
+                     [0.33, 0.33, 0.34]]),
+
+            np.array([[0.88, 0.06, 0.06],
+                     [0.22, 0.48, 0.30],
+                     [0.33, 0.33, 0.34]])]
+
+    assert_array_equal(np.sum(prob[0], axis=1), np.ones(len(prob[0])))
+
+    y_pred = np.array([[0, 3], [1, 4], [2, 5]])
+    y_true = np.array([[0, 3], [2, 5], [0, 4]])
+
+    return prob, y_pred, y_true
+
 def test_local_brier_score(uncertainty_data):
     prob, _, y_true = uncertainty_data
     obj = HierarchicalClassifier()
@@ -377,9 +399,39 @@ def test_local_brier_score(uncertainty_data):
     classifier.classes_ = [[0, 1, 2]]
     classifier.separator_ = "::HiClass::Separator::"
     
-    brier_score = _multiclass_brier_score(classifier, y_true, prob, level=0)
+    brier_score = _multiclass_brier_score(classifier, y_true, prob[0], level=0)
     assert math.isclose(brier_score, 0.34852, abs_tol=1e-4)
 
+def test_brier_score_multi_level(uncertainty_data_multi_level):
+    prob, _, y_true = uncertainty_data_multi_level
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = [[0, 1, 2], [3, 4, 5]]
+    classifier.separator_ = "::HiClass::Separator::"
+    brier_score_avg = multiclass_brier_score(classifier, y_true, prob, agg="average")
+    brier_score_sum = multiclass_brier_score(classifier, y_true, prob, agg="sum")
+    brier_score_per_level = multiclass_brier_score(classifier, y_true, prob, agg=None)
+    assert math.isclose(brier_score_avg, 0.48793, abs_tol=1e-4)
+    assert math.isclose(brier_score_sum, 0.97586, abs_tol=1e-4)
+    assert math.isclose(brier_score_per_level[0], 0.48793, abs_tol=1e-4)
+    assert math.isclose(brier_score_per_level[1], 0.48793, abs_tol=1e-4)
+
+def test_brier_score_single_level(uncertainty_data_multi_level):
+    prob, _, y_true = uncertainty_data_multi_level
+    prob = prob[1]
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = [[0, 1, 2], [3, 4, 5]]
+    classifier.separator_ = "::HiClass::Separator::"
+
+    brier_score_1 = multiclass_brier_score(classifier, y_true, prob, level=1)
+    brier_score_2 = multiclass_brier_score(classifier, y_true, [prob], level=1)
+    assert math.isclose(brier_score_1, 0.48793, abs_tol=1e-4)
+    assert math.isclose(brier_score_2, 0.48793, abs_tol=1e-4)
+
+
 def test_local_log_loss(uncertainty_data):
     prob, _, y_true = uncertainty_data
     obj = HierarchicalClassifier()
@@ -388,10 +440,39 @@ def test_local_log_loss(uncertainty_data):
     classifier.classes_ = [[0, 1, 2]]
     classifier.separator_ = "::HiClass::Separator::"
 
-    log_loss = _log_loss(classifier, y_true, prob, level=0)
+    log_loss = _log_loss(classifier, y_true, prob[0], level=0)
     assert math.isclose(log_loss, 0.61790, abs_tol=1e-4)
 
-def test_expected_calibration_error(uncertainty_data):
+def test_log_loss_multi_level(uncertainty_data_multi_level):
+    prob, _, y_true = uncertainty_data_multi_level
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = [[0, 1, 2], [3, 4, 5]]
+    classifier.separator_ = "::HiClass::Separator::"
+    log_loss_avg = log_loss(classifier, y_true, prob, agg="average")
+    log_loss_sum = log_loss(classifier, y_true, prob, agg="sum")
+    log_loss_per_level = log_loss(classifier, y_true, prob, agg=None)
+    assert math.isclose(log_loss_avg, 0.81348, abs_tol=1e-4)
+    assert math.isclose(log_loss_sum, 1.62697, abs_tol=1e-4)
+    assert math.isclose(log_loss_per_level[0], 0.81348, abs_tol=1e-4)
+    assert math.isclose(log_loss_per_level[1], 0.81348, abs_tol=1e-4)
+
+def test_log_loss_single_level(uncertainty_data_multi_level):
+    prob, _, y_true = uncertainty_data_multi_level
+    prob = prob[1]
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = [[0, 1, 2], [3, 4, 5]]
+    classifier.separator_ = "::HiClass::Separator::"
+
+    log_loss_1 = log_loss(classifier, y_true, prob, level=1)
+    log_loss_2 = log_loss(classifier, y_true, [prob], level=1)
+    assert math.isclose(log_loss_1, 0.48793, abs_tol=1e-4)
+    assert math.isclose(log_loss_2, 0.48793, abs_tol=1e-4)
+
+def test_local_expected_calibration_error(uncertainty_data):
     prob, y_pred, y_true = uncertainty_data
     obj = HierarchicalClassifier()
     classifier = Mock(spec=obj)
@@ -399,10 +480,38 @@ def test_expected_calibration_error(uncertainty_data):
     classifier.classes_ = [[0, 1, 2]]
     classifier.separator_ = "::HiClass::Separator::"
 
-    ece = _expected_calibration_error(classifier, y_true, prob, y_pred, level=0, n_bins=3)
+    ece = _expected_calibration_error(classifier, y_true, prob[0], y_pred, level=0, n_bins=3)
     assert math.isclose(ece, 0.118, abs_tol=1e-4)
 
-def test_statistical_calibration_error(uncertainty_data):
+def test_expected_calibration_error_multi_level(uncertainty_data_multi_level):
+    prob, y_pred, y_true = uncertainty_data_multi_level
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = [[0, 1, 2], [3, 4, 5]]
+    classifier.separator_ = "::HiClass::Separator::"
+    ece_avg = expected_calibration_error(classifier, y_true, prob, y_pred, agg="average")
+    ece_sum = expected_calibration_error(classifier, y_true, prob, y_pred, agg="sum")
+    ece_per_level = expected_calibration_error(classifier, y_true, prob, y_pred, agg=None)
+    assert math.isclose(ece_avg, 0.31333, abs_tol=1e-4)
+    assert math.isclose(ece_sum, 0.62666, abs_tol=1e-4)
+    assert math.isclose(ece_per_level[0], 0.31333, abs_tol=1e-4)
+    assert math.isclose(ece_per_level[1], 0.31333, abs_tol=1e-4)
+
+def test_expected_calibration_error_single_level(uncertainty_data_multi_level):
+    prob, y_pred, y_true = uncertainty_data_multi_level
+    prob = prob[1]
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = [[0, 1, 2], [3, 4, 5]]
+    classifier.separator_ = "::HiClass::Separator::"
+    ece_1 = expected_calibration_error(classifier, y_true, prob, y_pred, level=1)
+    ece_2 = expected_calibration_error(classifier, y_true, [prob], y_pred, level=1)
+    assert math.isclose(ece_1, 0.31333, abs_tol=1e-4)
+    assert math.isclose(ece_2, 0.31333, abs_tol=1e-4)
+
+def test_local_static_calibration_error(uncertainty_data):
     prob, y_pred, y_true = uncertainty_data
     obj = HierarchicalClassifier()
     classifier = Mock(spec=obj)
@@ -410,10 +519,40 @@ def test_statistical_calibration_error(uncertainty_data):
     classifier.classes_ = [[0, 1, 2]]
     classifier.separator_ = "::HiClass::Separator::"
 
-    sce = _statistical_calibration_error(classifier, y_true, prob, y_pred, level=0, n_bins=3)
+    sce = _static_calibration_error(classifier, y_true, prob[0], y_pred, level=0, n_bins=3)
     assert math.isclose(sce, 0.3889, abs_tol=1e-3)
 
-def test_adaptive_calibration_error(uncertainty_data):
+def test_static_calibration_error_multi_level(uncertainty_data_multi_level):
+    prob, y_pred, y_true = uncertainty_data_multi_level
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = [[0, 1, 2], [3, 4, 5]]
+    classifier.separator_ = "::HiClass::Separator::"
+
+    sce_avg = static_calibration_error(classifier, y_true, prob, y_pred, agg="average")
+    sce_sum = static_calibration_error(classifier, y_true, prob, y_pred, agg="sum")
+    sce_per_level = static_calibration_error(classifier, y_true, prob, y_pred, agg=None)
+    assert math.isclose(sce_avg, 0.44444, abs_tol=1e-4)
+    assert math.isclose(sce_sum, 0.88888, abs_tol=1e-4)
+    assert math.isclose(sce_per_level[0], 0.44444, abs_tol=1e-4)
+    assert math.isclose(sce_per_level[1], 0.44444, abs_tol=1e-4)
+
+
+def test_static_calibration_error_single_level(uncertainty_data_multi_level):
+    prob, y_pred, y_true = uncertainty_data_multi_level
+    prob = prob[1]
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = [[0, 1, 2], [3, 4, 5]]
+    classifier.separator_ = "::HiClass::Separator::"
+    sce_1 = static_calibration_error(classifier, y_true, prob, y_pred, level=1)
+    sce_2 = static_calibration_error(classifier, y_true, [prob], y_pred, level=1)
+    assert math.isclose(sce_1, 0.44444, abs_tol=1e-4)
+    assert math.isclose(sce_2, 0.44444, abs_tol=1e-4)
+
+def test_local_adaptive_calibration_error(uncertainty_data):
     prob, y_pred, y_true = uncertainty_data
     obj = HierarchicalClassifier()
     classifier = Mock(spec=obj)
@@ -421,5 +560,35 @@ def test_adaptive_calibration_error(uncertainty_data):
     classifier.classes_ = [[0, 1, 2]]
     classifier.separator_ = "::HiClass::Separator::"
 
-    ace = _adaptive_calibration_error(classifier, y_true, prob, y_pred, level=0, n_ranges=3)
+    ace = _adaptive_calibration_error(classifier, y_true, prob[0], y_pred, level=0, n_ranges=3)
     assert math.isclose(ace, 0.44, abs_tol=1e-3)
+
+def test_adaptive_calibration_error_multi_level(uncertainty_data_multi_level):
+    prob, y_pred, y_true = uncertainty_data_multi_level
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = [[0, 1, 2], [3, 4, 5]]
+    classifier.separator_ = "::HiClass::Separator::"
+
+    ace_avg = adaptive_calibration_error(classifier, y_true, prob, y_pred, agg="average")
+    ace_sum = adaptive_calibration_error(classifier, y_true, prob, y_pred, agg="sum")
+    ace_per_level = adaptive_calibration_error(classifier, y_true, prob, y_pred, agg=None)
+    assert math.isclose(ace_avg, 0.13333, abs_tol=1e-4)
+    assert math.isclose(ace_sum, 0.26666, abs_tol=1e-4)
+    assert math.isclose(ace_per_level[0], 0.13333, abs_tol=1e-4)
+    assert math.isclose(ace_per_level[1], 0.13333, abs_tol=1e-4)
+
+def test_adaptive_calibration_error_single_level(uncertainty_data_multi_level):
+    prob, y_pred, y_true = uncertainty_data_multi_level
+    prob = prob[1]
+    obj = HierarchicalClassifier()
+    classifier = Mock(spec=obj)
+    classifier._disambiguate = obj._disambiguate
+    classifier.classes_ = [[0, 1, 2], [3, 4, 5]]
+    classifier.separator_ = "::HiClass::Separator::"
+
+    ace_1 = adaptive_calibration_error(classifier, y_true, prob, y_pred, level=1)
+    ace_2 = adaptive_calibration_error(classifier, y_true, [prob], y_pred, level=1)
+    assert math.isclose(ace_1, 0.13333, abs_tol=1e-4)
+    assert math.isclose(ace_2, 0.13333, abs_tol=1e-4)
\ No newline at end of file

From 89dc3fa717090893a4e0b6e2d23298750f069afe Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Tue, 9 Apr 2024 16:45:09 +0200
Subject: [PATCH 46/65] add type hints

---
 hiclass/_calibration/BetaCalibrator.py        |  4 ++--
 hiclass/_calibration/BinaryCalibrator.py      |  5 +++--
 hiclass/_calibration/Calibrator.py            |  8 ++++----
 hiclass/_calibration/IsotonicRegression.py    |  5 +++--
 hiclass/_calibration/PlattScaling.py          |  5 +++--
 hiclass/_calibration/VennAbersCalibrator.py   | 13 ++++++------
 hiclass/_calibration/calibration_utils.py     |  4 +++-
 hiclass/metrics.py                            | 20 +++++++++----------
 .../ArithmeticMeanCombiner.py                 |  3 ++-
 .../GeometricMeanCombiner.py                  |  3 ++-
 .../probability_combiner/MultiplyCombiner.py  |  3 ++-
 .../ProbabilityCombiner.py                    |  7 ++++---
 12 files changed, 45 insertions(+), 35 deletions(-)

diff --git a/hiclass/_calibration/BetaCalibrator.py b/hiclass/_calibration/BetaCalibrator.py
index e2d91469..4e94d3a3 100644
--- a/hiclass/_calibration/BetaCalibrator.py
+++ b/hiclass/_calibration/BetaCalibrator.py
@@ -11,7 +11,7 @@ def __init__(self) -> None:
         super().__init__()
         self.skip_calibration = False
 
-    def fit(self, y, scores, X=None):
+    def fit(self, y: np.ndarray, scores: np.ndarray, X: np.ndarray = None):
         unique_labels = len(np.unique(y))
         if unique_labels < 2:
             self.skip_calibration = True
@@ -30,7 +30,7 @@ def fit(self, y, scores, X=None):
         self._is_fitted = True
         return self
 
-    def predict_proba(self, scores, X=None):
+    def predict_proba(self, scores: np.ndarray, X: np.ndarray = None):
         check_is_fitted(self)
         if self.skip_calibration:
             return scores
diff --git a/hiclass/_calibration/BinaryCalibrator.py b/hiclass/_calibration/BinaryCalibrator.py
index 78f4ac28..29fb7c92 100644
--- a/hiclass/_calibration/BinaryCalibrator.py
+++ b/hiclass/_calibration/BinaryCalibrator.py
@@ -1,14 +1,15 @@
 import abc
+import numpy as np
 
 
 class _BinaryCalibrator(abc.ABC):
 
     @abc.abstractmethod
-    def fit(self, y, scores, X=None):  # pragma: no cover
+    def fit(self, y: np.ndarray, scores: np.ndarray, X: np.ndarray = None):  # pragma: no cover
         ...
 
     @abc.abstractmethod
-    def predict_proba(self, scores, X=None):  # pragma: no cover
+    def predict_proba(self, scores: np.ndarray, X: np.ndarray = None):  # pragma: no cover
         ...
 
     def __sklearn_is_fitted__(self):
diff --git a/hiclass/_calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
index 8e82f4ea..895174f1 100644
--- a/hiclass/_calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -13,7 +13,7 @@ class _Calibrator(BaseEstimator):
     available_methods = ["ivap", "cvap", "sigmoid", "isotonic", "beta"]
     _multiclass_methods = ["cvap"]
 
-    def __init__(self, estimator, method="ivap", **method_params) -> None:
+    def __init__(self, estimator: BaseEstimator, method: str = "ivap", **method_params) -> None:
         assert callable(getattr(estimator, 'predict_proba', None))
         self.estimator = estimator
         self.method_params = method_params
@@ -24,7 +24,7 @@ def __init__(self, estimator, method="ivap", **method_params) -> None:
             raise ValueError(f"{method} is not a valid calibration method.")
         self.method = method
 
-    def fit(self, X, y):
+    def fit(self, X: np.ndarray, y: np.ndarray):
         """
         Fit a calibrator.
 
@@ -77,7 +77,7 @@ def fit(self, X, y):
         self._is_fitted = True
         return self
 
-    def predict_proba(self, X):
+    def predict_proba(self, X: np.ndarray):
         test_scores = self.estimator.predict_proba(X)
 
         if self.multiclass:
@@ -102,7 +102,7 @@ def predict_proba(self, X):
 
         return probabilities
 
-    def _create_calibrator(self, name, params):
+    def _create_calibrator(self, name: str, params):
         if name == "ivap":
             return _InductiveVennAbersCalibrator(**params)
         elif name == "cvap":
diff --git a/hiclass/_calibration/IsotonicRegression.py b/hiclass/_calibration/IsotonicRegression.py
index c68009ea..f6c159b2 100644
--- a/hiclass/_calibration/IsotonicRegression.py
+++ b/hiclass/_calibration/IsotonicRegression.py
@@ -1,6 +1,7 @@
 from hiclass._calibration.BinaryCalibrator import _BinaryCalibrator
 from sklearn.isotonic import IsotonicRegression as SkLearnIR
 from sklearn.utils.validation import check_is_fitted
+import numpy as np
 
 
 class _IsotonicRegression(_BinaryCalibrator):
@@ -12,11 +13,11 @@ def __init__(self, params={}) -> None:
             params["out_of_bounds"] = "clip"
         self.isotonic_regression = SkLearnIR(**params)
 
-    def fit(self, y, scores, X=None):
+    def fit(self, y: np.ndarray, scores: np.ndarray, X: np.ndarray = None):
         self.isotonic_regression.fit(scores, y)
         self._is_fitted = True
         return self
 
-    def predict_proba(self, scores, X=None):
+    def predict_proba(self, scores: np.ndarray, X: np.ndarray = None):
         check_is_fitted(self)
         return self.isotonic_regression.predict(scores)
diff --git a/hiclass/_calibration/PlattScaling.py b/hiclass/_calibration/PlattScaling.py
index b3109003..60d73e8e 100644
--- a/hiclass/_calibration/PlattScaling.py
+++ b/hiclass/_calibration/PlattScaling.py
@@ -1,6 +1,7 @@
 from hiclass._calibration.BinaryCalibrator import _BinaryCalibrator
 from sklearn.calibration import _SigmoidCalibration
 from sklearn.utils.validation import check_is_fitted
+import numpy as np
 
 
 class _PlattScaling(_BinaryCalibrator):
@@ -10,11 +11,11 @@ def __init__(self) -> None:
         self._is_fitted = False
         self.platt_scaling = _SigmoidCalibration()
 
-    def fit(self, y, scores, X=None):
+    def fit(self, y: np.ndarray, scores: np.ndarray, X: np.ndarray = None):
         self.platt_scaling.fit(scores, y)
         self._is_fitted = True
         return self
 
-    def predict_proba(self, scores, X=None):
+    def predict_proba(self, scores: np.ndarray, X: np.ndarray = None):
         check_is_fitted(self)
         return self.platt_scaling.predict(scores)
diff --git a/hiclass/_calibration/VennAbersCalibrator.py b/hiclass/_calibration/VennAbersCalibrator.py
index 5365981e..d9e06f2b 100644
--- a/hiclass/_calibration/VennAbersCalibrator.py
+++ b/hiclass/_calibration/VennAbersCalibrator.py
@@ -5,6 +5,7 @@
 from hiclass._calibration.calibration_utils import _one_vs_rest_split
 from collections import defaultdict
 from sklearn.utils.validation import check_is_fitted
+from sklearn.base import BaseEstimator
 
 
 class _InductiveVennAbersCalibrator(_BinaryCalibrator):
@@ -13,7 +14,7 @@ class _InductiveVennAbersCalibrator(_BinaryCalibrator):
     def __init__(self) -> None:
         super().__init__()
 
-    def fit(self, y, scores, X=None):
+    def fit(self, y: np.ndarray, scores: np.ndarray, X: np.ndarray = None):
         positive_label = 1
         unique_labels = np.unique(y)
         assert len(unique_labels) <= 2
@@ -137,7 +138,7 @@ def _compute_f0(self, prev_stack, csd):
             stack.append(csd[i])
         return F0
 
-    def predict_proba(self, scores, X=None):
+    def predict_proba(self, scores: np.ndarray, X: np.ndarray = None):
         check_is_fitted(self)
         lower = np.searchsorted(self._unique_elements, scores, side="left")
         upper = np.searchsorted(self._unique_elements[:-1], scores, side="right") + 1
@@ -147,7 +148,7 @@ def predict_proba(self, scores, X=None):
 
         return p1 / (1 - p0 + p1)
 
-    def predict_intervall(self, scores):
+    def predict_intervall(self, scores: np.ndarray):
         lower = np.searchsorted(self._unique_elements, scores, side="left")
         upper = np.searchsorted(self._unique_elements[:-1], scores, side="right") + 1
         p0 = self._F0[lower]
@@ -159,7 +160,7 @@ def predict_intervall(self, scores):
 class _CrossVennAbersCalibrator(_BinaryCalibrator):
     name = "CrossVennAbersCalibrator"
 
-    def __init__(self, estimator, n_folds=5) -> None:
+    def __init__(self, estimator: BaseEstimator, n_folds: int = 5) -> None:
         self._is_fitted = False
         self.n_folds = n_folds
         self.estimator_type = type(estimator)
@@ -169,7 +170,7 @@ def __init__(self, estimator, n_folds=5) -> None:
         self.use_estimator_fallback = False
         self.used_cv = True
 
-    def fit(self, y, scores, X):
+    def fit(self, y: np.ndarray, scores: np.ndarray, X: np.ndarray):
         unique_labels = np.unique(y)
         assert len(unique_labels) >= 2
         self.ivaps = []
@@ -242,7 +243,7 @@ def fit(self, y, scores, X):
 
         return self
 
-    def predict_proba(self, scores):
+    def predict_proba(self, scores: np.ndarray):
         check_is_fitted(self)
 
         if self.use_estimator_fallback:
diff --git a/hiclass/_calibration/calibration_utils.py b/hiclass/_calibration/calibration_utils.py
index 33672a0a..c1ea6731 100644
--- a/hiclass/_calibration/calibration_utils.py
+++ b/hiclass/_calibration/calibration_utils.py
@@ -1,7 +1,9 @@
 from sklearn.preprocessing import LabelBinarizer
+from sklearn.base import BaseEstimator
+import numpy as np
 
 
-def _one_vs_rest_split(y, scores, estimator):
+def _one_vs_rest_split(y: np.ndarray, scores: np.ndarray, estimator: BaseEstimator):
     # binarize multiclass labels
     label_binarizer = LabelBinarizer()
     label_binarizer.fit(estimator.classes_)
diff --git a/hiclass/metrics.py b/hiclass/metrics.py
index 47e45a33..68c368c1 100644
--- a/hiclass/metrics.py
+++ b/hiclass/metrics.py
@@ -283,7 +283,7 @@ def _prepare_data(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob
 _calibration_aggregations = ["average", "sum", "None"]
 
 
-def _aggregate_scores(scores, agg):
+def _aggregate_scores(scores: np.ndarray, agg: str):
     if agg == 'average':
         return np.mean(scores)
     if agg == 'sum':
@@ -292,7 +292,7 @@ def _aggregate_scores(scores, agg):
         return scores
 
 
-def _validate_args(agg, y_prob, level):
+def _validate_args(agg: str, y_prob: np.ndarray, level: int):
     if agg and agg not in _calibration_aggregations:
         raise ValueError(f"{agg} is not a valid aggregation function.")
     if isinstance(y_prob, list) and len(y_prob) == 0:
@@ -304,7 +304,7 @@ def _validate_args(agg, y_prob, level):
     return y_prob
 
 
-def multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Union[np.ndarray | List], agg='average', level=None):
+def multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Union[np.ndarray | List], agg: str = 'average', level: int = None):
     """Compute the brier score for two or more classes.
 
     Parameters
@@ -337,7 +337,7 @@ def multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarra
     return _multiclass_brier_score(classifier, y_true, y_prob, level)
 
 
-def log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Union[np.ndarray | List], agg='average', level=None):
+def log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Union[np.ndarray | List], agg: str = 'average', level: int = None):
     """Compute the log loss of predicted probabilities.
 
     Parameters
@@ -370,7 +370,7 @@ def log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Uni
     return _multiclass_brier_score(classifier, y_true, y_prob, level)
 
 
-def expected_calibration_error(classifier: HierarchicalClassifier, y_true, y_prob: Union[np.ndarray | List], y_pred, n_bins=10, agg='average', level=None):
+def expected_calibration_error(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Union[np.ndarray | List], y_pred: np.ndarray, n_bins: int = 10, agg: str = 'average', level: int = None):
     """Compute the expected calibration error.
 
     Parameters
@@ -407,7 +407,7 @@ def expected_calibration_error(classifier: HierarchicalClassifier, y_true, y_pro
     return _expected_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins)
 
 
-def static_calibration_error(classifier, y_true, y_prob: Union[np.ndarray | List], y_pred, n_bins=10, agg='average', level=None):
+def static_calibration_error(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Union[np.ndarray | List], y_pred: np.ndarray, n_bins: int = 10, agg: str = 'average', level: int = None):
     """Compute the static calibration error.
 
     Parameters
@@ -444,7 +444,7 @@ def static_calibration_error(classifier, y_true, y_prob: Union[np.ndarray | List
     return _static_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins=n_bins)
 
 
-def adaptive_calibration_error(classifier, y_true, y_prob: Union[np.ndarray | List], y_pred, n_ranges=10, agg='average', level=None):
+def adaptive_calibration_error(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Union[np.ndarray | List], y_pred: np.ndarray, n_ranges: int = 10, agg: str = 'average', level: int = None):
     """Compute the adaptive calibration error.
 
     Parameters
@@ -494,7 +494,7 @@ def _log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np
     return sk_log_loss(y_true, y_prob, labels=labels)
 
 
-def _expected_calibration_error(classifier: HierarchicalClassifier, y_true, y_prob, y_pred, level, n_bins=10):
+def _expected_calibration_error(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, y_pred: np.ndarray, level: int, n_bins: int = 10):
     y_true, y_pred, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level, y_pred)
 
     n = len(y_true)
@@ -531,7 +531,7 @@ def _expected_calibration_error(classifier: HierarchicalClassifier, y_true, y_pr
     return ece
 
 
-def _static_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins=10):
+def _static_calibration_error(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, y_pred: np.ndarray, level: int, n_bins: int = 10):
     y_true, y_pred, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level, y_pred)
 
     n_samples, n_classes = y_prob.shape
@@ -576,7 +576,7 @@ def _static_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins=
     return np.mean(class_error)
 
 
-def _adaptive_calibration_error(classifier, y_true, y_prob, y_pred, level, n_ranges=10):
+def _adaptive_calibration_error(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, y_pred: np.ndarray, level: int, n_ranges: int = 10):
     y_true, y_pred, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level, y_pred)
 
     _, n_classes = y_prob.shape
diff --git a/hiclass/probability_combiner/ArithmeticMeanCombiner.py b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
index 7ec1795c..de50a02e 100644
--- a/hiclass/probability_combiner/ArithmeticMeanCombiner.py
+++ b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
@@ -1,12 +1,13 @@
 """Defines the ArithmeticMeanCombiner."""
 import numpy as np
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
+from typing import List
 
 
 class ArithmeticMeanCombiner(ProbabilityCombiner):
     """Combine probabilities of multiple levels by taking their arithmetic mean."""
 
-    def combine(self, proba):
+    def combine(self, proba: List[np.ndarray]):
         """Combine probabilities of each level with probabilities of previous levels.
 
         Calculate the arithmetic mean of node probabilities and the probabilities of its predecessors.
diff --git a/hiclass/probability_combiner/GeometricMeanCombiner.py b/hiclass/probability_combiner/GeometricMeanCombiner.py
index 1f449ca0..f3c49cb2 100644
--- a/hiclass/probability_combiner/GeometricMeanCombiner.py
+++ b/hiclass/probability_combiner/GeometricMeanCombiner.py
@@ -1,12 +1,13 @@
 """Defines the GeometricMeanCombiner."""
 import numpy as np
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
+from typing import List
 
 
 class GeometricMeanCombiner(ProbabilityCombiner):
     """Combine probabilities of multiple levels by taking their geometric mean."""
 
-    def combine(self, proba):
+    def combine(self, proba: List[np.ndarray]):
         """Combine probabilities of each level with probabilities of previous levels.
 
         Calculate the geometric mean of node probabilities and the probabilities of its predecessors.
diff --git a/hiclass/probability_combiner/MultiplyCombiner.py b/hiclass/probability_combiner/MultiplyCombiner.py
index a06812fc..386ecdf9 100644
--- a/hiclass/probability_combiner/MultiplyCombiner.py
+++ b/hiclass/probability_combiner/MultiplyCombiner.py
@@ -1,12 +1,13 @@
 """Defines the MultiplyCombiner."""
 import numpy as np
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
+from typing import List
 
 
 class MultiplyCombiner(ProbabilityCombiner):
     """Combine probabilities of multiple levels by multiplication."""
 
-    def combine(self, proba):
+    def combine(self, proba: List[np.ndarray]):
         """Combine probabilities of each level with probabilities of previous levels.
 
         Multiply node probabilities with the probabilities of its predecessors.
diff --git a/hiclass/probability_combiner/ProbabilityCombiner.py b/hiclass/probability_combiner/ProbabilityCombiner.py
index d4edbb4b..4b5b77e3 100644
--- a/hiclass/probability_combiner/ProbabilityCombiner.py
+++ b/hiclass/probability_combiner/ProbabilityCombiner.py
@@ -4,12 +4,13 @@
 from typing import List
 from collections import defaultdict
 from networkx.exception import NetworkXError
+from hiclass import HierarchicalClassifier
 
 
 class ProbabilityCombiner(abc.ABC):
     """Abstract class defining the structure of a probability combiner."""
 
-    def __init__(self, classifier, normalize=True) -> None:
+    def __init__(self, classifier: HierarchicalClassifier, normalize: bool = True) -> None:
         """Initialize probability combiner object."""
         self.classifier = classifier
         self.normalize = normalize
@@ -19,13 +20,13 @@ def combine(self, proba: List[np.ndarray]) -> List[np.ndarray]:
         """Combine probabilities over multiple levels."""
         ...
 
-    def _normalize(self, proba):
+    def _normalize(self, proba: List[np.ndarray]):
         return [
             np.nan_to_num(level_probabilities / level_probabilities.sum(axis=1, keepdims=True))
             for level_probabilities in proba
         ]
 
-    def _find_predecessors(self, level):
+    def _find_predecessors(self, level: int):
         predecessors = defaultdict(list)
         for node in self.classifier.global_classes_[level]:
             try:

From 34eacd30a1920984f841434ce92697916088c0ee Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Thu, 11 Apr 2024 16:28:58 +0200
Subject: [PATCH 47/65] run black formatter

---
 hiclass/HierarchicalClassifier.py             |  94 +++++--
 hiclass/LocalClassifierPerLevel.py            |  83 ++++--
 hiclass/LocalClassifierPerNode.py             |  68 +++--
 hiclass/LocalClassifierPerParentNode.py       |  87 +++++--
 hiclass/_calibration/BetaCalibrator.py        |   9 +-
 hiclass/_calibration/BinaryCalibrator.py      |   9 +-
 hiclass/_calibration/Calibrator.py            |  21 +-
 hiclass/_calibration/VennAbersCalibrator.py   |  49 +++-
 hiclass/metrics.py                            | 242 ++++++++++++++----
 .../ArithmeticMeanCombiner.py                 |  17 +-
 .../GeometricMeanCombiner.py                  |  19 +-
 .../probability_combiner/MultiplyCombiner.py  |  14 +-
 .../ProbabilityCombiner.py                    |   8 +-
 hiclass/probability_combiner/__init__.py      |   2 +-
 14 files changed, 567 insertions(+), 155 deletions(-)

diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index bbf698de..0112d768 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -17,7 +17,7 @@
 from hiclass.probability_combiner import (
     GeometricMeanCombiner,
     ArithmeticMeanCombiner,
-    MultiplyCombiner
+    MultiplyCombiner,
 )
 
 try:
@@ -174,19 +174,47 @@ def _pre_fit(self, X, y, sample_weight):
         self.y_ = self._disambiguate(self.y_)
 
         if self.y_.ndim > 1:
-            self.max_level_dimensions_ = np.array([len(np.unique(self.y_[:, level])) for level in range(self.y_.shape[1])])
-            self.global_classes_ = [np.unique(self.y_[:, level]).astype("str") for level in range(self.y_.shape[1])]
-            self.global_class_to_index_mapping_ = [{self.global_classes_[level][index]: index for index in range(len(self.global_classes_[level]))} for level in range(self.y_.shape[1])]
+            self.max_level_dimensions_ = np.array(
+                [len(np.unique(self.y_[:, level])) for level in range(self.y_.shape[1])]
+            )
+            self.global_classes_ = [
+                np.unique(self.y_[:, level]).astype("str")
+                for level in range(self.y_.shape[1])
+            ]
+            self.global_class_to_index_mapping_ = [
+                {
+                    self.global_classes_[level][index]: index
+                    for index in range(len(self.global_classes_[level]))
+                }
+                for level in range(self.y_.shape[1])
+            ]
         else:
             self.max_level_dimensions_ = np.array([len(np.unique(self.y_))])
             self.global_classes_ = [np.unique(self.y_).astype("str")]
-            self.global_class_to_index_mapping_ = [{self.global_classes_[0][index] : index for index in range(len(self.global_classes_[0]))}]
+            self.global_class_to_index_mapping_ = [
+                {
+                    self.global_classes_[0][index]: index
+                    for index in range(len(self.global_classes_[0]))
+                }
+            ]
 
         classes_ = [self.global_classes_[0]]
         for level in range(1, len(self.max_level_dimensions_)):
-            classes_.append(np.sort(np.unique([label.split(self.separator_)[level] for label in self.global_classes_[level]])))
+            classes_.append(
+                np.sort(
+                    np.unique(
+                        [
+                            label.split(self.separator_)[level]
+                            for label in self.global_classes_[level]
+                        ]
+                    )
+                )
+            )
         self.classes_ = classes_
-        self.class_to_index_mapping_ = [{local_labels[index]: index for index in range(len(local_labels))} for local_labels in classes_]
+        self.class_to_index_mapping_ = [
+            {local_labels[index]: index for index in range(len(local_labels))}
+            for local_labels in classes_
+        ]
 
         # Create and configure logger
         self._create_logger()
@@ -427,21 +455,31 @@ def logging_wrapper(func, idx, node, node_length):
                         ignore_reinit_error=True,
                     )
                 lcppn = ray.put(self)
-                _parallel_fit = ray.remote(self._fit_classifier)  # TODO: use logging wrapper
+                _parallel_fit = ray.remote(
+                    self._fit_classifier
+                )  # TODO: use logging wrapper
                 results = [_parallel_fit.remote(lcppn, node) for node in nodes]
                 classifiers = ray.get(results)
             else:
                 classifiers = Parallel(n_jobs=self.n_jobs)(
-                    delayed(logging_wrapper)(self._fit_classifier, idx, node, len(nodes)) for idx, node in enumerate(nodes)
+                    delayed(logging_wrapper)(
+                        self._fit_classifier, idx, node, len(nodes)
+                    )
+                    for idx, node in enumerate(nodes)
                 )
 
         else:
-            classifiers = [logging_wrapper(self._fit_classifier, idx, node, len(nodes)) for idx, node in enumerate(nodes)]
+            classifiers = [
+                logging_wrapper(self._fit_classifier, idx, node, len(nodes))
+                for idx, node in enumerate(nodes)
+            ]
 
         for classifier, node in zip(classifiers, nodes):
             self.hierarchy_.nodes[node]["classifier"] = classifier
 
-    def _fit_node_calibrator(self, nodes, local_mode: bool = False, use_joblib: bool = False):
+    def _fit_node_calibrator(
+        self, nodes, local_mode: bool = False, use_joblib: bool = False
+    ):
         def logging_wrapper(func, idx, node, node_length):
             self.logger_.info(f"calibrating node {idx+1}/{node_length}: {str(node)}")
             return func(self, node)
@@ -456,16 +494,24 @@ def logging_wrapper(func, idx, node, node_length):
                     )
                 lcppn = ray.put(self)
                 _parallel_fit = ray.remote(self._fit_calibrator)
-                results = [_parallel_fit.remote(lcppn, node) for idx, node in enumerate(nodes)]  # TODO: use logging wrapper
+                results = [
+                    _parallel_fit.remote(lcppn, node) for idx, node in enumerate(nodes)
+                ]  # TODO: use logging wrapper
                 calibrators = ray.get(results)
                 ray.shutdown()
             else:
                 calibrators = Parallel(n_jobs=self.n_jobs)(
-                    delayed(logging_wrapper)(self._fit_calibrator, idx, node, len(nodes)) for idx, node in enumerate(nodes)
+                    delayed(logging_wrapper)(
+                        self._fit_calibrator, idx, node, len(nodes)
+                    )
+                    for idx, node in enumerate(nodes)
                 )
 
         else:
-            calibrators = [logging_wrapper(self._fit_calibrator, idx, node, len(nodes)) for idx, node in enumerate(nodes)]
+            calibrators = [
+                logging_wrapper(self._fit_calibrator, idx, node, len(nodes))
+                for idx, node in enumerate(nodes)
+            ]
 
         for calibrator, node in zip(calibrators, nodes):
             self.hierarchy_.nodes[node]["calibrator"] = calibrator
@@ -481,30 +527,32 @@ def _fit_calibrator(self, node):
         raise NotImplementedError("Method should be implemented in the LCPN and LCPPN")
 
     def _create_probability_combiner(self, name):
-        if name == 'geometric':
+        if name == "geometric":
             return GeometricMeanCombiner(self)
-        elif name == 'arithmetic':
+        elif name == "arithmetic":
             return ArithmeticMeanCombiner(self)
-        elif name == 'multiply':
+        elif name == "multiply":
             return MultiplyCombiner(self)
 
     def _clean_up(self):
         self.logger_.info("Cleaning up variables that can take a lot of disk space")
-        if hasattr(self, 'X_'):
+        if hasattr(self, "X_"):
             del self.X_
-        if hasattr(self, 'y_'):
+        if hasattr(self, "y_"):
             del self.y_
-        if hasattr(self, 'sample_weight') and self.sample_weight_ is not None:
+        if hasattr(self, "sample_weight") and self.sample_weight_ is not None:
             del self.sample_weight_
-        if hasattr(self, 'X_cal'):
+        if hasattr(self, "X_cal"):
             del self.X_cal
-        if hasattr(self, 'y_cal'):
+        if hasattr(self, "y_cal"):
             del self.y_cal
 
     def _combine_and_reorder(self, proba):
         res = [proba[0]]
         for level in range(1, self.max_levels_):
-            res_proba = np.zeros(shape=(proba[level].shape[0], len(self.classes_[level])))
+            res_proba = np.zeros(
+                shape=(proba[level].shape[0], len(self.classes_[level]))
+            )
 
             for old_label in self.global_classes_[level]:
                 old_idx = self.global_class_to_index_mapping_[level][old_label]
diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index e72fb1d2..f62c842f 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -18,7 +18,9 @@
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
 from hiclass._calibration.Calibrator import _Calibrator
 
-from hiclass.probability_combiner import init_strings as probability_combiner_init_strings
+from hiclass.probability_combiner import (
+    init_strings as probability_combiner_init_strings,
+)
 
 try:
     import ray
@@ -110,8 +112,13 @@ def __init__(
         self.return_all_probabilities = return_all_probabilities
         self.probability_combiner = probability_combiner
 
-        if self.probability_combiner and self.probability_combiner not in probability_combiner_init_strings:
-            raise ValueError(f"probability_combiner must be one of {', '.join(probability_combiner_init_strings)} or None.")
+        if (
+            self.probability_combiner
+            and self.probability_combiner not in probability_combiner_init_strings
+        ):
+            raise ValueError(
+                f"probability_combiner must be one of {', '.join(probability_combiner_init_strings)} or None."
+            )
 
     def fit(self, X, y, sample_weight=None):
         """
@@ -223,7 +230,9 @@ def predict_proba(self, X):
             X = np.array(X)
 
         if not self.calibration_method:
-            self.logger_.info("It is not recommended to use predict_proba() without calibration")
+            self.logger_.info(
+                "It is not recommended to use predict_proba() without calibration"
+            )
 
         # Initialize array that holds predictions
         y = np.empty((X.shape[0], self.max_levels_), dtype=self.dtype_)
@@ -232,7 +241,11 @@ def predict_proba(self, X):
 
         # Predict first level
         classifier = self.local_classifiers_[0]
-        calibrator = self.local_calibrators_[0] if hasattr(self, 'self.local_calibrators_') else None
+        calibrator = (
+            self.local_calibrators_[0]
+            if hasattr(self, "self.local_calibrators_")
+            else None
+        )
 
         # use classifier as a fallback if no calibrator is available
         calibrator = calibrator or classifier
@@ -244,17 +257,31 @@ def predict_proba(self, X):
 
         # combine probabilities
         if self.probability_combiner:
-            probability_combiner_ = self._create_probability_combiner(self.probability_combiner)
-            self.logger_.info(f"Combining probabilities using {type(probability_combiner_).__name__}")
-            level_probability_list = probability_combiner_.combine(level_probability_list)
+            probability_combiner_ = self._create_probability_combiner(
+                self.probability_combiner
+            )
+            self.logger_.info(
+                f"Combining probabilities using {type(probability_combiner_).__name__}"
+            )
+            level_probability_list = probability_combiner_.combine(
+                level_probability_list
+            )
 
-        return level_probability_list if self.return_all_probabilities else level_probability_list[-1]
+        return (
+            level_probability_list
+            if self.return_all_probabilities
+            else level_probability_list[-1]
+        )
 
     def _predict_proba_remaining_levels(self, X, y):
         level_probability_list = []
         for level in range(1, y.shape[1]):
             classifier = self.local_classifiers_[level]
-            calibrator = self.local_calibrators_[level] if hasattr(self, 'self.local_calibrators_') else None
+            calibrator = (
+                self.local_calibrators_[level]
+                if hasattr(self, "self.local_calibrators_")
+                else None
+            )
             # use classifier as a fallback if no calibrator is available
             calibrator = calibrator or classifier
             probabilities = calibrator.predict_proba(X)
@@ -314,7 +341,8 @@ def _initialize_local_classifiers(self):
     def _initialize_local_calibrators(self):
         super()._initialize_local_calibrators()
         self.local_calibrators_ = [
-            _Calibrator(estimator=local_classifier, method=self.calibration_method) for local_classifier in self.local_classifiers_
+            _Calibrator(estimator=local_classifier, method=self.calibration_method)
+            for local_classifier in self.local_classifiers_
         ]
 
     def _fit_digraph(self, local_mode: bool = False, use_joblib: bool = False):
@@ -341,12 +369,22 @@ def logging_wrapper(func, level, separator, max_level):
                 classifiers = ray.get(results)
             else:
                 classifiers = Parallel(n_jobs=self.n_jobs)(
-                    delayed(logging_wrapper)(self._fit_classifier, level, self.separator_, len(self.local_classifiers_))
+                    delayed(logging_wrapper)(
+                        self._fit_classifier,
+                        level,
+                        self.separator_,
+                        len(self.local_classifiers_),
+                    )
                     for level in range(len(self.local_classifiers_))
                 )
         else:
             classifiers = [
-                logging_wrapper(self._fit_classifier, level, self.separator_, len(self.local_classifiers_))
+                logging_wrapper(
+                    self._fit_classifier,
+                    level,
+                    self.separator_,
+                    len(self.local_classifiers_),
+                )
                 for level in range(len(self.local_classifiers_))
             ]
         for level, classifier in enumerate(classifiers):
@@ -377,11 +415,22 @@ def logging_wrapper(func, level, separator, max_level):
 
             else:
                 calibrators = Parallel(n_jobs=self.n_jobs)(
-                    delayed(logging_wrapper)(self._fit_calibrator, level, self.separator_, len(self.local_calibrators_)) for level in range(len(self.local_calibrators_))
+                    delayed(logging_wrapper)(
+                        self._fit_calibrator,
+                        level,
+                        self.separator_,
+                        len(self.local_calibrators_),
+                    )
+                    for level in range(len(self.local_calibrators_))
                 )
         else:
             calibrators = [
-                logging_wrapper(self._fit_calibrator, level, self.separator_, len(self.local_calibrators_))
+                logging_wrapper(
+                    self._fit_calibrator,
+                    level,
+                    self.separator_,
+                    len(self.local_calibrators_),
+                )
                 for level in range(len(self.local_calibrators_))
             ]
 
@@ -430,7 +479,9 @@ def _fit_calibrator(self, level, separator):
             separator, self.X_cal, self.y_cal[:, level], None
         )
         if len(y) == 0 or len(np.unique(y)) < 2:
-            self.logger_.info(f"No calibration samples to fit calibrator for level: {str(level)}")
+            self.logger_.info(
+                f"No calibration samples to fit calibrator for level: {str(level)}"
+            )
             return None
         calibrator.fit(X, y)
         return calibrator
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index 158c70ee..ba02edbc 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -19,7 +19,9 @@
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
 from hiclass._calibration.Calibrator import _Calibrator
 
-from hiclass.probability_combiner import init_strings as probability_combiner_init_strings
+from hiclass.probability_combiner import (
+    init_strings as probability_combiner_init_strings,
+)
 
 
 class LocalClassifierPerNode(BaseEstimator, HierarchicalClassifier):
@@ -117,8 +119,13 @@ def __init__(
         self.return_all_probabilities = return_all_probabilities
         self.probability_combiner = probability_combiner
 
-        if self.probability_combiner and self.probability_combiner not in probability_combiner_init_strings:
-            raise ValueError(f"probability_combiner must be one of {', '.join(probability_combiner_init_strings)} or None.")
+        if (
+            self.probability_combiner
+            and self.probability_combiner not in probability_combiner_init_strings
+        ):
+            raise ValueError(
+                f"probability_combiner must be one of {', '.join(probability_combiner_init_strings)} or None."
+            )
 
     def fit(self, X, y, sample_weight=None):
         """
@@ -255,7 +262,9 @@ def predict_proba(self, X):
             X = np.array(X)
 
         if not self.calibration_method:
-            self.logger_.info("It is not recommended to use predict_proba() without calibration")
+            self.logger_.info(
+                "It is not recommended to use predict_proba() without calibration"
+            )
         bfs = nx.bfs_successors(self.hierarchy_, source=self.root_)
         self.logger_.info("Predicting Probability")
 
@@ -264,9 +273,7 @@ def predict_proba(self, X):
         last_level = -1
 
         for predecessor, successors in bfs:
-            level = nx.shortest_path_length(
-                self.hierarchy_, self.root_, predecessor
-            )
+            level = nx.shortest_path_length(self.hierarchy_, self.root_, predecessor)
             level_dimension = self.max_level_dimensions_[level]
 
             if last_level != level:
@@ -284,14 +291,21 @@ def predict_proba(self, X):
             if subset_x.shape[0] > 0:
                 local_probabilities = np.zeros((subset_x.shape[0], len(successors)))
                 for i, successor in enumerate(successors):
-                    self.logger_.info(f"Predicting probabilities for node '{str(successor)}'")
+                    self.logger_.info(
+                        f"Predicting probabilities for node '{str(successor)}'"
+                    )
                     classifier = self.hierarchy_.nodes[successor]["classifier"]
                     # use classifier as a fallback if no calibrator is available
-                    calibrator = self.hierarchy_.nodes[successor].get("calibrator", classifier) or classifier
+                    calibrator = (
+                        self.hierarchy_.nodes[successor].get("calibrator", classifier)
+                        or classifier
+                    )
                     positive_index = np.where(calibrator.classes_ == 1)[0]
                     proba = calibrator.predict_proba(subset_x)[:, positive_index][:, 0]
                     local_probabilities[:, i] = proba
-                    class_index = self.global_class_to_index_mapping_[level][str(successor)]
+                    class_index = self.global_class_to_index_mapping_[level][
+                        str(successor)
+                    ]
                     level_probability_list[-1][mask, class_index] = proba
 
                 highest_local_probability = np.argmax(local_probabilities, axis=1)
@@ -307,7 +321,9 @@ def predict_proba(self, X):
 
         # normalize probabilities
         level_probability_list = [
-            np.nan_to_num(level_probabilities / level_probabilities.sum(axis=1, keepdims=True))
+            np.nan_to_num(
+                level_probabilities / level_probabilities.sum(axis=1, keepdims=True)
+            )
             for level_probabilities in level_probability_list
         ]
 
@@ -316,11 +332,21 @@ def predict_proba(self, X):
 
         # combine probabilities vertically
         if self.probability_combiner:
-            probability_combiner_ = self._create_probability_combiner(self.probability_combiner)
-            self.logger_.info(f"Combining probabilities using {type(probability_combiner_).__name__}")
-            level_probability_list = probability_combiner_.combine(level_probability_list)
+            probability_combiner_ = self._create_probability_combiner(
+                self.probability_combiner
+            )
+            self.logger_.info(
+                f"Combining probabilities using {type(probability_combiner_).__name__}"
+            )
+            level_probability_list = probability_combiner_.combine(
+                level_probability_list
+            )
 
-        return level_probability_list if self.return_all_probabilities else level_probability_list[-1]
+        return (
+            level_probability_list
+            if self.return_all_probabilities
+            else level_probability_list[-1]
+        )
 
     def _initialize_binary_policy(self, calibration=False):
         if isinstance(self.binary_policy, str):
@@ -367,7 +393,9 @@ def _initialize_local_calibrators(self):
                 # get classifier from node
                 local_classifier = self.hierarchy_.nodes[node]["classifier"]
                 local_calibrators[node] = {
-                    "calibrator": _Calibrator(estimator=local_classifier, method=self.calibration_method)
+                    "calibrator": _Calibrator(
+                        estimator=local_classifier, method=self.calibration_method
+                    )
                 }
         nx.set_node_attributes(self.hierarchy_, local_calibrators)
 
@@ -423,14 +451,16 @@ def _fit_calibrator(self, node):
             return None
         X, y, _ = self.cal_binary_policy_.get_binary_examples(node)
         if len(y) == 0 or len(np.unique(y)) < 2:
-            self.logger_.info(f"No calibration samples to fit calibrator for node: {str(node)}")
+            self.logger_.info(
+                f"No calibration samples to fit calibrator for node: {str(node)}"
+            )
             return None
         calibrator.fit(X, y)
         return calibrator
 
     def _clean_up(self):
         super()._clean_up()
-        if hasattr(self, 'binary_policy_'):
+        if hasattr(self, "binary_policy_"):
             del self.binary_policy_
-        if hasattr(self, 'cal_binary_policy_'):
+        if hasattr(self, "cal_binary_policy_"):
             del self.cal_binary_policy_
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index 0cde2db3..67dfec7a 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -18,7 +18,9 @@
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
 from hiclass._calibration.Calibrator import _Calibrator
 
-from hiclass.probability_combiner import init_strings as probability_combiner_init_strings
+from hiclass.probability_combiner import (
+    init_strings as probability_combiner_init_strings,
+)
 
 
 class LocalClassifierPerParentNode(BaseEstimator, HierarchicalClassifier):
@@ -103,8 +105,13 @@ def __init__(
         self.return_all_probabilities = return_all_probabilities
         self.probability_combiner = probability_combiner
 
-        if self.probability_combiner and self.probability_combiner not in probability_combiner_init_strings:
-            raise ValueError(f"probability_combiner must be one of {', '.join(probability_combiner_init_strings)} or None.")
+        if (
+            self.probability_combiner
+            and self.probability_combiner not in probability_combiner_init_strings
+        ):
+            raise ValueError(
+                f"probability_combiner must be one of {', '.join(probability_combiner_init_strings)} or None."
+            )
 
     def fit(self, X, y, sample_weight=None):
         """
@@ -216,7 +223,9 @@ def predict_proba(self, X):
             X = np.array(X)
 
         if not self.calibration_method:
-            self.logger_.info("It is not recommended to use predict_proba() without calibration")
+            self.logger_.info(
+                "It is not recommended to use predict_proba() without calibration"
+            )
 
         self.logger_.info("Predicting Probability")
 
@@ -226,7 +235,10 @@ def predict_proba(self, X):
         # Predict first level
         classifier = self.hierarchy_.nodes[self.root_]["classifier"]
         # use classifier as a fallback if no calibrator is available
-        calibrator = self.hierarchy_.nodes[self.root_].get("calibrator", classifier) or classifier
+        calibrator = (
+            self.hierarchy_.nodes[self.root_].get("calibrator", classifier)
+            or classifier
+        )
         proba = calibrator.predict_proba(X)
 
         y[:, 0] = calibrator.classes_[np.argmax(proba, axis=1)]
@@ -236,11 +248,21 @@ def predict_proba(self, X):
 
         # combine probabilities
         if self.probability_combiner:
-            probability_combiner_ = self._create_probability_combiner(self.probability_combiner)
-            self.logger_.info(f"Combining probabilities using {type(probability_combiner_).__name__}")
-            level_probability_list = probability_combiner_.combine(level_probability_list)
-
-        return level_probability_list if self.return_all_probabilities else level_probability_list[-1]
+            probability_combiner_ = self._create_probability_combiner(
+                self.probability_combiner
+            )
+            self.logger_.info(
+                f"Combining probabilities using {type(probability_combiner_).__name__}"
+            )
+            level_probability_list = probability_combiner_.combine(
+                level_probability_list
+            )
+
+        return (
+            level_probability_list
+            if self.return_all_probabilities
+            else level_probability_list[-1]
+        )
 
     def _predict_proba_remaining_levels(self, X, y):
         level_probability_list = []
@@ -258,22 +280,35 @@ def _predict_proba_remaining_levels(self, X, y):
                     if len(successors) > 0:
                         classifier = self.hierarchy_.nodes[predecessor]["classifier"]
                         # use classifier as a fallback if no calibrator is available
-                        calibrator = self.hierarchy_.nodes[predecessor].get("calibrator", classifier) or classifier
+                        calibrator = (
+                            self.hierarchy_.nodes[predecessor].get(
+                                "calibrator", classifier
+                            )
+                            or classifier
+                        )
 
                         proba = calibrator.predict_proba(predecessor_x)
                         y[mask, level] = calibrator.classes_[np.argmax(proba, axis=1)]
 
                         for successor in successors:
-                            class_index = self.global_class_to_index_mapping_[level][str(successor)]
+                            class_index = self.global_class_to_index_mapping_[level][
+                                str(successor)
+                            ]
 
-                            proba_index = np.where(calibrator.classes_ == successor)[0][0]
-                            cur_level_probabilities[mask, class_index] = proba[:, proba_index]
+                            proba_index = np.where(calibrator.classes_ == successor)[0][
+                                0
+                            ]
+                            cur_level_probabilities[mask, class_index] = proba[
+                                :, proba_index
+                            ]
 
             level_probability_list.append(cur_level_probabilities)
 
         # normalize probabilities
         level_probability_list = [
-            np.nan_to_num(level_probabilities / level_probabilities.sum(axis=1, keepdims=True))
+            np.nan_to_num(
+                level_probabilities / level_probabilities.sum(axis=1, keepdims=True)
+            )
             for level_probabilities in level_probability_list
         ]
 
@@ -307,7 +342,9 @@ def _initialize_local_calibrators(self):
         for node in nodes:
             local_classifier = self.hierarchy_.nodes[node]["classifier"]
             local_calibrators[node] = {
-                "calibrator": _Calibrator(estimator=local_classifier, method=self.calibration_method)
+                "calibrator": _Calibrator(
+                    estimator=local_classifier, method=self.calibration_method
+                )
             }
         nx.set_node_attributes(self.hierarchy_, local_calibrators)
 
@@ -322,7 +359,11 @@ def _get_parents(self):
 
     def _get_successors(self, node, calibration=False):
         successors = list(self.hierarchy_.successors(node))
-        mask = np.isin(self.y_cal, successors).any(axis=1) if calibration else np.isin(self.y_, successors).any(axis=1)
+        mask = (
+            np.isin(self.y_cal, successors).any(axis=1)
+            if calibration
+            else np.isin(self.y_, successors).any(axis=1)
+        )
         X = self.X_cal[mask] if calibration else self.X_[mask]
         y = []
         masked_labels = self.y_cal[mask] if calibration else self.y_[mask]
@@ -332,8 +373,12 @@ def _get_successors(self, node, calibration=False):
             else:
                 y.append(row[np.where(row == node)[0][0] + 1])
         y = np.array(y)
-        sample_weight = None if calibration else (
-            self.sample_weight_[mask] if self.sample_weight_ is not None else None
+        sample_weight = (
+            None
+            if calibration
+            else (
+                self.sample_weight_[mask] if self.sample_weight_ is not None else None
+            )
         )
         return X, y, sample_weight
 
@@ -374,7 +419,9 @@ def _fit_calibrator(self, node):
             return None
         X, y, _ = self._get_successors(node, calibration=True)
         if len(y) == 0 or len(np.unique(y)) < 2:
-            self.logger_.info(f"No calibration samples to fit calibrator for node: {str(node)}")
+            self.logger_.info(
+                f"No calibration samples to fit calibrator for node: {str(node)}"
+            )
             return None
         calibrator.fit(X, y)
         return calibrator
diff --git a/hiclass/_calibration/BetaCalibrator.py b/hiclass/_calibration/BetaCalibrator.py
index 4e94d3a3..22c36dde 100644
--- a/hiclass/_calibration/BetaCalibrator.py
+++ b/hiclass/_calibration/BetaCalibrator.py
@@ -34,4 +34,11 @@ def predict_proba(self, scores: np.ndarray, X: np.ndarray = None):
         check_is_fitted(self)
         if self.skip_calibration:
             return scores
-        return 1 / (1 + 1 / (np.exp(self.c) * (np.power(scores, self.a) / np.power((1 - scores), self.b))))
+        return 1 / (
+            1
+            + 1
+            / (
+                np.exp(self.c)
+                * (np.power(scores, self.a) / np.power((1 - scores), self.b))
+            )
+        )
diff --git a/hiclass/_calibration/BinaryCalibrator.py b/hiclass/_calibration/BinaryCalibrator.py
index 29fb7c92..8300c473 100644
--- a/hiclass/_calibration/BinaryCalibrator.py
+++ b/hiclass/_calibration/BinaryCalibrator.py
@@ -3,13 +3,16 @@
 
 
 class _BinaryCalibrator(abc.ABC):
-
     @abc.abstractmethod
-    def fit(self, y: np.ndarray, scores: np.ndarray, X: np.ndarray = None):  # pragma: no cover
+    def fit(
+        self, y: np.ndarray, scores: np.ndarray, X: np.ndarray = None
+    ):  # pragma: no cover
         ...
 
     @abc.abstractmethod
-    def predict_proba(self, scores: np.ndarray, X: np.ndarray = None):  # pragma: no cover
+    def predict_proba(
+        self, scores: np.ndarray, X: np.ndarray = None
+    ):  # pragma: no cover
         ...
 
     def __sklearn_is_fitted__(self):
diff --git a/hiclass/_calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
index 895174f1..aacacae6 100644
--- a/hiclass/_calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -2,7 +2,10 @@
 from sklearn.base import BaseEstimator
 from sklearn.preprocessing import LabelBinarizer
 from sklearn.preprocessing import LabelEncoder
-from hiclass._calibration.VennAbersCalibrator import _InductiveVennAbersCalibrator, _CrossVennAbersCalibrator
+from hiclass._calibration.VennAbersCalibrator import (
+    _InductiveVennAbersCalibrator,
+    _CrossVennAbersCalibrator,
+)
 from hiclass._calibration.IsotonicRegression import _IsotonicRegression
 from hiclass._calibration.PlattScaling import _PlattScaling
 from hiclass._calibration.BetaCalibrator import _BetaCalibrator
@@ -13,13 +16,15 @@ class _Calibrator(BaseEstimator):
     available_methods = ["ivap", "cvap", "sigmoid", "isotonic", "beta"]
     _multiclass_methods = ["cvap"]
 
-    def __init__(self, estimator: BaseEstimator, method: str = "ivap", **method_params) -> None:
-        assert callable(getattr(estimator, 'predict_proba', None))
+    def __init__(
+        self, estimator: BaseEstimator, method: str = "ivap", **method_params
+    ) -> None:
+        assert callable(getattr(estimator, "predict_proba", None))
         self.estimator = estimator
         self.method_params = method_params
         # self.classes_ = self.estimator.classes_
         self.multiclass = False
-        self.multiclass_support = (method in self._multiclass_methods)
+        self.multiclass_support = method in self._multiclass_methods
         if method not in self.available_methods:
             raise ValueError(f"{method} is not a valid calibration method.")
         self.method = method
@@ -61,10 +66,14 @@ def fit(self, X: np.ndarray, y: np.ndarray):
 
             else:
                 # do one vs rest calibration
-                score_splits, label_splits = _one_vs_rest_split(y, calibration_scores, self.estimator)
+                score_splits, label_splits = _one_vs_rest_split(
+                    y, calibration_scores, self.estimator
+                )
                 for i in range(len(score_splits)):
                     # create a calibrator for each split
-                    calibrator = self._create_calibrator(self.method, self.method_params)
+                    calibrator = self._create_calibrator(
+                        self.method, self.method_params
+                    )
                     calibrator.fit(label_splits[i], score_splits[i], X)
                     self.calibrators.append(calibrator)
 
diff --git a/hiclass/_calibration/VennAbersCalibrator.py b/hiclass/_calibration/VennAbersCalibrator.py
index d9e06f2b..6a799ee8 100644
--- a/hiclass/_calibration/VennAbersCalibrator.py
+++ b/hiclass/_calibration/VennAbersCalibrator.py
@@ -24,9 +24,17 @@ def fit(self, y: np.ndarray, scores: np.ndarray, X: np.ndarray = None):
 
         # sort all scores s1, ..., sk in increasing order
         order_idx = np.lexsort([y, scores])
-        ordered_calibration_scores, ordered_calibration_labels = scores[order_idx], y[order_idx]
-        unique_elements, unique_idx, unique_element_counts = np.unique(ordered_calibration_scores, return_index=True, return_counts=True)
-        ordered_unique_calibration_scores, _ = ordered_calibration_scores[unique_idx], ordered_calibration_labels[unique_idx]
+        ordered_calibration_scores, ordered_calibration_labels = (
+            scores[order_idx],
+            y[order_idx],
+        )
+        unique_elements, unique_idx, unique_element_counts = np.unique(
+            ordered_calibration_scores, return_index=True, return_counts=True
+        )
+        ordered_unique_calibration_scores, _ = (
+            ordered_calibration_scores[unique_idx],
+            ordered_calibration_labels[unique_idx],
+        )
 
         self.k_distinct = len(unique_idx)
 
@@ -46,7 +54,9 @@ def fit(self, y: np.ndarray, scores: np.ndarray, X: np.ndarray = None):
         csd_1[1:, 1] = np.cumsum(y * unique_element_counts)
 
         csd_0 = csd_1.copy()
-        csd_0 = np.append(csd_0, [np.array([csd_0[-1][0] + 1, csd_0[-1][1] + 0])], axis=0)
+        csd_0 = np.append(
+            csd_0, [np.array([csd_0[-1][0] + 1, csd_0[-1][1] + 0])], axis=0
+        )
         csd_1 = np.insert(csd_1, 0, [np.array([(-1, -1)])], axis=0)
 
         f1_stack = self._initialize_f1_corners(csd_1)
@@ -74,7 +84,9 @@ def _initialize_f1_corners(self, csd):
         stack.append(csd[1])
 
         for i in range(2, len(csd)):
-            while len(stack) > 1 and self._non_left_angle_turn(next_to_top=stack[-2], top=stack[-1], p_i=csd[i]):
+            while len(stack) > 1 and self._non_left_angle_turn(
+                next_to_top=stack[-2], top=stack[-1], p_i=csd[i]
+            ):
                 stack.pop()
             stack.append(csd[i])
 
@@ -87,7 +99,9 @@ def _initialize_f0_corners(self, csd):
         stack.append(csd[-2])
 
         for i in range(len(csd) - 3, -1, -1):
-            while len(stack) > 1 and self._non_right_angle_turn(next_to_top=stack[-2], top=stack[-1], p_i=csd[i]):
+            while len(stack) > 1 and self._non_right_angle_turn(
+                next_to_top=stack[-2], top=stack[-1], p_i=csd[i]
+            ):
                 stack.pop()
             stack.append(csd[i])
         return stack
@@ -115,7 +129,9 @@ def _compute_f1(self, prev_stack, csd):
                 continue
 
             stack.pop()
-            while len(stack) > 1 and self._non_left_angle_turn(p_temp, stack[-1], stack[-2]):
+            while len(stack) > 1 and self._non_left_angle_turn(
+                p_temp, stack[-1], stack[-2]
+            ):
                 stack.pop()
             stack.append(p_temp)
         return F1
@@ -133,7 +149,9 @@ def _compute_f0(self, prev_stack, csd):
             if self._at_or_above(csd[i], F0[i], top=stack[-1]):
                 continue
             stack.pop()
-            while len(stack) > 1 and self._non_right_angle_turn(csd[i], stack[-1], stack[-2]):
+            while len(stack) > 1 and self._non_right_angle_turn(
+                csd[i], stack[-1], stack[-2]
+            ):
                 stack.pop()
             stack.append(csd[i])
         return F0
@@ -186,13 +204,20 @@ def fit(self, y: np.ndarray, scores: np.ndarray, X: np.ndarray):
             splits_x, splits_y = [], []
 
         # don't use cross validation
-        if len(splits_x) == 0 or any([(len(np.unique(y_train)) < 2 or len(np.unique(y_cal)) < 2) for y_train, y_cal in splits_y]):
+        if len(splits_x) == 0 or any(
+            [
+                (len(np.unique(y_train)) < 2 or len(np.unique(y_cal)) < 2)
+                for y_train, y_cal in splits_y
+            ]
+        ):
             self.used_cv = False
             print("skip cv split due to lack of positive samples!")
 
             if len(unique_labels) > 2:
                 # use one vs rest
-                score_splits, label_splits = _one_vs_rest_split(y, scores, self.estimator)  # TODO use only original calibration samples
+                score_splits, label_splits = _one_vs_rest_split(
+                    y, scores, self.estimator
+                )  # TODO use only original calibration samples
                 for i in range(len(score_splits)):
                     # create a calibrator for each split
                     calibrator = _InductiveVennAbersCalibrator()
@@ -223,7 +248,9 @@ def fit(self, y: np.ndarray, scores: np.ndarray, X: np.ndarray):
                 if calibration_scores.shape[1] > 2:
                     self.multiclass = True
                     # one vs rest calibration
-                    score_splits, label_splits = _one_vs_rest_split(y_cal, calibration_scores, model)
+                    score_splits, label_splits = _one_vs_rest_split(
+                        y_cal, calibration_scores, model
+                    )
                     for idx in range(len(score_splits)):
                         # create a calibrator for each split
                         calibrator = _InductiveVennAbersCalibrator()
diff --git a/hiclass/metrics.py b/hiclass/metrics.py
index 68c368c1..71b2f497 100644
--- a/hiclass/metrics.py
+++ b/hiclass/metrics.py
@@ -254,7 +254,13 @@ def _compute_macro(y_true: np.ndarray, y_pred: np.ndarray, _micro_function):
     return overall_sum / len(y_true)
 
 
-def _prepare_data(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, level: int, y_pred: np.ndarray = None):
+def _prepare_data(
+    classifier: HierarchicalClassifier,
+    y_true: np.ndarray,
+    y_prob: np.ndarray,
+    level: int,
+    y_pred: np.ndarray = None,
+):
     classifier_classes = np.array(classifier.classes_[level]).astype("str")
     y_true = make_leveled(y_true)
     y_true = classifier._disambiguate(y_true)
@@ -265,7 +271,9 @@ def _prepare_data(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob
         y_pred = make_leveled(y_pred)
         y_pred = classifier._disambiguate(y_pred)
         y_pred = np.array(list(map(lambda x: x[level], y_pred)))
-        y_pred = np.array([label.split(classifier.separator_)[level] for label in y_pred])
+        y_pred = np.array(
+            [label.split(classifier.separator_)[level] for label in y_pred]
+        )
 
     unique_labels = np.unique(y_true).astype("str")
     # add labels not seen in the training process
@@ -284,11 +292,11 @@ def _prepare_data(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob
 
 
 def _aggregate_scores(scores: np.ndarray, agg: str):
-    if agg == 'average':
+    if agg == "average":
         return np.mean(scores)
-    if agg == 'sum':
+    if agg == "sum":
         return np.sum(scores)
-    if agg is None or agg == 'None':
+    if agg is None or agg == "None":
         return scores
 
 
@@ -297,14 +305,24 @@ def _validate_args(agg: str, y_prob: np.ndarray, level: int):
         raise ValueError(f"{agg} is not a valid aggregation function.")
     if isinstance(y_prob, list) and len(y_prob) == 0:
         raise ValueError("y_prob is empty.")
-    if (isinstance(y_prob, list) and len(y_prob) == 1 or isinstance(y_prob, np.ndarray)) and level is None:
-        raise ValueError("If y_prob is not a list of probabilities the level must be specified.")
+    if (
+        isinstance(y_prob, list) and len(y_prob) == 1 or isinstance(y_prob, np.ndarray)
+    ) and level is None:
+        raise ValueError(
+            "If y_prob is not a list of probabilities the level must be specified."
+        )
     if isinstance(y_prob, list) and len(y_prob) == 1:
         return y_prob[0]
     return y_prob
 
 
-def multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Union[np.ndarray | List], agg: str = 'average', level: int = None):
+def multiclass_brier_score(
+    classifier: HierarchicalClassifier,
+    y_true: np.ndarray,
+    y_prob: Union[np.ndarray | List],
+    agg: str = "average",
+    level: int = None,
+):
     """Compute the brier score for two or more classes.
 
     Parameters
@@ -332,12 +350,20 @@ def multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarra
     if isinstance(y_prob, list):
         scores = []
         for level in range(make_leveled(y_true).shape[1]):
-            scores.append(_multiclass_brier_score(classifier, y_true, y_prob[level], level))
+            scores.append(
+                _multiclass_brier_score(classifier, y_true, y_prob[level], level)
+            )
         return _aggregate_scores(scores, agg)
     return _multiclass_brier_score(classifier, y_true, y_prob, level)
 
 
-def log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Union[np.ndarray | List], agg: str = 'average', level: int = None):
+def log_loss(
+    classifier: HierarchicalClassifier,
+    y_true: np.ndarray,
+    y_prob: Union[np.ndarray | List],
+    agg: str = "average",
+    level: int = None,
+):
     """Compute the log loss of predicted probabilities.
 
     Parameters
@@ -370,7 +396,15 @@ def log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Uni
     return _multiclass_brier_score(classifier, y_true, y_prob, level)
 
 
-def expected_calibration_error(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Union[np.ndarray | List], y_pred: np.ndarray, n_bins: int = 10, agg: str = 'average', level: int = None):
+def expected_calibration_error(
+    classifier: HierarchicalClassifier,
+    y_true: np.ndarray,
+    y_prob: Union[np.ndarray | List],
+    y_pred: np.ndarray,
+    n_bins: int = 10,
+    agg: str = "average",
+    level: int = None,
+):
     """Compute the expected calibration error.
 
     Parameters
@@ -402,12 +436,26 @@ def expected_calibration_error(classifier: HierarchicalClassifier, y_true: np.nd
     if isinstance(y_prob, list):
         scores = []
         for level in range(make_leveled(y_true).shape[1]):
-            scores.append(_expected_calibration_error(classifier, y_true, y_prob[level], y_pred, level, n_bins))
+            scores.append(
+                _expected_calibration_error(
+                    classifier, y_true, y_prob[level], y_pred, level, n_bins
+                )
+            )
         return _aggregate_scores(scores, agg)
-    return _expected_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins)
-
-
-def static_calibration_error(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Union[np.ndarray | List], y_pred: np.ndarray, n_bins: int = 10, agg: str = 'average', level: int = None):
+    return _expected_calibration_error(
+        classifier, y_true, y_prob, y_pred, level, n_bins
+    )
+
+
+def static_calibration_error(
+    classifier: HierarchicalClassifier,
+    y_true: np.ndarray,
+    y_prob: Union[np.ndarray | List],
+    y_pred: np.ndarray,
+    n_bins: int = 10,
+    agg: str = "average",
+    level: int = None,
+):
     """Compute the static calibration error.
 
     Parameters
@@ -439,12 +487,26 @@ def static_calibration_error(classifier: HierarchicalClassifier, y_true: np.ndar
     if isinstance(y_prob, list):
         scores = []
         for level in range(make_leveled(y_true).shape[1]):
-            scores.append(_static_calibration_error(classifier, y_true, y_prob[level], y_pred, level, n_bins=n_bins))
+            scores.append(
+                _static_calibration_error(
+                    classifier, y_true, y_prob[level], y_pred, level, n_bins=n_bins
+                )
+            )
         return _aggregate_scores(scores, agg)
-    return _static_calibration_error(classifier, y_true, y_prob, y_pred, level, n_bins=n_bins)
-
-
-def adaptive_calibration_error(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: Union[np.ndarray | List], y_pred: np.ndarray, n_ranges: int = 10, agg: str = 'average', level: int = None):
+    return _static_calibration_error(
+        classifier, y_true, y_prob, y_pred, level, n_bins=n_bins
+    )
+
+
+def adaptive_calibration_error(
+    classifier: HierarchicalClassifier,
+    y_true: np.ndarray,
+    y_prob: Union[np.ndarray | List],
+    y_pred: np.ndarray,
+    n_ranges: int = 10,
+    agg: str = "average",
+    level: int = None,
+):
     """Compute the adaptive calibration error.
 
     Parameters
@@ -476,26 +538,53 @@ def adaptive_calibration_error(classifier: HierarchicalClassifier, y_true: np.nd
     if isinstance(y_prob, list):
         scores = []
         for level in range(make_leveled(y_true).shape[1]):
-            scores.append(_adaptive_calibration_error(classifier, y_true, y_prob[level], y_pred, level, n_ranges=n_ranges))
+            scores.append(
+                _adaptive_calibration_error(
+                    classifier, y_true, y_prob[level], y_pred, level, n_ranges=n_ranges
+                )
+            )
         return _aggregate_scores(scores, agg)
-    return _adaptive_calibration_error(classifier, y_true, y_prob, y_pred, level, n_ranges=n_ranges)
+    return _adaptive_calibration_error(
+        classifier, y_true, y_prob, y_pred, level, n_ranges=n_ranges
+    )
 
 
-def _multiclass_brier_score(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, level: int):
+def _multiclass_brier_score(
+    classifier: HierarchicalClassifier,
+    y_true: np.ndarray,
+    y_prob: np.ndarray,
+    level: int,
+):
     y_true, _, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level)
     label_encoder = LabelEncoder()
     label_encoder.fit(labels)
     y_true_encoded = label_encoder.transform(y_true)
-    return (1 / y_prob.shape[0]) * np.sum(np.sum(np.square(y_prob - np.eye(y_prob.shape[1])[y_true_encoded]), axis=1))
+    return (1 / y_prob.shape[0]) * np.sum(
+        np.sum(np.square(y_prob - np.eye(y_prob.shape[1])[y_true_encoded]), axis=1)
+    )
 
 
-def _log_loss(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, level: int):
+def _log_loss(
+    classifier: HierarchicalClassifier,
+    y_true: np.ndarray,
+    y_prob: np.ndarray,
+    level: int,
+):
     y_true, _, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level)
     return sk_log_loss(y_true, y_prob, labels=labels)
 
 
-def _expected_calibration_error(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, y_pred: np.ndarray, level: int, n_bins: int = 10):
-    y_true, y_pred, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level, y_pred)
+def _expected_calibration_error(
+    classifier: HierarchicalClassifier,
+    y_true: np.ndarray,
+    y_prob: np.ndarray,
+    y_pred: np.ndarray,
+    level: int,
+    n_bins: int = 10,
+):
+    y_true, y_pred, labels, y_prob = _prepare_data(
+        classifier, y_true, y_prob, level, y_pred
+    )
 
     n = len(y_true)
     label_encoder = LabelEncoder()
@@ -525,14 +614,35 @@ def _expected_calibration_error(classifier: HierarchicalClassifier, y_true: np.n
     conf = np.zeros(n_bins)
     ece = 0
     for i in range(n_bins):
-        acc[i] = 1 / (bins[i].shape[0]) * np.sum((bins[i][:, 1] == bins[i][:, 2])) if bins[i].shape[0] != 0 else 0
-        conf[i] = 1 / (bins[i].shape[0]) * np.sum(bins[i][:, 0]) if bins[i].shape[0] != 0 else 0
-        ece += (bins[i].shape[0] / n) * abs(acc[i] - conf[i]) if bins[i].shape[0] != 0 else 0
+        acc[i] = (
+            1 / (bins[i].shape[0]) * np.sum((bins[i][:, 1] == bins[i][:, 2]))
+            if bins[i].shape[0] != 0
+            else 0
+        )
+        conf[i] = (
+            1 / (bins[i].shape[0]) * np.sum(bins[i][:, 0])
+            if bins[i].shape[0] != 0
+            else 0
+        )
+        ece += (
+            (bins[i].shape[0] / n) * abs(acc[i] - conf[i])
+            if bins[i].shape[0] != 0
+            else 0
+        )
     return ece
 
 
-def _static_calibration_error(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, y_pred: np.ndarray, level: int, n_bins: int = 10):
-    y_true, y_pred, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level, y_pred)
+def _static_calibration_error(
+    classifier: HierarchicalClassifier,
+    y_true: np.ndarray,
+    y_prob: np.ndarray,
+    y_pred: np.ndarray,
+    level: int,
+    n_bins: int = 10,
+):
+    y_true, y_pred, labels, y_prob = _prepare_data(
+        classifier, y_true, y_prob, level, y_pred
+    )
 
     n_samples, n_classes = y_prob.shape
     assert n_classes > 2
@@ -567,17 +677,38 @@ def _static_calibration_error(classifier: HierarchicalClassifier, y_true: np.nda
         conf = np.zeros(n_bins)
         error = 0
         for i in range(n_bins):
-            acc[i] = 1 / (bins[i].shape[0]) * np.sum((bins[i][:, 1] == bins[i][:, 2])) if bins[i].shape[0] != 0 else 0
-            conf[i] = 1 / (bins[i].shape[0]) * np.sum(bins[i][:, 0]) if bins[i].shape[0] != 0 else 0
-            error += (bins[i].shape[0] / n_samples) * abs(acc[i] - conf[i]) if bins[i].shape[0] != 0 else 0
+            acc[i] = (
+                1 / (bins[i].shape[0]) * np.sum((bins[i][:, 1] == bins[i][:, 2]))
+                if bins[i].shape[0] != 0
+                else 0
+            )
+            conf[i] = (
+                1 / (bins[i].shape[0]) * np.sum(bins[i][:, 0])
+                if bins[i].shape[0] != 0
+                else 0
+            )
+            error += (
+                (bins[i].shape[0] / n_samples) * abs(acc[i] - conf[i])
+                if bins[i].shape[0] != 0
+                else 0
+            )
 
         class_error[k] = error
 
     return np.mean(class_error)
 
 
-def _adaptive_calibration_error(classifier: HierarchicalClassifier, y_true: np.ndarray, y_prob: np.ndarray, y_pred: np.ndarray, level: int, n_ranges: int = 10):
-    y_true, y_pred, labels, y_prob = _prepare_data(classifier, y_true, y_prob, level, y_pred)
+def _adaptive_calibration_error(
+    classifier: HierarchicalClassifier,
+    y_true: np.ndarray,
+    y_prob: np.ndarray,
+    y_pred: np.ndarray,
+    level: int,
+    n_ranges: int = 10,
+):
+    y_true, y_pred, labels, y_prob = _prepare_data(
+        classifier, y_true, y_prob, level, y_pred
+    )
 
     _, n_classes = y_prob.shape
     label_encoder = LabelEncoder()
@@ -593,11 +724,26 @@ def _adaptive_calibration_error(classifier: HierarchicalClassifier, y_true: np.n
 
         # sort by score probability
         idx = np.argsort([class_scores])[0]
-        class_scores, ordered_y_pred_labels, ordered_y_true = class_scores[idx], y_pred_encoded[idx], y_true_encoded[idx]
-        stacked = np.column_stack([np.array(range(len(class_scores))), class_scores, ordered_y_pred_labels, ordered_y_true])
+        class_scores, ordered_y_pred_labels, ordered_y_true = (
+            class_scores[idx],
+            y_pred_encoded[idx],
+            y_true_encoded[idx],
+        )
+        stacked = np.column_stack(
+            [
+                np.array(range(len(class_scores))),
+                class_scores,
+                ordered_y_pred_labels,
+                ordered_y_true,
+            ]
+        )
 
-        bin_edges = np.floor(np.linspace(0, len(class_scores), n_ranges + 1, endpoint=True)).astype(int)
-        _, bin_edges = np.histogram(stacked, bins=bin_edges, range=(0, len(class_scores)))
+        bin_edges = np.floor(
+            np.linspace(0, len(class_scores), n_ranges + 1, endpoint=True)
+        ).astype(int)
+        _, bin_edges = np.histogram(
+            stacked, bins=bin_edges, range=(0, len(class_scores))
+        )
         bin_indices = np.digitize(stacked, bin_edges)[:, 0]
 
         # add bin index to each data point
@@ -614,8 +760,16 @@ def _adaptive_calibration_error(classifier: HierarchicalClassifier, y_true: np.n
         conf = np.zeros(n_ranges)
         error = 0
         for i in range(n_ranges):
-            acc[i] = 1 / (bins[i].shape[0]) * np.sum((bins[i][:, 2] == bins[i][:, 3])) if bins[i].shape[0] != 0 else 0
-            conf[i] = 1 / (bins[i].shape[0]) * np.sum(bins[i][:, 1]) if bins[i].shape[0] != 0 else 0
+            acc[i] = (
+                1 / (bins[i].shape[0]) * np.sum((bins[i][:, 2] == bins[i][:, 3]))
+                if bins[i].shape[0] != 0
+                else 0
+            )
+            conf[i] = (
+                1 / (bins[i].shape[0]) * np.sum(bins[i][:, 1])
+                if bins[i].shape[0] != 0
+                else 0
+            )
             error += abs(acc[i] - conf[i]) if bins[i].shape[0] != 0 else 0
 
         class_error[k] = error
diff --git a/hiclass/probability_combiner/ArithmeticMeanCombiner.py b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
index de50a02e..01419bfc 100644
--- a/hiclass/probability_combiner/ArithmeticMeanCombiner.py
+++ b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
@@ -23,10 +23,21 @@ def combine(self, proba: List[np.ndarray]):
             for node in predecessors.keys():
                 index = self.classifier.class_to_index_mapping_[level][node]
                 # find indices of all predecessors
-                predecessor_indices = [self.classifier.class_to_index_mapping_[level - 1][predecessor] for predecessor in predecessors[node]]
+                predecessor_indices = [
+                    self.classifier.class_to_index_mapping_[level - 1][predecessor]
+                    for predecessor in predecessors[node]
+                ]
                 # combine probabilities of all predecessors
-                predecessors_combined_prob = np.sum([sums[level - 1][:, pre_index] for pre_index in predecessor_indices], axis=0)
-                level_sum[:, index] += proba[level][:, index] + predecessors_combined_prob
+                predecessors_combined_prob = np.sum(
+                    [
+                        sums[level - 1][:, pre_index]
+                        for pre_index in predecessor_indices
+                    ],
+                    axis=0,
+                )
+                level_sum[:, index] += (
+                    proba[level][:, index] + predecessors_combined_prob
+                )
                 level_probs[:, index] = level_sum[:, index] / (level + 1)
 
             res.append(level_probs)
diff --git a/hiclass/probability_combiner/GeometricMeanCombiner.py b/hiclass/probability_combiner/GeometricMeanCombiner.py
index f3c49cb2..2a700983 100644
--- a/hiclass/probability_combiner/GeometricMeanCombiner.py
+++ b/hiclass/probability_combiner/GeometricMeanCombiner.py
@@ -23,11 +23,24 @@ def combine(self, proba: List[np.ndarray]):
             for node in predecessors.keys():
                 index = self.classifier.class_to_index_mapping_[level][node]
                 # find indices of all predecessors
-                predecessor_indices = [self.classifier.class_to_index_mapping_[level - 1][predecessor] for predecessor in predecessors[node]]
+                predecessor_indices = [
+                    self.classifier.class_to_index_mapping_[level - 1][predecessor]
+                    for predecessor in predecessors[node]
+                ]
                 # combine probabilities of all predecessors
-                predecessors_combined_log_prob = np.log(np.sum([np.exp(log_sum[level - 1][:, pre_index]) for pre_index in predecessor_indices], axis=0))
+                predecessors_combined_log_prob = np.log(
+                    np.sum(
+                        [
+                            np.exp(log_sum[level - 1][:, pre_index])
+                            for pre_index in predecessor_indices
+                        ],
+                        axis=0,
+                    )
+                )
 
-                level_log_sum[:, index] += (np.log(proba[level][:, index]) + predecessors_combined_log_prob)
+                level_log_sum[:, index] += (
+                    np.log(proba[level][:, index]) + predecessors_combined_log_prob
+                )
                 level_probs[:, index] = np.exp(level_log_sum[:, index] / (level + 1))
 
             log_sum.append(level_log_sum)
diff --git a/hiclass/probability_combiner/MultiplyCombiner.py b/hiclass/probability_combiner/MultiplyCombiner.py
index 386ecdf9..36badb82 100644
--- a/hiclass/probability_combiner/MultiplyCombiner.py
+++ b/hiclass/probability_combiner/MultiplyCombiner.py
@@ -21,10 +21,18 @@ def combine(self, proba: List[np.ndarray]):
             for node in predecessors.keys():
                 index = self.classifier.class_to_index_mapping_[level][node]
                 # find indices of all predecessors
-                predecessor_indices = [self.classifier.class_to_index_mapping_[level - 1][predecessor] for predecessor in predecessors[node]]
+                predecessor_indices = [
+                    self.classifier.class_to_index_mapping_[level - 1][predecessor]
+                    for predecessor in predecessors[node]
+                ]
                 # combine probabilities of all predecessors
-                predecessors_combined_prob = np.sum([res[level - 1][:, pre_index] for pre_index in predecessor_indices], axis=0)
-                level_probs[:, index] = predecessors_combined_prob * proba[level][:, index]
+                predecessors_combined_prob = np.sum(
+                    [res[level - 1][:, pre_index] for pre_index in predecessor_indices],
+                    axis=0,
+                )
+                level_probs[:, index] = (
+                    predecessors_combined_prob * proba[level][:, index]
+                )
 
             res.append(level_probs)
         return self._normalize(res) if self.normalize else res
diff --git a/hiclass/probability_combiner/ProbabilityCombiner.py b/hiclass/probability_combiner/ProbabilityCombiner.py
index 4b5b77e3..be9569f8 100644
--- a/hiclass/probability_combiner/ProbabilityCombiner.py
+++ b/hiclass/probability_combiner/ProbabilityCombiner.py
@@ -10,7 +10,9 @@
 class ProbabilityCombiner(abc.ABC):
     """Abstract class defining the structure of a probability combiner."""
 
-    def __init__(self, classifier: HierarchicalClassifier, normalize: bool = True) -> None:
+    def __init__(
+        self, classifier: HierarchicalClassifier, normalize: bool = True
+    ) -> None:
         """Initialize probability combiner object."""
         self.classifier = classifier
         self.normalize = normalize
@@ -22,7 +24,9 @@ def combine(self, proba: List[np.ndarray]) -> List[np.ndarray]:
 
     def _normalize(self, proba: List[np.ndarray]):
         return [
-            np.nan_to_num(level_probabilities / level_probabilities.sum(axis=1, keepdims=True))
+            np.nan_to_num(
+                level_probabilities / level_probabilities.sum(axis=1, keepdims=True)
+            )
             for level_probabilities in proba
         ]
 
diff --git a/hiclass/probability_combiner/__init__.py b/hiclass/probability_combiner/__init__.py
index 1d7e069d..bf762883 100644
--- a/hiclass/probability_combiner/__init__.py
+++ b/hiclass/probability_combiner/__init__.py
@@ -14,4 +14,4 @@
     "multiply",
     "geometric",
     "arithmetic",
-]
\ No newline at end of file
+]

From 2d0b9db9828f1ce6542429a463d8161cd55707de Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Thu, 11 Apr 2024 16:34:52 +0200
Subject: [PATCH 48/65] make metrics type hints compatible with older python
 versions

---
 hiclass/metrics.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/hiclass/metrics.py b/hiclass/metrics.py
index 71b2f497..d05940e1 100644
--- a/hiclass/metrics.py
+++ b/hiclass/metrics.py
@@ -319,7 +319,7 @@ def _validate_args(agg: str, y_prob: np.ndarray, level: int):
 def multiclass_brier_score(
     classifier: HierarchicalClassifier,
     y_true: np.ndarray,
-    y_prob: Union[np.ndarray | List],
+    y_prob: Union[np.ndarray, List],
     agg: str = "average",
     level: int = None,
 ):
@@ -360,7 +360,7 @@ def multiclass_brier_score(
 def log_loss(
     classifier: HierarchicalClassifier,
     y_true: np.ndarray,
-    y_prob: Union[np.ndarray | List],
+    y_prob: Union[np.ndarray, List],
     agg: str = "average",
     level: int = None,
 ):
@@ -399,7 +399,7 @@ def log_loss(
 def expected_calibration_error(
     classifier: HierarchicalClassifier,
     y_true: np.ndarray,
-    y_prob: Union[np.ndarray | List],
+    y_prob: Union[np.ndarray, List],
     y_pred: np.ndarray,
     n_bins: int = 10,
     agg: str = "average",
@@ -450,7 +450,7 @@ def expected_calibration_error(
 def static_calibration_error(
     classifier: HierarchicalClassifier,
     y_true: np.ndarray,
-    y_prob: Union[np.ndarray | List],
+    y_prob: Union[np.ndarray, List],
     y_pred: np.ndarray,
     n_bins: int = 10,
     agg: str = "average",
@@ -501,7 +501,7 @@ def static_calibration_error(
 def adaptive_calibration_error(
     classifier: HierarchicalClassifier,
     y_true: np.ndarray,
-    y_prob: Union[np.ndarray | List],
+    y_prob: Union[np.ndarray, List],
     y_pred: np.ndarray,
     n_ranges: int = 10,
     agg: str = "average",

From 121e1347c95389d3534955056d752838dc76d244 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Thu, 11 Apr 2024 22:08:05 +0200
Subject: [PATCH 49/65] black formatting for tests

---
 tests/test_LocalClassifierPerLevel.py      | 105 +++++--
 tests/test_LocalClassifierPerNode.py       | 168 ++++++++---
 tests/test_LocalClassifierPerParentNode.py | 123 ++++++--
 tests/test_ProbabilityCombiner.py          | 204 ++++++++++---
 tests/test_calibration.py                  | 336 +++++++++++++++------
 tests/test_metrics.py                      |  93 ++++--
 6 files changed, 756 insertions(+), 273 deletions(-)

diff --git a/tests/test_LocalClassifierPerLevel.py b/tests/test_LocalClassifierPerLevel.py
index 6286a06d..880fc904 100644
--- a/tests/test_LocalClassifierPerLevel.py
+++ b/tests/test_LocalClassifierPerLevel.py
@@ -21,7 +21,9 @@ def test_sklearn_compatible_estimator(estimator, check):
 
 @pytest.fixture
 def digraph_logistic_regression():
-    digraph = LocalClassifierPerLevel(local_classifier=LogisticRegression(), calibration_method="ivap")
+    digraph = LocalClassifierPerLevel(
+        local_classifier=LogisticRegression(), calibration_method="ivap"
+    )
     digraph.hierarchy_ = nx.DiGraph([("a", "b"), ("a", "c")])
     digraph.y_ = np.array([["a", "b"], ["a", "c"]])
     digraph.X_ = np.array([[1, 2], [3, 4]])
@@ -46,14 +48,12 @@ def test_initialize_local_classifiers(digraph_logistic_regression):
             LogisticRegression,
         )
 
+
 def test_initialize_local_calibrators(digraph_logistic_regression):
     digraph_logistic_regression._initialize_local_classifiers()
     digraph_logistic_regression._initialize_local_calibrators()
     for calibrator in digraph_logistic_regression.local_calibrators_:
-        assert isinstance(
-            calibrator,
-            _Calibrator
-        )
+        assert isinstance(calibrator, _Calibrator)
 
 
 def test_fit_digraph(digraph_logistic_regression):
@@ -71,6 +71,7 @@ def test_fit_digraph(digraph_logistic_regression):
             pytest.fail(repr(e))
     assert 1
 
+
 def test_calibrate_digraph(digraph_logistic_regression):
     classifiers = [
         LogisticRegression(),
@@ -80,7 +81,10 @@ def test_calibrate_digraph(digraph_logistic_regression):
     digraph_logistic_regression.local_classifiers_ = classifiers
     digraph_logistic_regression._fit_digraph(local_mode=True)
 
-    calibrators = [_Calibrator(classifier) for classifier in digraph_logistic_regression.local_classifiers_]
+    calibrators = [
+        _Calibrator(classifier)
+        for classifier in digraph_logistic_regression.local_classifiers_
+    ]
     digraph_logistic_regression.local_calibrators_ = calibrators
     digraph_logistic_regression._calibrate_digraph(local_mode=True)
 
@@ -91,7 +95,6 @@ def test_calibrate_digraph(digraph_logistic_regression):
     assert 1
 
 
-
 def test_fit_digraph_joblib_multiprocessing(digraph_logistic_regression):
     classifiers = [
         LogisticRegression(),
@@ -108,6 +111,7 @@ def test_fit_digraph_joblib_multiprocessing(digraph_logistic_regression):
             pytest.fail(repr(e))
     assert 1
 
+
 def test_calibrate_digraph_joblib_multiprocessing(digraph_logistic_regression):
     classifiers = [
         LogisticRegression(),
@@ -117,7 +121,10 @@ def test_calibrate_digraph_joblib_multiprocessing(digraph_logistic_regression):
     digraph_logistic_regression.local_classifiers_ = classifiers
     digraph_logistic_regression._fit_digraph(local_mode=True, use_joblib=True)
 
-    calibrators = [_Calibrator(classifier) for classifier in digraph_logistic_regression.local_classifiers_]
+    calibrators = [
+        _Calibrator(classifier)
+        for classifier in digraph_logistic_regression.local_classifiers_
+    ]
     digraph_logistic_regression.local_calibrators_ = calibrators
     digraph_logistic_regression._calibrate_digraph(local_mode=True, use_joblib=True)
 
@@ -128,15 +135,15 @@ def test_calibrate_digraph_joblib_multiprocessing(digraph_logistic_regression):
     assert 1
 
 
-
 @pytest.fixture
 def fitted_logistic_regression():
     digraph = LocalClassifierPerLevel(
         local_classifier=LogisticRegression(),
         return_all_probabilities=True,
         calibration_method="ivap",
-        probability_combiner=None)
-    
+        probability_combiner=None,
+    )
+
     digraph.separator_ = "::HiClass::Separator::"
     digraph.hierarchy_ = nx.DiGraph(
         [("r", "1"), ("r", "2"), ("1", "1.1"), ("1", "1.2"), ("2", "2.1"), ("2", "2.2")]
@@ -148,16 +155,38 @@ def fitted_logistic_regression():
 
     # for predict_proba
     tmp_labels = digraph._disambiguate(make_leveled(digraph.y_))
-    digraph.max_level_dimensions_ = np.array([len(np.unique(tmp_labels[:, level])) for level in range(tmp_labels.shape[1])])
-    digraph.global_classes_ = [np.unique(tmp_labels[:, level]).astype("str") for level in range(tmp_labels.shape[1])]
-    digraph.global_class_to_index_mapping_ = [{digraph.global_classes_[level][index]: index for index in range(len(digraph.global_classes_[level]))} for level in range(tmp_labels.shape[1])]
+    digraph.max_level_dimensions_ = np.array(
+        [len(np.unique(tmp_labels[:, level])) for level in range(tmp_labels.shape[1])]
+    )
+    digraph.global_classes_ = [
+        np.unique(tmp_labels[:, level]).astype("str")
+        for level in range(tmp_labels.shape[1])
+    ]
+    digraph.global_class_to_index_mapping_ = [
+        {
+            digraph.global_classes_[level][index]: index
+            for index in range(len(digraph.global_classes_[level]))
+        }
+        for level in range(tmp_labels.shape[1])
+    ]
 
     classes_ = [digraph.global_classes_[0]]
     for level in range(1, digraph.max_levels_):
-        classes_.append(np.sort(np.unique([label.split(digraph.separator_)[level] for label in digraph.global_classes_[level]])))
+        classes_.append(
+            np.sort(
+                np.unique(
+                    [
+                        label.split(digraph.separator_)[level]
+                        for label in digraph.global_classes_[level]
+                    ]
+                )
+            )
+        )
     digraph.classes_ = classes_
-    digraph.class_to_index_mapping_ = [{local_labels[index]: index for index in range(len(local_labels))} for local_labels in classes_]
-
+    digraph.class_to_index_mapping_ = [
+        {local_labels[index]: index for index in range(len(local_labels))}
+        for local_labels in classes_
+    ]
 
     digraph.dtype_ = "<U3"
     digraph.root_ = "r"
@@ -194,17 +223,28 @@ def test_predict_proba(fitted_logistic_regression):
     assert len(proba) == 2
     assert proba[0].shape == (4, 2)
     assert proba[1].shape == (4, 4)
-    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
-    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+    assert_array_almost_equal(
+        np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10
+    )
+    assert_array_almost_equal(
+        np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10
+    )
 
 
 def test_predict_proba_sparse(fitted_logistic_regression):
-    proba = fitted_logistic_regression.predict_proba(csr_matrix([[7, 8], [5, 6], [3, 4], [1, 2]]))
+    proba = fitted_logistic_regression.predict_proba(
+        csr_matrix([[7, 8], [5, 6], [3, 4], [1, 2]])
+    )
     assert len(proba) == 2
     assert proba[0].shape == (4, 2)
     assert proba[1].shape == (4, 4)
-    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
-    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+    assert_array_almost_equal(
+        np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10
+    )
+    assert_array_almost_equal(
+        np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10
+    )
+
 
 def test_fit_predict():
     lcpl = LocalClassifierPerLevel(local_classifier=LogisticRegression())
@@ -223,13 +263,15 @@ def test_fit_predict():
     predictions = lcpl.predict(x)
     assert_array_equal(y, predictions)
 
+
 def test_fit_calibrate_predict_proba():
     lcpl = LocalClassifierPerLevel(
-        local_classifier=LogisticRegression(), 
+        local_classifier=LogisticRegression(),
         return_all_probabilities=True,
         calibration_method="ivap",
-        probability_combiner="geometric")
-    
+        probability_combiner="geometric",
+    )
+
     x = np.array([[1, 2], [3, 4]])
     y = np.array([["a", "b"], ["b", "c"]])
     ground_truth = np.array(
@@ -250,15 +292,20 @@ def test_fit_calibrate_predict_proba():
     assert len(proba) == 2
     assert proba[0].shape == (2, 2)
     assert proba[1].shape == (2, 2)
-    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
-    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+    assert_array_almost_equal(
+        np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10
+    )
+    assert_array_almost_equal(
+        np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10
+    )
+
 
 def test_fit_calibrate_predict_predict_proba_bert():
     classifier = LocalClassifierPerLevel(
-        local_classifier=LogisticRegression(), 
+        local_classifier=LogisticRegression(),
         return_all_probabilities=True,
         calibration_method="ivap",
-        probability_combiner="geometric" 
+        probability_combiner="geometric",
     )
 
     classifier.logger_ = logging.getLogger("HC")
diff --git a/tests/test_LocalClassifierPerNode.py b/tests/test_LocalClassifierPerNode.py
index 533802c6..2bcdf45c 100644
--- a/tests/test_LocalClassifierPerNode.py
+++ b/tests/test_LocalClassifierPerNode.py
@@ -49,7 +49,9 @@ def digraph_with_unknown_policy():
 
 def test_initialize_unknown_binary_policy(digraph_with_unknown_policy):
     with pytest.raises(KeyError):
-        digraph_with_unknown_policy.binary_policy_ = digraph_with_unknown_policy._initialize_binary_policy()
+        digraph_with_unknown_policy.binary_policy_ = (
+            digraph_with_unknown_policy._initialize_binary_policy()
+        )
         digraph_with_unknown_policy._initialize_binary_policy()
 
 
@@ -69,7 +71,9 @@ def test_initialize_object_binary_policy(digraph_with_object_policy):
 
 @pytest.fixture
 def digraph_logistic_regression():
-    digraph = LocalClassifierPerNode(local_classifier=LogisticRegression(), calibration_method="ivap")
+    digraph = LocalClassifierPerNode(
+        local_classifier=LogisticRegression(), calibration_method="ivap"
+    )
     digraph.hierarchy_ = nx.DiGraph([("a", "b"), ("a", "c")])
     digraph.y_ = np.array([["a", "b"], ["a", "c"]])
     digraph.X_ = np.array([[1, 2], [3, 4]])
@@ -98,6 +102,7 @@ def test_initialize_local_classifiers(digraph_logistic_regression):
                     LogisticRegression,
                 )
 
+
 def test_initialize_local_calibrators(digraph_logistic_regression):
     digraph_logistic_regression._initialize_local_classifiers()
     digraph_logistic_regression._initialize_local_calibrators()
@@ -135,6 +140,7 @@ def test_fit_digraph(digraph_logistic_regression):
             pytest.fail(repr(e))
     assert 1
 
+
 def test_calibrate_digraph(digraph_logistic_regression):
     classifiers = {
         "b": {"classifier": LogisticRegression()},
@@ -145,12 +151,20 @@ def test_calibrate_digraph(digraph_logistic_regression):
     digraph_logistic_regression._fit_digraph(local_mode=True)
 
     calibrators = {
-        "b": {"calibrator": _Calibrator(digraph_logistic_regression.hierarchy_.nodes["b"]["classifier"])},
-        "c": {"calibrator": _Calibrator(digraph_logistic_regression.hierarchy_.nodes["c"]["classifier"])}
+        "b": {
+            "calibrator": _Calibrator(
+                digraph_logistic_regression.hierarchy_.nodes["b"]["classifier"]
+            )
+        },
+        "c": {
+            "calibrator": _Calibrator(
+                digraph_logistic_regression.hierarchy_.nodes["c"]["classifier"]
+            )
+        },
     }
     nx.set_node_attributes(digraph_logistic_regression.hierarchy_, calibrators)
     digraph_logistic_regression._calibrate_digraph(local_mode=True)
-    
+
     with pytest.raises(KeyError):
         check_is_fitted(digraph_logistic_regression.hierarchy_.nodes["a"]["calibrator"])
     for node in ["b", "c"]:
@@ -182,6 +196,7 @@ def test_fit_digraph_joblib_multiprocessing(digraph_logistic_regression):
             pytest.fail(repr(e))
     assert 1
 
+
 def test_calibrate_digraph_joblib_multiprocessing(digraph_logistic_regression):
     classifiers = {
         "b": {"classifier": LogisticRegression()},
@@ -192,8 +207,16 @@ def test_calibrate_digraph_joblib_multiprocessing(digraph_logistic_regression):
     digraph_logistic_regression._fit_digraph(local_mode=True, use_joblib=True)
 
     calibrators = {
-        "b": {"calibrator": _Calibrator(digraph_logistic_regression.hierarchy_.nodes["b"]["classifier"])},
-        "c": {"calibrator": _Calibrator(digraph_logistic_regression.hierarchy_.nodes["c"]["classifier"])}
+        "b": {
+            "calibrator": _Calibrator(
+                digraph_logistic_regression.hierarchy_.nodes["b"]["classifier"]
+            )
+        },
+        "c": {
+            "calibrator": _Calibrator(
+                digraph_logistic_regression.hierarchy_.nodes["c"]["classifier"]
+            )
+        },
     }
     nx.set_node_attributes(digraph_logistic_regression.hierarchy_, calibrators)
     digraph_logistic_regression._calibrate_digraph(local_mode=True, use_joblib=True)
@@ -228,27 +251,26 @@ def fitted_logistic_regression():
         local_classifier=LogisticRegression(),
         return_all_probabilities=True,
         calibration_method="ivap",
-        probability_combiner="geometric")
-    
+        probability_combiner="geometric",
+    )
+
     digraph.separator_ = "::HiClass::Separator::"
-    #digraph.hierarchy_ = nx.DiGraph(
+    # digraph.hierarchy_ = nx.DiGraph(
     #    [("r", "1"), ("r", "2"), ("1", "1.1"), ("1", "1.2"), ("2", "2.1"), ("2", "2.2")]
-    #)
+    # )
 
     digraph.hierarchy_ = nx.DiGraph(
-        [("r", "1"), 
-         ("r", "2"), 
-         ("1", "1"+digraph.separator_+"1.1"),
-         ("1", "1"+digraph.separator_+"1.2"), 
-         ("2", "2"+digraph.separator_+"2.1"), 
-         ("2", "2"+digraph.separator_+"2.2")]
+        [
+            ("r", "1"),
+            ("r", "2"),
+            ("1", "1" + digraph.separator_ + "1.1"),
+            ("1", "1" + digraph.separator_ + "1.2"),
+            ("2", "2" + digraph.separator_ + "2.1"),
+            ("2", "2" + digraph.separator_ + "2.2"),
+        ]
     )
-    digraph.y_ = np.array([
-        ["1", "1.1"], 
-        ["1", "1.2"], 
-        ["2", "2.1"], 
-        ["2", "2.2"]])
-    
+    digraph.y_ = np.array([["1", "1.1"], ["1", "1.2"], ["2", "2.1"], ["2", "2.2"]])
+
     digraph.X_ = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
     digraph.logger_ = logging.getLogger("LCPN")
     digraph.max_levels_ = 2
@@ -257,25 +279,59 @@ def fitted_logistic_regression():
 
     # for predict_proba
     tmp_labels = digraph._disambiguate(make_leveled(digraph.y_))
-    #digraph.y_ = tmp_labels
-    digraph.max_level_dimensions_ = np.array([len(np.unique(tmp_labels[:, level])) for level in range(tmp_labels.shape[1])])
-    digraph.global_classes_ = [np.unique(tmp_labels[:, level]).astype("str") for level in range(tmp_labels.shape[1])]
-    digraph.global_class_to_index_mapping_ = [{digraph.global_classes_[level][index]: index for index in range(len(digraph.global_classes_[level]))} for level in range(tmp_labels.shape[1])]
+    # digraph.y_ = tmp_labels
+    digraph.max_level_dimensions_ = np.array(
+        [len(np.unique(tmp_labels[:, level])) for level in range(tmp_labels.shape[1])]
+    )
+    digraph.global_classes_ = [
+        np.unique(tmp_labels[:, level]).astype("str")
+        for level in range(tmp_labels.shape[1])
+    ]
+    digraph.global_class_to_index_mapping_ = [
+        {
+            digraph.global_classes_[level][index]: index
+            for index in range(len(digraph.global_classes_[level]))
+        }
+        for level in range(tmp_labels.shape[1])
+    ]
 
     classes_ = [digraph.global_classes_[0]]
     for level in range(1, digraph.max_levels_):
-        classes_.append(np.sort(np.unique([label.split(digraph.separator_)[level] for label in digraph.global_classes_[level]])))
+        classes_.append(
+            np.sort(
+                np.unique(
+                    [
+                        label.split(digraph.separator_)[level]
+                        for label in digraph.global_classes_[level]
+                    ]
+                )
+            )
+        )
     digraph.classes_ = classes_
-    digraph.class_to_index_mapping_ = [{local_labels[index]: index for index in range(len(local_labels))} for local_labels in classes_]
+    digraph.class_to_index_mapping_ = [
+        {local_labels[index]: index for index in range(len(local_labels))}
+        for local_labels in classes_
+    ]
 
-    classifiers = {key: {"classifier": LogisticRegression()} for key in np.hstack([digraph.global_classes_[0], digraph.global_classes_[1]])}
+    classifiers = {
+        key: {"classifier": LogisticRegression()}
+        for key in np.hstack([digraph.global_classes_[0], digraph.global_classes_[1]])
+    }
 
     classifiers["1"]["classifier"].fit(digraph.X_, [1, 1, 0, 0])
-    classifiers["1"+digraph.separator_+"1.1"]["classifier"].fit(digraph.X_, [1, 0, 0, 0])
-    classifiers["1"+digraph.separator_+"1.2"]["classifier"].fit(digraph.X_, [0, 1, 0, 0])
+    classifiers["1" + digraph.separator_ + "1.1"]["classifier"].fit(
+        digraph.X_, [1, 0, 0, 0]
+    )
+    classifiers["1" + digraph.separator_ + "1.2"]["classifier"].fit(
+        digraph.X_, [0, 1, 0, 0]
+    )
     classifiers["2"]["classifier"].fit(digraph.X_, [0, 0, 1, 1])
-    classifiers["2"+digraph.separator_+"2.1"]["classifier"].fit(digraph.X_, [0, 0, 1, 0])
-    classifiers["2"+digraph.separator_+"2.2"]["classifier"].fit(digraph.X_, [0, 0, 0, 1])
+    classifiers["2" + digraph.separator_ + "2.1"]["classifier"].fit(
+        digraph.X_, [0, 0, 1, 0]
+    )
+    classifiers["2" + digraph.separator_ + "2.2"]["classifier"].fit(
+        digraph.X_, [0, 0, 0, 1]
+    )
     nx.set_node_attributes(digraph.hierarchy_, classifiers)
     return digraph
 
@@ -293,21 +349,33 @@ def test_predict_sparse(fitted_logistic_regression):
     )
     assert_array_equal(ground_truth, prediction)
 
+
 def test_predict_proba(fitted_logistic_regression):
     proba = fitted_logistic_regression.predict_proba([[7, 8], [5, 6], [3, 4], [1, 2]])
     assert len(proba) == 2
     assert proba[0].shape == (4, 2)
     assert proba[1].shape == (4, 4)
-    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
-    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+    assert_array_almost_equal(
+        np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10
+    )
+    assert_array_almost_equal(
+        np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10
+    )
+
 
 def test_predict_proba_sparse(fitted_logistic_regression):
-    proba = fitted_logistic_regression.predict_proba(csr_matrix([[7, 8], [5, 6], [3, 4], [1, 2]]))
+    proba = fitted_logistic_regression.predict_proba(
+        csr_matrix([[7, 8], [5, 6], [3, 4], [1, 2]])
+    )
     assert len(proba) == 2
     assert proba[0].shape == (4, 2)
     assert proba[1].shape == (4, 4)
-    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
-    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+    assert_array_almost_equal(
+        np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10
+    )
+    assert_array_almost_equal(
+        np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10
+    )
 
 
 def test_fit_predict():
@@ -318,8 +386,13 @@ def test_fit_predict():
     predictions = lcpn.predict(x)
     assert_array_equal(y, predictions)
 
+
 def test_fit_calibrate_predict_proba():
-    lcpn = LocalClassifierPerNode(local_classifier=LogisticRegression(), calibration_method="ivap", return_all_probabilities=True)
+    lcpn = LocalClassifierPerNode(
+        local_classifier=LogisticRegression(),
+        calibration_method="ivap",
+        return_all_probabilities=True,
+    )
     x = np.array([[1, 2], [3, 4]])
     y = np.array([["a", "b"], ["b", "c"]])
     lcpn.fit(x, y)
@@ -331,15 +404,20 @@ def test_fit_calibrate_predict_proba():
     assert len(proba) == 2
     assert proba[0].shape == (2, 2)
     assert proba[1].shape == (2, 2)
-    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
-    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+    assert_array_almost_equal(
+        np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10
+    )
+    assert_array_almost_equal(
+        np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10
+    )
+
 
 def test_fit_calibrate_predict_predict_proba_bert():
     classifier = LocalClassifierPerNode(
-        local_classifier=LogisticRegression(), 
+        local_classifier=LogisticRegression(),
         return_all_probabilities=True,
         calibration_method="ivap",
-        probability_combiner="geometric" 
+        probability_combiner="geometric",
     )
 
     classifier.logger_ = logging.getLogger("HC")
@@ -350,4 +428,4 @@ def test_fit_calibrate_predict_predict_proba_bert():
     classifier.fit(x, y, sample_weight)
     classifier.calibrate(x, y)
     classifier.predict(x)
-    classifier.predict_proba(x)
\ No newline at end of file
+    classifier.predict_proba(x)
diff --git a/tests/test_LocalClassifierPerParentNode.py b/tests/test_LocalClassifierPerParentNode.py
index 9480dbab..ad1a5469 100644
--- a/tests/test_LocalClassifierPerParentNode.py
+++ b/tests/test_LocalClassifierPerParentNode.py
@@ -22,7 +22,9 @@ def test_sklearn_compatible_estimator(estimator, check):
 
 @pytest.fixture
 def digraph_logistic_regression():
-    digraph = LocalClassifierPerParentNode(local_classifier=LogisticRegression(), calibration_method="ivap")
+    digraph = LocalClassifierPerParentNode(
+        local_classifier=LogisticRegression(), calibration_method="ivap"
+    )
     digraph.hierarchy_ = nx.DiGraph([("a", "b"), ("a", "c"), ("b", "d"), ("c", "f")])
     digraph.y_ = np.array([["a", "b"], ["a", "c"], ["b", "d"], ["c", "f"]])
     digraph.X_ = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
@@ -50,6 +52,7 @@ def test_initialize_local_classifiers(digraph_logistic_regression):
                     LogisticRegression,
                 )
 
+
 def test_initialize_local_calibrators(digraph_logistic_regression):
     digraph_logistic_regression._initialize_local_classifiers()
     digraph_logistic_regression._initialize_local_calibrators()
@@ -88,6 +91,7 @@ def test_fit_digraph(digraph_logistic_regression):
             )
     assert 1
 
+
 def test_calibrate_digraph(digraph_logistic_regression):
     classifiers = {
         "a": {"classifier": LogisticRegression()},
@@ -99,7 +103,11 @@ def test_calibrate_digraph(digraph_logistic_regression):
     digraph_logistic_regression._fit_digraph(local_mode=True)
 
     calibrators = {
-        "a": {"calibrator": _Calibrator(digraph_logistic_regression.hierarchy_.nodes["a"]["classifier"])},
+        "a": {
+            "calibrator": _Calibrator(
+                digraph_logistic_regression.hierarchy_.nodes["a"]["classifier"]
+            )
+        },
     }
     nx.set_node_attributes(digraph_logistic_regression.hierarchy_, calibrators)
     digraph_logistic_regression._calibrate_digraph(local_mode=True)
@@ -136,6 +144,7 @@ def test_fit_digraph_joblib_multiprocessing(digraph_logistic_regression):
             )
     assert 1
 
+
 def test_calibrate_digraph_joblib_multiprocessing(digraph_logistic_regression):
     classifiers = {
         "a": {"classifier": LogisticRegression()},
@@ -147,7 +156,11 @@ def test_calibrate_digraph_joblib_multiprocessing(digraph_logistic_regression):
     digraph_logistic_regression._fit_digraph(local_mode=True, use_joblib=True)
 
     calibrators = {
-        "a": {"calibrator": _Calibrator(digraph_logistic_regression.hierarchy_.nodes["a"]["classifier"])},
+        "a": {
+            "calibrator": _Calibrator(
+                digraph_logistic_regression.hierarchy_.nodes["a"]["classifier"]
+            )
+        },
     }
     nx.set_node_attributes(digraph_logistic_regression.hierarchy_, calibrators)
     digraph_logistic_regression._calibrate_digraph(local_mode=True, use_joblib=True)
@@ -163,6 +176,7 @@ def test_calibrate_digraph_joblib_multiprocessing(digraph_logistic_regression):
             )
     assert 1
 
+
 @pytest.fixture
 def digraph_2d():
     classifier = LocalClassifierPerParentNode()
@@ -211,24 +225,24 @@ def test_get_successors(x_and_y_arrays):
 @pytest.fixture
 def fitted_logistic_regression():
     digraph = LocalClassifierPerParentNode(
-        local_classifier=LogisticRegression(), 
+        local_classifier=LogisticRegression(),
         calibration_method="ivap",
         probability_combiner="geometric",
-        return_all_probabilities=True)
-    
+        return_all_probabilities=True,
+    )
+
     digraph.separator_ = "::HiClass::Separator::"
     digraph.hierarchy_ = nx.DiGraph(
         [
             ("r", "1"),
             ("r", "2"),
-            ("1", "1"+digraph.separator_+"1.1"),
-            ("1", "1"+digraph.separator_+"1.2"),
-            ("2", "2"+digraph.separator_+"2.1"),
-            ("2", "2"+digraph.separator_+"2.2"),
+            ("1", "1" + digraph.separator_ + "1.1"),
+            ("1", "1" + digraph.separator_ + "1.2"),
+            ("2", "2" + digraph.separator_ + "2.1"),
+            ("2", "2" + digraph.separator_ + "2.2"),
         ]
     )
-    digraph.y_ = np.array([
-        ["1", "1.1"], ["1", "1.2"], ["2", "2.1"], ["2", "2.2"]])
+    digraph.y_ = np.array([["1", "1.1"], ["1", "1.2"], ["2", "2.1"], ["2", "2.2"]])
     digraph.X_ = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
     digraph.logger_ = logging.getLogger("LCPPN")
     digraph.max_levels_ = 2
@@ -238,15 +252,36 @@ def fitted_logistic_regression():
 
     # for predict_proba
     tmp_labels = digraph._disambiguate(make_leveled(digraph.y_))
-    #digraph.y_ = tmp_labels
-    digraph.global_classes_ = [np.unique(tmp_labels[:, level]).astype("str") for level in range(tmp_labels.shape[1])]
-    digraph.global_class_to_index_mapping_ = [{digraph.global_classes_[level][index]: index for index in range(len(digraph.global_classes_[level]))} for level in range(tmp_labels.shape[1])]
+    # digraph.y_ = tmp_labels
+    digraph.global_classes_ = [
+        np.unique(tmp_labels[:, level]).astype("str")
+        for level in range(tmp_labels.shape[1])
+    ]
+    digraph.global_class_to_index_mapping_ = [
+        {
+            digraph.global_classes_[level][index]: index
+            for index in range(len(digraph.global_classes_[level]))
+        }
+        for level in range(tmp_labels.shape[1])
+    ]
 
     classes_ = [digraph.global_classes_[0]]
     for level in range(1, digraph.max_levels_):
-        classes_.append(np.sort(np.unique([label.split(digraph.separator_)[level] for label in digraph.global_classes_[level]])))
+        classes_.append(
+            np.sort(
+                np.unique(
+                    [
+                        label.split(digraph.separator_)[level]
+                        for label in digraph.global_classes_[level]
+                    ]
+                )
+            )
+        )
     digraph.classes_ = classes_
-    digraph.class_to_index_mapping_ = [{local_labels[index]: index for index in range(len(local_labels))} for local_labels in classes_]
+    digraph.class_to_index_mapping_ = [
+        {local_labels[index]: index for index in range(len(local_labels))}
+        for local_labels in classes_
+    ]
 
     classifiers = {
         "r": {"classifier": LogisticRegression()},
@@ -254,8 +289,14 @@ def fitted_logistic_regression():
         "2": {"classifier": LogisticRegression()},
     }
     classifiers["r"]["classifier"].fit(digraph.X_, ["1", "1", "2", "2"])
-    classifiers["1"]["classifier"].fit(digraph.X_[:2], ["1"+digraph.separator_+"1.1", "1"+digraph.separator_+"1.2"])
-    classifiers["2"]["classifier"].fit(digraph.X_[2:], ["2"+digraph.separator_+"2.1", "2"+digraph.separator_+"2.2"])
+    classifiers["1"]["classifier"].fit(
+        digraph.X_[:2],
+        ["1" + digraph.separator_ + "1.1", "1" + digraph.separator_ + "1.2"],
+    )
+    classifiers["2"]["classifier"].fit(
+        digraph.X_[2:],
+        ["2" + digraph.separator_ + "2.1", "2" + digraph.separator_ + "2.2"],
+    )
     nx.set_node_attributes(digraph.hierarchy_, classifiers)
     return digraph
 
@@ -273,21 +314,33 @@ def test_predict_sparse(fitted_logistic_regression):
     )
     assert_array_equal(ground_truth, prediction)
 
+
 def test_predict_proba(fitted_logistic_regression):
     proba = fitted_logistic_regression.predict_proba([[7, 8], [5, 6], [3, 4], [1, 2]])
     assert len(proba) == 2
     assert proba[0].shape == (4, 2)
     assert proba[1].shape == (4, 4)
-    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
-    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+    assert_array_almost_equal(
+        np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10
+    )
+    assert_array_almost_equal(
+        np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10
+    )
+
 
 def test_predict_proba_sparse(fitted_logistic_regression):
-    proba = fitted_logistic_regression.predict_proba(csr_matrix([[7, 8], [5, 6], [3, 4], [1, 2]]))
+    proba = fitted_logistic_regression.predict_proba(
+        csr_matrix([[7, 8], [5, 6], [3, 4], [1, 2]])
+    )
     assert len(proba) == 2
     assert proba[0].shape == (4, 2)
     assert proba[1].shape == (4, 4)
-    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
-    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+    assert_array_almost_equal(
+        np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10
+    )
+    assert_array_almost_equal(
+        np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10
+    )
 
 
 def test_fit_predict():
@@ -298,8 +351,13 @@ def test_fit_predict():
     predictions = lcppn.predict(x)
     assert_array_equal(y, predictions)
 
+
 def test_fit_calibrate_predict_proba():
-    lcppn = LocalClassifierPerParentNode(local_classifier=LogisticRegression(), calibration_method="ivap", return_all_probabilities=True)
+    lcppn = LocalClassifierPerParentNode(
+        local_classifier=LogisticRegression(),
+        calibration_method="ivap",
+        return_all_probabilities=True,
+    )
     x = np.array([[1, 2], [3, 4]])
     y = np.array([["a", "b"], ["b", "c"]])
     lcppn.fit(x, y)
@@ -310,15 +368,20 @@ def test_fit_calibrate_predict_proba():
     assert len(proba) == 2
     assert proba[0].shape == (2, 2)
     assert proba[1].shape == (2, 2)
-    assert_array_almost_equal(np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10)
-    assert_array_almost_equal(np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10)
+    assert_array_almost_equal(
+        np.sum(proba[0], axis=1), np.ones(len(proba[0])), decimal=10
+    )
+    assert_array_almost_equal(
+        np.sum(proba[1], axis=1), np.ones(len(proba[1])), decimal=10
+    )
+
 
 def test_fit_calibrate_predict_predict_proba_bert():
     classifier = LocalClassifierPerParentNode(
-        local_classifier=LogisticRegression(), 
+        local_classifier=LogisticRegression(),
         return_all_probabilities=True,
         calibration_method="ivap",
-        probability_combiner="geometric" 
+        probability_combiner="geometric",
     )
 
     classifier.logger_ = logging.getLogger("HC")
@@ -329,4 +392,4 @@ def test_fit_calibrate_predict_predict_proba_bert():
     classifier.fit(x, y, sample_weight)
     classifier.calibrate(x, y)
     classifier.predict(x)
-    classifier.predict_proba(x)
\ No newline at end of file
+    classifier.predict_proba(x)
diff --git a/tests/test_ProbabilityCombiner.py b/tests/test_ProbabilityCombiner.py
index db4c81b8..a9f21aa7 100644
--- a/tests/test_ProbabilityCombiner.py
+++ b/tests/test_ProbabilityCombiner.py
@@ -9,7 +9,11 @@
 from hiclass.HierarchicalClassifier import make_leveled
 
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
-from hiclass.probability_combiner import MultiplyCombiner, ArithmeticMeanCombiner, GeometricMeanCombiner
+from hiclass.probability_combiner import (
+    MultiplyCombiner,
+    ArithmeticMeanCombiner,
+    GeometricMeanCombiner,
+)
 
 
 def _disambiguate(y, separator):
@@ -25,6 +29,7 @@ def _disambiguate(y, separator):
         return np.array(new_y)
     return y
 
+
 @pytest.fixture
 def one_sample_probs_with_hierarchy():
     hierarchy = nx.DiGraph()
@@ -35,27 +40,55 @@ def one_sample_probs_with_hierarchy():
 
     for i in range(3):
         # first level
-        first_level_node = f'node_{i}'
+        first_level_node = f"node_{i}"
         hierarchy.add_node(first_level_node)
         hierarchy.add_edge(root_node, first_level_node)
         for j in range(3):
             # second level
-            second_level_node = f'node_{i}:{j}'
-            second_level_label = _disambiguate(np.array([[first_level_node, second_level_node]]), separator)[0][1]
+            second_level_node = f"node_{i}:{j}"
+            second_level_label = _disambiguate(
+                np.array([[first_level_node, second_level_node]]), separator
+            )[0][1]
             hierarchy.add_node(second_level_label)
             hierarchy.add_edge(first_level_node, second_level_label)
             for k in range(2):
                 # third level
-                third_level_node = f'node_{i}:{j}:{k}'
-                third_level_label = _disambiguate(np.array([[first_level_node, second_level_node, third_level_node]]), separator)[0][2]
+                third_level_node = f"node_{i}:{j}:{k}"
+                third_level_label = _disambiguate(
+                    np.array([[first_level_node, second_level_node, third_level_node]]),
+                    separator,
+                )[0][2]
                 y_.append([first_level_node, second_level_node, third_level_node])
                 hierarchy.add_node(third_level_label)
                 hierarchy.add_edge(second_level_label, third_level_label)
 
     probs = [
-        np.array([[0.3, 0.5, 0.2]]), # level 0
-        np.array([[0.19, 0.10, 0.16, 0.14, 0.07, 0.08, 0.19, 0.03, 0.04]]), # level 1
-        np.array([[0.03, 0.17, 0.07, 0.01, 0.00, 0.06, 0.17, 0.02, 0.01, 0.07, 0.20, 0.01, 0.10, 0.00, 0.05, 0.02, 0.00, 0.01]]) # level 2
+        np.array([[0.3, 0.5, 0.2]]),  # level 0
+        np.array([[0.19, 0.10, 0.16, 0.14, 0.07, 0.08, 0.19, 0.03, 0.04]]),  # level 1
+        np.array(
+            [
+                [
+                    0.03,
+                    0.17,
+                    0.07,
+                    0.01,
+                    0.00,
+                    0.06,
+                    0.17,
+                    0.02,
+                    0.01,
+                    0.07,
+                    0.20,
+                    0.01,
+                    0.10,
+                    0.00,
+                    0.05,
+                    0.02,
+                    0.00,
+                    0.01,
+                ]
+            ]
+        ),  # level 2
     ]
     assert all([np.sum(probs[level], axis=1) == 1 for level in range(3)])
 
@@ -63,18 +96,36 @@ def one_sample_probs_with_hierarchy():
     y_ = make_leveled(y_)
     y_ = _disambiguate(y_, separator)
 
-    global_classes_ = [np.unique(y_[:, level]).astype("str") for level in range(y_.shape[1])]
+    global_classes_ = [
+        np.unique(y_[:, level]).astype("str") for level in range(y_.shape[1])
+    ]
 
     classes_ = [global_classes_[0]]
     for level in range(1, len(y_[1])):
-        classes_.append(np.sort(np.unique([label.split(separator)[level] for label in global_classes_[level]])))
-
-    class_to_index_mapping_ = [{local_labels[index]: index for index in range(len(local_labels))} for local_labels in classes_]
+        classes_.append(
+            np.sort(
+                np.unique(
+                    [label.split(separator)[level] for label in global_classes_[level]]
+                )
+            )
+        )
+
+    class_to_index_mapping_ = [
+        {local_labels[index]: index for index in range(len(local_labels))}
+        for local_labels in classes_
+    ]
 
     return hierarchy, probs, global_classes_, classes_, class_to_index_mapping_
 
+
 def test_multiply_combiner(one_sample_probs_with_hierarchy):
-    hierarchy, probs, global_classes, classes_, class_to_index_mapping_ = one_sample_probs_with_hierarchy
+    (
+        hierarchy,
+        probs,
+        global_classes,
+        classes_,
+        class_to_index_mapping_,
+    ) = one_sample_probs_with_hierarchy
     obj = HierarchicalClassifier()
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
@@ -82,7 +133,7 @@ def test_multiply_combiner(one_sample_probs_with_hierarchy):
     classifier.max_levels_ = len(global_classes)
     classifier.classes_ = classes_
     classifier.class_to_index_mapping_ = class_to_index_mapping_
-    
+
     classifier.hierarchy_ = hierarchy
     classifier.separator_ = "::HiClass::Separator::"
 
@@ -91,24 +142,43 @@ def test_multiply_combiner(one_sample_probs_with_hierarchy):
 
     # check combined probability of first node for both levels
     assert math.isclose(combined_probs[1][0][0], 0.0569, abs_tol=1e-4)
-    assert math.isclose(combined_probs[1][0][0], probs[0][0][0] * probs[1][0][0], abs_tol=1e-4)
-    
+    assert math.isclose(
+        combined_probs[1][0][0], probs[0][0][0] * probs[1][0][0], abs_tol=1e-4
+    )
+
     assert math.isclose(combined_probs[2][0][0], 0.0017, abs_tol=1e-4)
-    assert math.isclose(combined_probs[2][0][0], probs[0][0][0] * probs[1][0][0] * probs[2][0][0], abs_tol=1e-4)
+    assert math.isclose(
+        combined_probs[2][0][0],
+        probs[0][0][0] * probs[1][0][0] * probs[2][0][0],
+        abs_tol=1e-4,
+    )
 
     # check combined probability of last node for both levels
     assert math.isclose(combined_probs[1][0][-1], 0.008, abs_tol=1e-4)
-    assert math.isclose(combined_probs[1][0][-1], probs[0][0][-1] * probs[1][0][-1], abs_tol=1e-4)
-    
+    assert math.isclose(
+        combined_probs[1][0][-1], probs[0][0][-1] * probs[1][0][-1], abs_tol=1e-4
+    )
+
     assert math.isclose(combined_probs[2][0][-1], 8e-5, abs_tol=1e-4)
-    assert math.isclose(combined_probs[2][0][-1], probs[0][0][-1] * probs[1][0][-1] * probs[2][0][-1], abs_tol=1e-4)
+    assert math.isclose(
+        combined_probs[2][0][-1],
+        probs[0][0][-1] * probs[1][0][-1] * probs[2][0][-1],
+        abs_tol=1e-4,
+    )
+
 
 def test_arithmetic_mean_combiner(one_sample_probs_with_hierarchy):
-    hierarchy, probs, global_classes, classes_, class_to_index_mapping_ = one_sample_probs_with_hierarchy
+    (
+        hierarchy,
+        probs,
+        global_classes,
+        classes_,
+        class_to_index_mapping_,
+    ) = one_sample_probs_with_hierarchy
     obj = HierarchicalClassifier()
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
-    
+
     classifier.global_classes_ = global_classes
     classifier.max_levels_ = len(global_classes)
     classifier.classes_ = classes_
@@ -121,20 +191,39 @@ def test_arithmetic_mean_combiner(one_sample_probs_with_hierarchy):
 
     # check combined probability of first node for both levels
     assert math.isclose(combined_probs[1][0][0], 0.245, abs_tol=1e-4)
-    assert math.isclose(combined_probs[1][0][0], (probs[0][0][0] + probs[1][0][0]) / 2, abs_tol=1e-4)
+    assert math.isclose(
+        combined_probs[1][0][0], (probs[0][0][0] + probs[1][0][0]) / 2, abs_tol=1e-4
+    )
 
     assert math.isclose(combined_probs[2][0][0], 0.1733, abs_tol=1e-4)
-    assert math.isclose(combined_probs[2][0][0], (probs[0][0][0] + probs[1][0][0] + probs[2][0][0]) / 3, abs_tol=1e-4)
+    assert math.isclose(
+        combined_probs[2][0][0],
+        (probs[0][0][0] + probs[1][0][0] + probs[2][0][0]) / 3,
+        abs_tol=1e-4,
+    )
 
     # check combined probability of last node for both levels
     assert math.isclose(combined_probs[1][0][-1], 0.12, abs_tol=1e-4)
-    assert math.isclose(combined_probs[1][0][-1], (probs[0][0][-1] + probs[1][0][-1]) / 2, abs_tol=1e-4)
-    
+    assert math.isclose(
+        combined_probs[1][0][-1], (probs[0][0][-1] + probs[1][0][-1]) / 2, abs_tol=1e-4
+    )
+
     assert math.isclose(combined_probs[2][0][-1], 0.0833, abs_tol=1e-4)
-    assert math.isclose(combined_probs[2][0][-1], (probs[0][0][-1] + probs[1][0][-1] + probs[2][0][-1]) / 3, abs_tol=1e-4)
+    assert math.isclose(
+        combined_probs[2][0][-1],
+        (probs[0][0][-1] + probs[1][0][-1] + probs[2][0][-1]) / 3,
+        abs_tol=1e-4,
+    )
+
 
 def test_geometric_mean_combiner(one_sample_probs_with_hierarchy):
-    hierarchy, probs, global_classes, classes_, class_to_index_mapping_ = one_sample_probs_with_hierarchy
+    (
+        hierarchy,
+        probs,
+        global_classes,
+        classes_,
+        class_to_index_mapping_,
+    ) = one_sample_probs_with_hierarchy
     obj = HierarchicalClassifier()
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
@@ -151,20 +240,41 @@ def test_geometric_mean_combiner(one_sample_probs_with_hierarchy):
 
     # check combined probability of first node for both levels
     assert math.isclose(combined_probs[1][0][0], 0.2387, abs_tol=1e-4)
-    assert math.isclose(combined_probs[1][0][0], gmean([probs[0][0][0], probs[1][0][0]]), abs_tol=1e-4)
+    assert math.isclose(
+        combined_probs[1][0][0], gmean([probs[0][0][0], probs[1][0][0]]), abs_tol=1e-4
+    )
 
     assert math.isclose(combined_probs[2][0][0], 0.1195, abs_tol=1e-4)
-    assert math.isclose(combined_probs[2][0][0], gmean([probs[0][0][0], probs[1][0][0], probs[2][0][0]]), abs_tol=1e-4)
+    assert math.isclose(
+        combined_probs[2][0][0],
+        gmean([probs[0][0][0], probs[1][0][0], probs[2][0][0]]),
+        abs_tol=1e-4,
+    )
 
     # check combined probability of last node for both levels
     assert math.isclose(combined_probs[1][0][-1], 0.0894, abs_tol=1e-4)
-    assert math.isclose(combined_probs[1][0][-1], gmean([probs[0][0][-1], probs[1][0][-1]]), abs_tol=1e-4)
-    
+    assert math.isclose(
+        combined_probs[1][0][-1],
+        gmean([probs[0][0][-1], probs[1][0][-1]]),
+        abs_tol=1e-4,
+    )
+
     assert math.isclose(combined_probs[2][0][-1], 0.0430, abs_tol=1e-4)
-    assert math.isclose(combined_probs[2][0][-1], gmean([probs[0][0][-1], probs[1][0][-1], probs[2][0][-1]]), abs_tol=1e-4)
+    assert math.isclose(
+        combined_probs[2][0][-1],
+        gmean([probs[0][0][-1], probs[1][0][-1], probs[2][0][-1]]),
+        abs_tol=1e-4,
+    )
+
 
 def test_mean_combiner_with_normalization(one_sample_probs_with_hierarchy):
-    hierarchy, probs, global_classes, classes_, class_to_index_mapping_ = one_sample_probs_with_hierarchy
+    (
+        hierarchy,
+        probs,
+        global_classes,
+        classes_,
+        class_to_index_mapping_,
+    ) = one_sample_probs_with_hierarchy
     obj = HierarchicalClassifier()
     classifier = Mock(spec=obj)
     classifier._disambiguate = obj._disambiguate
@@ -176,7 +286,11 @@ def test_mean_combiner_with_normalization(one_sample_probs_with_hierarchy):
     classifier.class_to_index_mapping_ = class_to_index_mapping_
     classifier.hierarchy_ = hierarchy
 
-    combiners = [MultiplyCombiner(classifier), ArithmeticMeanCombiner(classifier), GeometricMeanCombiner(classifier)]
+    combiners = [
+        MultiplyCombiner(classifier),
+        ArithmeticMeanCombiner(classifier),
+        GeometricMeanCombiner(classifier),
+    ]
 
     for combiner in combiners:
         combined_probs = combiner.combine(probs)
@@ -185,6 +299,18 @@ def test_mean_combiner_with_normalization(one_sample_probs_with_hierarchy):
         assert combined_probs[0].shape == (1, 3)
         assert combined_probs[1].shape == (1, 9)
         assert combined_probs[2].shape == (1, 18)
-        assert_array_almost_equal(np.sum(combined_probs[0], axis=1), np.ones(len(combined_probs[0])), decimal=10)
-        assert_array_almost_equal(np.sum(combined_probs[1], axis=1), np.ones(len(combined_probs[1])), decimal=10)
-        assert_array_almost_equal(np.sum(combined_probs[2], axis=1), np.ones(len(combined_probs[2])), decimal=10)
+        assert_array_almost_equal(
+            np.sum(combined_probs[0], axis=1),
+            np.ones(len(combined_probs[0])),
+            decimal=10,
+        )
+        assert_array_almost_equal(
+            np.sum(combined_probs[1], axis=1),
+            np.ones(len(combined_probs[1])),
+            decimal=10,
+        )
+        assert_array_almost_equal(
+            np.sum(combined_probs[2], axis=1),
+            np.ones(len(combined_probs[2])),
+            decimal=10,
+        )
diff --git a/tests/test_calibration.py b/tests/test_calibration.py
index 0ef04f70..2f655c86 100644
--- a/tests/test_calibration.py
+++ b/tests/test_calibration.py
@@ -5,65 +5,88 @@
 from sklearn.linear_model import LogisticRegression
 from sklearn.exceptions import NotFittedError
 
-from hiclass._calibration.VennAbersCalibrator import _InductiveVennAbersCalibrator, _CrossVennAbersCalibrator
+from hiclass._calibration.VennAbersCalibrator import (
+    _InductiveVennAbersCalibrator,
+    _CrossVennAbersCalibrator,
+)
 from hiclass._calibration.PlattScaling import _PlattScaling
 from hiclass._calibration.IsotonicRegression import _IsotonicRegression
 from hiclass._calibration.BetaCalibrator import _BetaCalibrator
 from hiclass._calibration.Calibrator import _Calibrator
 
+
 @pytest.fixture
 def binary_calibration_data():
-    prob = np.array([[0.37, 0.63],
-                     [0.39, 0.61],
-                     [0.42, 0.58],
-                     [0.51, 0.49],
-                     [0.51, 0.49],
-                     [0.45, 0.55],
-                     [0.48, 0.52],
-                     [0.60, 0.40],
-                     [0.54, 0.46],
-                     [0.57, 0.43],
-                     [0.57, 0.43],
-                     [0.62, 0.38]]) 
-    
+    prob = np.array(
+        [
+            [0.37, 0.63],
+            [0.39, 0.61],
+            [0.42, 0.58],
+            [0.51, 0.49],
+            [0.51, 0.49],
+            [0.45, 0.55],
+            [0.48, 0.52],
+            [0.60, 0.40],
+            [0.54, 0.46],
+            [0.57, 0.43],
+            [0.57, 0.43],
+            [0.62, 0.38],
+        ]
+    )
+
     assert_array_equal(np.sum(prob, axis=1), np.ones(len(prob)))
 
     ground_truth_labels = np.array([1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0])
     return prob, ground_truth_labels
 
+
 @pytest.fixture
 def binary_test_scores():
-    X = np.array([[0.22, 0.78],
-                  [0.48, 0.52],
-                  [0.66, 0.34],
-                  [0.01, 0.99],
-                  [0.77, 0.23],
-                  [0.50, 0.50],
-                  [1.00, 0.00]])
-    
+    X = np.array(
+        [
+            [0.22, 0.78],
+            [0.48, 0.52],
+            [0.66, 0.34],
+            [0.01, 0.99],
+            [0.77, 0.23],
+            [0.50, 0.50],
+            [1.00, 0.00],
+        ]
+    )
+
     assert_array_equal(np.sum(X, axis=1), np.ones(len(X)))
     return X
 
+
 @pytest.fixture
 def binary_cal_X():
-    X = np.array([[1, 2], 
-                  [3, 4], 
-                  [5, 6], 
-                  [7, 8], 
-                  [1, 2], 
-                  [3, 4], 
-                  [5, 6], 
-                  [7, 8], 
-                  [1, 2], 
-                  [3, 4], 
-                  [5, 6], 
-                  [7, 8]])
+    X = np.array(
+        [
+            [1, 2],
+            [3, 4],
+            [5, 6],
+            [7, 8],
+            [1, 2],
+            [3, 4],
+            [5, 6],
+            [7, 8],
+            [1, 2],
+            [3, 4],
+            [5, 6],
+            [7, 8],
+        ]
+    )
     return X
 
+
 @pytest.fixture
 def binary_mock_estimator(binary_calibration_data, binary_test_scores):
     # return calibration scores or test scores depending on input size
-    side_effect = lambda X: binary_calibration_data[0] if len(X) == len(binary_calibration_data[0]) else binary_test_scores
+    side_effect = (
+        lambda X: binary_calibration_data[0]
+        if len(X) == len(binary_calibration_data[0])
+        else binary_test_scores
+    )
 
     lr = LogisticRegression()
     binary_estimator = Mock(spec=lr)
@@ -72,49 +95,63 @@ def binary_mock_estimator(binary_calibration_data, binary_test_scores):
     binary_estimator.get_params.return_value = lr.get_params()
     return binary_estimator
 
+
 @pytest.fixture
 def multiclass_calibration_data():
-    prob = np.array([[0.12, 0.24, 0.64],
-                     [0.44, 0.22, 0.34],
-                     [0.30, 0.40, 0.30],
-                     [0.11, 0.33, 0.56],
-                     [0.44, 0.22, 0.34],
-                     [0.66, 0.33, 0.01],
-                     [0.14, 0.66, 0.20],
-                     [0.44, 0.12, 0.44],
-                     [0.64, 0.24, 0.12],
-                     [0.20, 0.77, 0.03]
-    ])
+    prob = np.array(
+        [
+            [0.12, 0.24, 0.64],
+            [0.44, 0.22, 0.34],
+            [0.30, 0.40, 0.30],
+            [0.11, 0.33, 0.56],
+            [0.44, 0.22, 0.34],
+            [0.66, 0.33, 0.01],
+            [0.14, 0.66, 0.20],
+            [0.44, 0.12, 0.44],
+            [0.64, 0.24, 0.12],
+            [0.20, 0.77, 0.03],
+        ]
+    )
 
     assert_array_equal(np.sum(prob, axis=1), np.ones(len(prob)))
 
     ground_truth_labels = np.array([2, 0, 1, 2, 0, 0, 1, 0, 0, 1])
     return prob, ground_truth_labels
 
+
 @pytest.fixture
 def multiclass_test_scores():
-    X = np.array([[0.23, 0.47, 0.30],
-                  [0.44, 0.21, 0.35],
-                  [0.22, 0.22, 0.56],
-                  [0.25, 0.50, 0.25],
-                  [0.01, 0.72, 0.27],
-                  [0.12, 0.35, 0.53],
-                  [0.30, 0.23, 0.47]
-    ])
-    
+    X = np.array(
+        [
+            [0.23, 0.47, 0.30],
+            [0.44, 0.21, 0.35],
+            [0.22, 0.22, 0.56],
+            [0.25, 0.50, 0.25],
+            [0.01, 0.72, 0.27],
+            [0.12, 0.35, 0.53],
+            [0.30, 0.23, 0.47],
+        ]
+    )
+
     assert_array_equal(np.sum(X, axis=1), np.ones(len(X)))
     return X
 
+
 @pytest.fixture
 def multiclass_mock_estimator(multiclass_calibration_data, multiclass_test_scores):
     # return calibration scores or test scores depending on input size
-    side_effect = lambda X: multiclass_calibration_data[0] if len(X) == len(multiclass_calibration_data[0]) else multiclass_test_scores
+    side_effect = (
+        lambda X: multiclass_calibration_data[0]
+        if len(X) == len(multiclass_calibration_data[0])
+        else multiclass_test_scores
+    )
 
     multiclass_estimator = Mock(spec=LogisticRegression)
     multiclass_estimator.predict_proba.side_effect = side_effect
     multiclass_estimator.classes_ = np.array([0, 1, 2])
     return multiclass_estimator
 
+
 def test_inductive_venn_abers_calibrator(binary_calibration_data, binary_test_scores):
     scores, ground_truth_labels = binary_calibration_data
     test_scores = binary_test_scores
@@ -124,22 +161,76 @@ def test_inductive_venn_abers_calibrator(binary_calibration_data, binary_test_sc
     intervalls = calibrator.predict_intervall(test_scores[:, 1])
     proba = calibrator.predict_proba(test_scores[:, 1])
 
-    assert_array_almost_equal(calibrator._F1, np.array([0, 0.33333333, 0.375, 0.375, 0.4, 0.5, 0.5, 0.66666666, 0.66666666, 1.0, 1.0]))
-    assert_array_almost_equal(calibrator._F0, np.array([0, 0, 0.2, 0.2, 0.2, 0.25, 0.25, 0.33333333, 0.33333333, 0.5, 0.66666666]))
-    assert_array_almost_equal(intervalls, np.array([[0.66666666, 1.0], [0.25, 0.66666666], [0, 0.33333333], [0.66666666, 1.0], [0, 0.33333333], [0.25, 0.5], [0,0.33333333]]))
-    assert_array_almost_equal(proba, np.array([0.74999999, 0.47058823, 0.24999999, 0.74999999, 0.24999999, 0.4, 0.24999999]))
-
-def test_cross_venn_abers_calibrator(binary_calibration_data, binary_test_scores, binary_cal_X):
+    assert_array_almost_equal(
+        calibrator._F1,
+        np.array(
+            [
+                0,
+                0.33333333,
+                0.375,
+                0.375,
+                0.4,
+                0.5,
+                0.5,
+                0.66666666,
+                0.66666666,
+                1.0,
+                1.0,
+            ]
+        ),
+    )
+    assert_array_almost_equal(
+        calibrator._F0,
+        np.array(
+            [0, 0, 0.2, 0.2, 0.2, 0.25, 0.25, 0.33333333, 0.33333333, 0.5, 0.66666666]
+        ),
+    )
+    assert_array_almost_equal(
+        intervalls,
+        np.array(
+            [
+                [0.66666666, 1.0],
+                [0.25, 0.66666666],
+                [0, 0.33333333],
+                [0.66666666, 1.0],
+                [0, 0.33333333],
+                [0.25, 0.5],
+                [0, 0.33333333],
+            ]
+        ),
+    )
+    assert_array_almost_equal(
+        proba,
+        np.array(
+            [
+                0.74999999,
+                0.47058823,
+                0.24999999,
+                0.74999999,
+                0.24999999,
+                0.4,
+                0.24999999,
+            ]
+        ),
+    )
+
+
+def test_cross_venn_abers_calibrator(
+    binary_calibration_data, binary_test_scores, binary_cal_X
+):
     cal_scores, y_cal = binary_calibration_data
     test_scores = binary_test_scores
 
     calibrator = _CrossVennAbersCalibrator(LogisticRegression(), n_folds=5)
     calibrator.fit(y_cal, cal_scores, X=binary_cal_X)
     proba = calibrator.predict_proba(test_scores[:, 1])
-    expected = np.array([0.602499, 0.51438, 0.397501, 0.602499, 0.346239, 0.51438, 0.328119])
+    expected = np.array(
+        [0.602499, 0.51438, 0.397501, 0.602499, 0.346239, 0.51438, 0.328119]
+    )
     assert len(calibrator.ivaps) == 5
     assert_array_almost_equal(proba, expected)
 
+
 def test_platt_scaling(binary_calibration_data, binary_test_scores):
     cal_scores, cal_labels = binary_calibration_data
     calibrator = _PlattScaling()
@@ -147,16 +238,33 @@ def test_platt_scaling(binary_calibration_data, binary_test_scores):
     proba = calibrator.predict_proba(binary_test_scores[:, 1])
 
     assert proba.shape == (len(binary_test_scores),)
-    assert_array_almost_equal(proba, np.array([0.8827398, 0.46130191, 0.15976229, 0.97756289, 0.07045925, 0.42011212, 0.01095897]))
+    assert_array_almost_equal(
+        proba,
+        np.array(
+            [
+                0.8827398,
+                0.46130191,
+                0.15976229,
+                0.97756289,
+                0.07045925,
+                0.42011212,
+                0.01095897,
+            ]
+        ),
+    )
+
 
 def test_isotonic_regression(binary_calibration_data, binary_test_scores):
     cal_scores, cal_labels = binary_calibration_data
     calibrator = _IsotonicRegression()
     calibrator.fit(cal_labels, cal_scores[:, 1])
     proba = calibrator.predict_proba(binary_test_scores[:, 1])
-    
+
     assert proba.shape == (len(binary_test_scores),)
-    assert_array_almost_equal(proba, np.array([1.0, 0.33333333, 0.0, 1.0, 0.0, 0.33333333, 0.0]))
+    assert_array_almost_equal(
+        proba, np.array([1.0, 0.33333333, 0.0, 1.0, 0.0, 0.33333333, 0.0])
+    )
+
 
 def test_beta_calibration(binary_calibration_data, binary_test_scores):
     cal_scores, cal_labels = binary_calibration_data
@@ -165,22 +273,32 @@ def test_beta_calibration(binary_calibration_data, binary_test_scores):
     proba = calibrator.predict_proba(binary_test_scores[:, 1])
 
     assert proba.shape == (len(binary_test_scores),)
-    assert_array_almost_equal(proba, np.array([0.526125, 0.423743, 0.363907, 0.785855, 0.323201, 0.417089, 0.0]))
+    assert_array_almost_equal(
+        proba,
+        np.array([0.526125, 0.423743, 0.363907, 0.785855, 0.323201, 0.417089, 0.0]),
+    )
+
 
 def test_illegal_calibration_method_raises_error(binary_mock_estimator):
     with pytest.raises(ValueError, match="abc is not a valid calibration method."):
         _Calibrator(binary_mock_estimator, method="abc")
 
+
 def test_not_fitted_calibrator_throws_error(binary_test_scores, binary_mock_estimator):
-    for calibrator in [_PlattScaling(), 
-                       _IsotonicRegression(),
-                       _BetaCalibrator(),
-                       _InductiveVennAbersCalibrator(), 
-                       _CrossVennAbersCalibrator(binary_mock_estimator)]:
-            with pytest.raises(NotFittedError):
-                calibrator.predict_proba(binary_test_scores)
-
-def test_valid_calibration(binary_calibration_data, binary_test_scores, binary_cal_X, binary_mock_estimator):
+    for calibrator in [
+        _PlattScaling(),
+        _IsotonicRegression(),
+        _BetaCalibrator(),
+        _InductiveVennAbersCalibrator(),
+        _CrossVennAbersCalibrator(binary_mock_estimator),
+    ]:
+        with pytest.raises(NotFittedError):
+            calibrator.predict_proba(binary_test_scores)
+
+
+def test_valid_calibration(
+    binary_calibration_data, binary_test_scores, binary_cal_X, binary_mock_estimator
+):
     _, y_cal = binary_calibration_data
 
     for method in _Calibrator.available_methods:
@@ -191,9 +309,12 @@ def test_valid_calibration(binary_calibration_data, binary_test_scores, binary_c
         calibrator.fit(X=binary_cal_X, y=y_cal)
         proba = calibrator.predict_proba(binary_test_scores)
 
-        assert proba.shape == (len(binary_test_scores),2)
+        assert proba.shape == (len(binary_test_scores), 2)
 
-def test_multiclass_calibration(multiclass_calibration_data, multiclass_test_scores, multiclass_mock_estimator):
+
+def test_multiclass_calibration(
+    multiclass_calibration_data, multiclass_test_scores, multiclass_mock_estimator
+):
     scores, y_cal = multiclass_calibration_data
 
     calibrator = _Calibrator(multiclass_mock_estimator, method="ivap")
@@ -204,40 +325,57 @@ def test_multiclass_calibration(multiclass_calibration_data, multiclass_test_sco
     assert proba.shape == multiclass_test_scores.shape
     assert_array_almost_equal(np.sum(proba, axis=1), np.ones(len(proba)))
 
-    expected = np.array([[0.27777778, 0.55555556, 0.16666667],
-                         [0.52173913, 0.13043478, 0.34782609],
-                         [0.33333333, 0.16666667, 0.5       ],
-                         [0.28571429, 0.57142857, 0.14285714],
-                         [0.13483146, 0.70786517, 0.15730337],
-                         [0.16666667, 0.41666667, 0.41666667],
-                         [0.42857143, 0.14285714, 0.42857143]])
-    
+    expected = np.array(
+        [
+            [0.27777778, 0.55555556, 0.16666667],
+            [0.52173913, 0.13043478, 0.34782609],
+            [0.33333333, 0.16666667, 0.5],
+            [0.28571429, 0.57142857, 0.14285714],
+            [0.13483146, 0.70786517, 0.15730337],
+            [0.16666667, 0.41666667, 0.41666667],
+            [0.42857143, 0.14285714, 0.42857143],
+        ]
+    )
+
     assert_array_almost_equal(proba, expected)
 
-def test_multiclass_probability_merge(multiclass_mock_estimator, multiclass_calibration_data, multiclass_test_scores):
+
+def test_multiclass_probability_merge(
+    multiclass_mock_estimator, multiclass_calibration_data, multiclass_test_scores
+):
     calibrator = _Calibrator(estimator=multiclass_mock_estimator, method="ivap")
     X_cal, y_cal = multiclass_calibration_data
     calibrator.fit(X_cal, y_cal)
 
     calibrator_1 = Mock(spec=_PlattScaling)
-    calibrator_1.predict_proba.return_value = np.array([0.40, 0.60, 0.20, 0.80, 0.75, 0.25, 0.85])
-    
+    calibrator_1.predict_proba.return_value = np.array(
+        [0.40, 0.60, 0.20, 0.80, 0.75, 0.25, 0.85]
+    )
+
     calibrator_2 = Mock(spec=_PlattScaling)
-    calibrator_2.predict_proba.return_value = np.array([0.30, 0.70, 0.35, 0.65, 0.30, 0.70, 0.20])
+    calibrator_2.predict_proba.return_value = np.array(
+        [0.30, 0.70, 0.35, 0.65, 0.30, 0.70, 0.20]
+    )
 
     calibrator_3 = Mock(spec=_PlattScaling)
-    calibrator_3.predict_proba.return_value = np.array([0.55, 0.45, 0.10, 0.90, 0.45, 0.55, 0.40])
+    calibrator_3.predict_proba.return_value = np.array(
+        [0.55, 0.45, 0.10, 0.90, 0.45, 0.55, 0.40]
+    )
 
     calibrator.calibrators = [calibrator_1, calibrator_2, calibrator_3]
     proba = calibrator.predict_proba(multiclass_test_scores)
 
-    expected = np.array([[0.32, 0.24, 0.44],
-                         [0.34, 0.40, 0.26],
-                         [0.31, 0.54, 0.15],
-                         [0.34, 0.28, 0.38],
-                         [0.5,  0.20, 0.30],
-                         [0.17, 0.46, 0.37],
-                         [0.59, 0.14, 0.27]])
-    
+    expected = np.array(
+        [
+            [0.32, 0.24, 0.44],
+            [0.34, 0.40, 0.26],
+            [0.31, 0.54, 0.15],
+            [0.34, 0.28, 0.38],
+            [0.5, 0.20, 0.30],
+            [0.17, 0.46, 0.37],
+            [0.59, 0.14, 0.27],
+        ]
+    )
+
     assert_array_equal(np.sum(expected, axis=1), np.ones(len(expected)))
     assert_array_almost_equal(proba, expected, decimal=2)
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index 785adbb3..75b5dba9 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -7,10 +7,10 @@
 
 from hiclass.HierarchicalClassifier import HierarchicalClassifier
 from hiclass.metrics import (
-    precision, 
-    recall, 
-    f1, 
-    _multiclass_brier_score, 
+    precision,
+    recall,
+    f1,
+    _multiclass_brier_score,
     _log_loss,
     _expected_calibration_error,
     _static_calibration_error,
@@ -19,7 +19,7 @@
     log_loss,
     expected_calibration_error,
     static_calibration_error,
-    adaptive_calibration_error
+    adaptive_calibration_error,
 )
 
 
@@ -356,16 +356,22 @@ def test_empty_levels_2d_list_2():
 
 @pytest.fixture
 def uncertainty_data():
-    prob = [np.array([[0.88, 0.06, 0.06],
-                     [0.22, 0.48, 0.30],
-                     [0.33, 0.33, 0.34],
-                     [0.49, 0.40, 0.11],
-                     [0.23, 0.03, 0.74],
-                     [0.21, 0.67, 0.12],
-                     [0.34, 0.34, 0.32],
-                     [0.02, 0.77, 0.21],
-                     [0.44, 0.42, 0.14],
-                     [0.85, 0.13, 0.02]])]
+    prob = [
+        np.array(
+            [
+                [0.88, 0.06, 0.06],
+                [0.22, 0.48, 0.30],
+                [0.33, 0.33, 0.34],
+                [0.49, 0.40, 0.11],
+                [0.23, 0.03, 0.74],
+                [0.21, 0.67, 0.12],
+                [0.34, 0.34, 0.32],
+                [0.02, 0.77, 0.21],
+                [0.44, 0.42, 0.14],
+                [0.85, 0.13, 0.02],
+            ]
+        )
+    ]
 
     assert_array_equal(np.sum(prob[0], axis=1), np.ones(len(prob[0])))
 
@@ -374,15 +380,13 @@ def uncertainty_data():
 
     return prob, y_pred, y_true
 
+
 @pytest.fixture
 def uncertainty_data_multi_level():
-    prob = [np.array([[0.88, 0.06, 0.06],
-                     [0.22, 0.48, 0.30],
-                     [0.33, 0.33, 0.34]]),
-
-            np.array([[0.88, 0.06, 0.06],
-                     [0.22, 0.48, 0.30],
-                     [0.33, 0.33, 0.34]])]
+    prob = [
+        np.array([[0.88, 0.06, 0.06], [0.22, 0.48, 0.30], [0.33, 0.33, 0.34]]),
+        np.array([[0.88, 0.06, 0.06], [0.22, 0.48, 0.30], [0.33, 0.33, 0.34]]),
+    ]
 
     assert_array_equal(np.sum(prob[0], axis=1), np.ones(len(prob[0])))
 
@@ -391,6 +395,7 @@ def uncertainty_data_multi_level():
 
     return prob, y_pred, y_true
 
+
 def test_local_brier_score(uncertainty_data):
     prob, _, y_true = uncertainty_data
     obj = HierarchicalClassifier()
@@ -398,10 +403,11 @@ def test_local_brier_score(uncertainty_data):
     classifier._disambiguate = obj._disambiguate
     classifier.classes_ = [[0, 1, 2]]
     classifier.separator_ = "::HiClass::Separator::"
-    
+
     brier_score = _multiclass_brier_score(classifier, y_true, prob[0], level=0)
     assert math.isclose(brier_score, 0.34852, abs_tol=1e-4)
 
+
 def test_brier_score_multi_level(uncertainty_data_multi_level):
     prob, _, y_true = uncertainty_data_multi_level
     obj = HierarchicalClassifier()
@@ -417,6 +423,7 @@ def test_brier_score_multi_level(uncertainty_data_multi_level):
     assert math.isclose(brier_score_per_level[0], 0.48793, abs_tol=1e-4)
     assert math.isclose(brier_score_per_level[1], 0.48793, abs_tol=1e-4)
 
+
 def test_brier_score_single_level(uncertainty_data_multi_level):
     prob, _, y_true = uncertainty_data_multi_level
     prob = prob[1]
@@ -443,6 +450,7 @@ def test_local_log_loss(uncertainty_data):
     log_loss = _log_loss(classifier, y_true, prob[0], level=0)
     assert math.isclose(log_loss, 0.61790, abs_tol=1e-4)
 
+
 def test_log_loss_multi_level(uncertainty_data_multi_level):
     prob, _, y_true = uncertainty_data_multi_level
     obj = HierarchicalClassifier()
@@ -458,6 +466,7 @@ def test_log_loss_multi_level(uncertainty_data_multi_level):
     assert math.isclose(log_loss_per_level[0], 0.81348, abs_tol=1e-4)
     assert math.isclose(log_loss_per_level[1], 0.81348, abs_tol=1e-4)
 
+
 def test_log_loss_single_level(uncertainty_data_multi_level):
     prob, _, y_true = uncertainty_data_multi_level
     prob = prob[1]
@@ -472,6 +481,7 @@ def test_log_loss_single_level(uncertainty_data_multi_level):
     assert math.isclose(log_loss_1, 0.48793, abs_tol=1e-4)
     assert math.isclose(log_loss_2, 0.48793, abs_tol=1e-4)
 
+
 def test_local_expected_calibration_error(uncertainty_data):
     prob, y_pred, y_true = uncertainty_data
     obj = HierarchicalClassifier()
@@ -480,9 +490,12 @@ def test_local_expected_calibration_error(uncertainty_data):
     classifier.classes_ = [[0, 1, 2]]
     classifier.separator_ = "::HiClass::Separator::"
 
-    ece = _expected_calibration_error(classifier, y_true, prob[0], y_pred, level=0, n_bins=3)
+    ece = _expected_calibration_error(
+        classifier, y_true, prob[0], y_pred, level=0, n_bins=3
+    )
     assert math.isclose(ece, 0.118, abs_tol=1e-4)
 
+
 def test_expected_calibration_error_multi_level(uncertainty_data_multi_level):
     prob, y_pred, y_true = uncertainty_data_multi_level
     obj = HierarchicalClassifier()
@@ -490,14 +503,19 @@ def test_expected_calibration_error_multi_level(uncertainty_data_multi_level):
     classifier._disambiguate = obj._disambiguate
     classifier.classes_ = [[0, 1, 2], [3, 4, 5]]
     classifier.separator_ = "::HiClass::Separator::"
-    ece_avg = expected_calibration_error(classifier, y_true, prob, y_pred, agg="average")
+    ece_avg = expected_calibration_error(
+        classifier, y_true, prob, y_pred, agg="average"
+    )
     ece_sum = expected_calibration_error(classifier, y_true, prob, y_pred, agg="sum")
-    ece_per_level = expected_calibration_error(classifier, y_true, prob, y_pred, agg=None)
+    ece_per_level = expected_calibration_error(
+        classifier, y_true, prob, y_pred, agg=None
+    )
     assert math.isclose(ece_avg, 0.31333, abs_tol=1e-4)
     assert math.isclose(ece_sum, 0.62666, abs_tol=1e-4)
     assert math.isclose(ece_per_level[0], 0.31333, abs_tol=1e-4)
     assert math.isclose(ece_per_level[1], 0.31333, abs_tol=1e-4)
 
+
 def test_expected_calibration_error_single_level(uncertainty_data_multi_level):
     prob, y_pred, y_true = uncertainty_data_multi_level
     prob = prob[1]
@@ -511,6 +529,7 @@ def test_expected_calibration_error_single_level(uncertainty_data_multi_level):
     assert math.isclose(ece_1, 0.31333, abs_tol=1e-4)
     assert math.isclose(ece_2, 0.31333, abs_tol=1e-4)
 
+
 def test_local_static_calibration_error(uncertainty_data):
     prob, y_pred, y_true = uncertainty_data
     obj = HierarchicalClassifier()
@@ -519,9 +538,12 @@ def test_local_static_calibration_error(uncertainty_data):
     classifier.classes_ = [[0, 1, 2]]
     classifier.separator_ = "::HiClass::Separator::"
 
-    sce = _static_calibration_error(classifier, y_true, prob[0], y_pred, level=0, n_bins=3)
+    sce = _static_calibration_error(
+        classifier, y_true, prob[0], y_pred, level=0, n_bins=3
+    )
     assert math.isclose(sce, 0.3889, abs_tol=1e-3)
 
+
 def test_static_calibration_error_multi_level(uncertainty_data_multi_level):
     prob, y_pred, y_true = uncertainty_data_multi_level
     obj = HierarchicalClassifier()
@@ -552,6 +574,7 @@ def test_static_calibration_error_single_level(uncertainty_data_multi_level):
     assert math.isclose(sce_1, 0.44444, abs_tol=1e-4)
     assert math.isclose(sce_2, 0.44444, abs_tol=1e-4)
 
+
 def test_local_adaptive_calibration_error(uncertainty_data):
     prob, y_pred, y_true = uncertainty_data
     obj = HierarchicalClassifier()
@@ -560,9 +583,12 @@ def test_local_adaptive_calibration_error(uncertainty_data):
     classifier.classes_ = [[0, 1, 2]]
     classifier.separator_ = "::HiClass::Separator::"
 
-    ace = _adaptive_calibration_error(classifier, y_true, prob[0], y_pred, level=0, n_ranges=3)
+    ace = _adaptive_calibration_error(
+        classifier, y_true, prob[0], y_pred, level=0, n_ranges=3
+    )
     assert math.isclose(ace, 0.44, abs_tol=1e-3)
 
+
 def test_adaptive_calibration_error_multi_level(uncertainty_data_multi_level):
     prob, y_pred, y_true = uncertainty_data_multi_level
     obj = HierarchicalClassifier()
@@ -571,14 +597,19 @@ def test_adaptive_calibration_error_multi_level(uncertainty_data_multi_level):
     classifier.classes_ = [[0, 1, 2], [3, 4, 5]]
     classifier.separator_ = "::HiClass::Separator::"
 
-    ace_avg = adaptive_calibration_error(classifier, y_true, prob, y_pred, agg="average")
+    ace_avg = adaptive_calibration_error(
+        classifier, y_true, prob, y_pred, agg="average"
+    )
     ace_sum = adaptive_calibration_error(classifier, y_true, prob, y_pred, agg="sum")
-    ace_per_level = adaptive_calibration_error(classifier, y_true, prob, y_pred, agg=None)
+    ace_per_level = adaptive_calibration_error(
+        classifier, y_true, prob, y_pred, agg=None
+    )
     assert math.isclose(ace_avg, 0.13333, abs_tol=1e-4)
     assert math.isclose(ace_sum, 0.26666, abs_tol=1e-4)
     assert math.isclose(ace_per_level[0], 0.13333, abs_tol=1e-4)
     assert math.isclose(ace_per_level[1], 0.13333, abs_tol=1e-4)
 
+
 def test_adaptive_calibration_error_single_level(uncertainty_data_multi_level):
     prob, y_pred, y_true = uncertainty_data_multi_level
     prob = prob[1]
@@ -591,4 +622,4 @@ def test_adaptive_calibration_error_single_level(uncertainty_data_multi_level):
     ace_1 = adaptive_calibration_error(classifier, y_true, prob, y_pred, level=1)
     ace_2 = adaptive_calibration_error(classifier, y_true, [prob], y_pred, level=1)
     assert math.isclose(ace_1, 0.13333, abs_tol=1e-4)
-    assert math.isclose(ace_2, 0.13333, abs_tol=1e-4)
\ No newline at end of file
+    assert math.isclose(ace_2, 0.13333, abs_tol=1e-4)

From e4600d88aecd02b6335e44366cacc93054cb21c7 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Fri, 12 Apr 2024 11:25:52 +0200
Subject: [PATCH 50/65] change black version

---
 Pipfile                                       |   2 +-
 Pipfile.lock                                  | 357 +++++++++---------
 hiclass/Pipeline.py                           |   1 +
 .../ArithmeticMeanCombiner.py                 |   1 +
 .../GeometricMeanCombiner.py                  |   1 +
 .../probability_combiner/MultiplyCombiner.py  |   1 +
 .../ProbabilityCombiner.py                    |   1 +
 tests/test_calibration.py                     |  22 +-
 8 files changed, 196 insertions(+), 190 deletions(-)

diff --git a/Pipfile b/Pipfile
index c1c3eaa8..0c8fd579 100644
--- a/Pipfile
+++ b/Pipfile
@@ -19,7 +19,7 @@ pytest-cov = "3.0.0"
 twine = "*"
 sphinx = "4.1.1"
 sphinx-rtd-theme = "0.5.2"
-black = "22.10.0"
+black = {version = "24.3.0", extras = ["colorama"]}
 pre-commit = "2.20.0"
 pyfakefs = "*"
 shap = "0.44.1"
diff --git a/Pipfile.lock b/Pipfile.lock
index 55ad7e9a..29c3959b 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "c4fc4a3efecd8767f3637ef824942179038588d6b5288c3b7edc1e661a36446a"
+            "sha256": "3c7fbdc64a7f72c8b0ab1fb9b85784f94a53e91c6dffdf2a3eb4814fa4352aac"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -16,11 +16,11 @@
     "default": {
         "joblib": {
             "hashes": [
-                "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1",
-                "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9"
+                "sha256:1eb0dc091919cd384490de890cb5dfd538410a6d4b3b54eef09fb8c50b409b1c",
+                "sha256:42942470d4062537be4d54c83511186da1fc14ba354961a2114da91efa9a4ed7"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.3.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.4.0"
         },
         "networkx": {
             "hashes": [
@@ -76,31 +76,31 @@
         },
         "scikit-learn": {
             "hashes": [
-                "sha256:0df87de9ce1c0140f2818beef310fb2e2afdc1e66fc9ad587965577f17733649",
-                "sha256:14e4c88436ac96bf69eb6d746ac76a574c314a23c6961b7d344b38877f20fee1",
-                "sha256:1754b0c2409d6ed5a3380512d0adcf182a01363c669033a2b55cca429ed86a81",
-                "sha256:1afed6951bc9d2053c6ee9a518a466cbc9b07c6a3f9d43bfe734192b6125d508",
-                "sha256:1d491ef66e37f4e812db7e6c8286520c2c3fc61b34bf5e59b67b4ce528de93af",
-                "sha256:234b6bda70fdcae9e4abbbe028582ce99c280458665a155eed0b820599377d25",
-                "sha256:2a3ee19211ded1a52ee37b0a7b373a8bfc66f95353af058a210b692bd4cda0dd",
-                "sha256:4310bff71aa98b45b46cd26fa641309deb73a5d1c0461d181587ad4f30ea3c36",
-                "sha256:4ba516fcdc73d60e7f48cbb0bccb9acbdb21807de3651531208aac73c758e3ab",
-                "sha256:6145dfd9605b0b50ae72cdf72b61a2acd87501369a763b0d73d004710ebb76b5",
-                "sha256:629e09f772ad42f657ca60a1a52342eef786218dd20cf1369a3b8d085e55ef8f",
-                "sha256:712c1c69c45b58ef21635360b3d0a680ff7d83ac95b6f9b82cf9294070cda710",
-                "sha256:78cd27b4669513b50db4f683ef41ea35b5dddc797bd2bbd990d49897fd1c8a46",
-                "sha256:93d3d496ff1965470f9977d05e5ec3376fb1e63b10e4fda5e39d23c2d8969a30",
-                "sha256:9f43dd527dabff5521af2786a2f8de5ba381e182ec7292663508901cf6ceaf6e",
-                "sha256:a1e289f33f613cefe6707dead50db31930530dc386b6ccff176c786335a7b01c",
-                "sha256:aa0029b78ef59af22cfbd833e8ace8526e4df90212db7ceccbea582ebb5d6794",
-                "sha256:c02e27d65b0c7dc32f2c5eb601aaf5530b7a02bfbe92438188624524878336f2",
-                "sha256:c540aaf44729ab5cd4bd5e394f2b375e65ceaea9cdd8c195788e70433d91bbc5",
-                "sha256:ce03506ccf5f96b7e9030fea7eb148999b254c44c10182ac55857bc9b5d4815f",
-                "sha256:d7cd3a77c32879311f2aa93466d3c288c955ef71d191503cf0677c3340ae8ae0"
+                "sha256:1d0b25d9c651fd050555aadd57431b53d4cf664e749069da77f3d52c5ad14b3b",
+                "sha256:36f0ea5d0f693cb247a073d21a4123bdf4172e470e6d163c12b74cbb1536cf38",
+                "sha256:426d258fddac674fdf33f3cb2d54d26f49406e2599dbf9a32b4d1696091d4256",
+                "sha256:44c62f2b124848a28fd695db5bc4da019287abf390bfce602ddc8aa1ec186aae",
+                "sha256:45dee87ac5309bb82e3ea633955030df9bbcb8d2cdb30383c6cd483691c546cc",
+                "sha256:49d64ef6cb8c093d883e5a36c4766548d974898d378e395ba41a806d0e824db8",
+                "sha256:5460a1a5b043ae5ae4596b3126a4ec33ccba1b51e7ca2c5d36dac2169f62ab1d",
+                "sha256:5cd7b524115499b18b63f0c96f4224eb885564937a0b3477531b2b63ce331904",
+                "sha256:671e2f0c3f2c15409dae4f282a3a619601fa824d2c820e5b608d9d775f91780c",
+                "sha256:68b8404841f944a4a1459b07198fa2edd41a82f189b44f3e1d55c104dbc2e40c",
+                "sha256:81bf5d8bbe87643103334032dd82f7419bc8c8d02a763643a6b9a5c7288c5054",
+                "sha256:8539a41b3d6d1af82eb629f9c57f37428ff1481c1e34dddb3b9d7af8ede67ac5",
+                "sha256:87440e2e188c87db80ea4023440923dccbd56fbc2d557b18ced00fef79da0727",
+                "sha256:90378e1747949f90c8f385898fff35d73193dfcaec3dd75d6b542f90c4e89755",
+                "sha256:b0203c368058ab92efc6168a1507d388d41469c873e96ec220ca8e74079bf62e",
+                "sha256:c97a50b05c194be9146d61fe87dbf8eac62b203d9e87a3ccc6ae9aed2dfaf361",
+                "sha256:d36d0bc983336bbc1be22f9b686b50c964f593c8a9a913a792442af9bf4f5e68",
+                "sha256:d762070980c17ba3e9a4a1e043ba0518ce4c55152032f1af0ca6f39b376b5928",
+                "sha256:d9993d5e78a8148b1d0fdf5b15ed92452af5581734129998c26f481c46586d68",
+                "sha256:daa1c471d95bad080c6e44b4946c9390a4842adc3082572c20e4f8884e39e959",
+                "sha256:ff4effe5a1d4e8fed260a83a163f7dbf4f6087b54528d8880bab1d1377bd78be"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.9'",
-            "version": "==1.4.1.post1"
+            "version": "==1.4.2"
         },
         "scipy": {
             "hashes": [
@@ -184,32 +184,35 @@
             "version": "==1.0.0"
         },
         "black": {
-            "hashes": [
-                "sha256:14ff67aec0a47c424bc99b71005202045dc09270da44a27848d534600ac64fc7",
-                "sha256:197df8509263b0b8614e1df1756b1dd41be6738eed2ba9e9769f3880c2b9d7b6",
-                "sha256:1e464456d24e23d11fced2bc8c47ef66d471f845c7b7a42f3bd77bf3d1789650",
-                "sha256:2039230db3c6c639bd84efe3292ec7b06e9214a2992cd9beb293d639c6402edb",
-                "sha256:21199526696b8f09c3997e2b4db8d0b108d801a348414264d2eb8eb2532e540d",
-                "sha256:2644b5d63633702bc2c5f3754b1b475378fbbfb481f62319388235d0cd104c2d",
-                "sha256:432247333090c8c5366e69627ccb363bc58514ae3e63f7fc75c54b1ea80fa7de",
-                "sha256:444ebfb4e441254e87bad00c661fe32df9969b2bf224373a448d8aca2132b395",
-                "sha256:5b9b29da4f564ba8787c119f37d174f2b69cdfdf9015b7d8c5c16121ddc054ae",
-                "sha256:5cc42ca67989e9c3cf859e84c2bf014f6633db63d1cbdf8fdb666dcd9e77e3fa",
-                "sha256:5d8f74030e67087b219b032aa33a919fae8806d49c867846bfacde57f43972ef",
-                "sha256:72ef3925f30e12a184889aac03d77d031056860ccae8a1e519f6cbb742736383",
-                "sha256:819dc789f4498ecc91438a7de64427c73b45035e2e3680c92e18795a839ebb66",
-                "sha256:915ace4ff03fdfff953962fa672d44be269deb2eaf88499a0f8805221bc68c87",
-                "sha256:9311e99228ae10023300ecac05be5a296f60d2fd10fff31cf5c1fa4ca4b1988d",
-                "sha256:974308c58d057a651d182208a484ce80a26dac0caef2895836a92dd6ebd725e0",
-                "sha256:b8b49776299fece66bffaafe357d929ca9451450f5466e997a7285ab0fe28e3b",
-                "sha256:c957b2b4ea88587b46cf49d1dc17681c1e672864fd7af32fc1e9664d572b3458",
-                "sha256:e41a86c6c650bcecc6633ee3180d80a025db041a8e2398dcc059b3afa8382cd4",
-                "sha256:f513588da599943e0cde4e32cc9879e825d58720d6557062d1098c5ad80080e1",
-                "sha256:fba8a281e570adafb79f7755ac8721b6cf1bbf691186a287e990c7929c7692ff"
+            "extras": [
+                "colorama"
+            ],
+            "hashes": [
+                "sha256:2818cf72dfd5d289e48f37ccfa08b460bf469e67fb7c4abb07edc2e9f16fb63f",
+                "sha256:41622020d7120e01d377f74249e677039d20e6344ff5851de8a10f11f513bf93",
+                "sha256:4acf672def7eb1725f41f38bf6bf425c8237248bb0804faa3965c036f7672d11",
+                "sha256:4be5bb28e090456adfc1255e03967fb67ca846a03be7aadf6249096100ee32d0",
+                "sha256:4f1373a7808a8f135b774039f61d59e4be7eb56b2513d3d2f02a8b9365b8a8a9",
+                "sha256:56f52cfbd3dabe2798d76dbdd299faa046a901041faf2cf33288bc4e6dae57b5",
+                "sha256:65b76c275e4c1c5ce6e9870911384bff5ca31ab63d19c76811cb1fb162678213",
+                "sha256:65c02e4ea2ae09d16314d30912a58ada9a5c4fdfedf9512d23326128ac08ac3d",
+                "sha256:6905238a754ceb7788a73f02b45637d820b2f5478b20fec82ea865e4f5d4d9f7",
+                "sha256:79dcf34b33e38ed1b17434693763301d7ccbd1c5860674a8f871bd15139e7837",
+                "sha256:7bb041dca0d784697af4646d3b62ba4a6b028276ae878e53f6b4f74ddd6db99f",
+                "sha256:7d5e026f8da0322b5662fa7a8e752b3fa2dac1c1cbc213c3d7ff9bdd0ab12395",
+                "sha256:9f50ea1132e2189d8dff0115ab75b65590a3e97de1e143795adb4ce317934995",
+                "sha256:a0c9c4a0771afc6919578cec71ce82a3e31e054904e7197deacbc9382671c41f",
+                "sha256:aadf7a02d947936ee418777e0247ea114f78aff0d0959461057cae8a04f20597",
+                "sha256:b5991d523eee14756f3c8d5df5231550ae8993e2286b8014e2fdea7156ed0959",
+                "sha256:bf21b7b230718a5f08bd32d5e4f1db7fc8788345c8aea1d155fc17852b3410f5",
+                "sha256:c45f8dff244b3c431b36e3224b6be4a127c6aca780853574c00faf99258041eb",
+                "sha256:c7ed6668cbbfcd231fa0dc1b137d3e40c04c7f786e626b405c62bcd5db5857e4",
+                "sha256:d7de8d330763c66663661a1ffd432274a2f92f07feeddd89ffd085b5744f85e7",
+                "sha256:e19cb1c6365fd6dc38a6eae2dcb691d7d83935c10215aef8e6c38edee3f77abd",
+                "sha256:e2af80566f43c85f5797365077fb64a393861a3730bd110971ab7a0c94e873e7"
             ],
-            "index": "pypi",
-            "markers": "python_version >= '3.7'",
-            "version": "==22.10.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==24.3.0"
         },
         "certifi": {
             "hashes": [
@@ -425,11 +428,11 @@
         },
         "filelock": {
             "hashes": [
-                "sha256:5ffa845303983e7a0b7ae17636509bc97997d58afeafa72fb141a17b152284cb",
-                "sha256:a79895a25bbefdf55d1a2a0a80968f7dbb28edcd6d4234a0afb3f37ecde4b546"
+                "sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f",
+                "sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.13.3"
+            "version": "==3.13.4"
         },
         "flake8": {
             "hashes": [
@@ -450,11 +453,11 @@
         },
         "idna": {
             "hashes": [
-                "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca",
-                "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"
+                "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc",
+                "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==3.6"
+            "version": "==3.7"
         },
         "imagesize": {
             "hashes": [
@@ -514,11 +517,11 @@
         },
         "joblib": {
             "hashes": [
-                "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1",
-                "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9"
+                "sha256:1eb0dc091919cd384490de890cb5dfd538410a6d4b3b54eef09fb8c50b409b1c",
+                "sha256:42942470d4062537be4d54c83511186da1fc14ba354961a2114da91efa9a4ed7"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.3.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.4.0"
         },
         "keyring": {
             "hashes": [
@@ -769,38 +772,36 @@
         },
         "pandas": {
             "hashes": [
-                "sha256:04f6ec3baec203c13e3f8b139fb0f9f86cd8c0b94603ae3ae8ce9a422e9f5bee",
-                "sha256:06cf591dbaefb6da9de8472535b185cba556d0ce2e6ed28e21d919704fef1a9e",
-                "sha256:0ab90f87093c13f3e8fa45b48ba9f39181046e8f3317d3aadb2fffbb1b978572",
-                "sha256:0f573ab277252ed9aaf38240f3b54cfc90fff8e5cab70411ee1d03f5d51f3944",
-                "sha256:101d0eb9c5361aa0146f500773395a03839a5e6ecde4d4b6ced88b7e5a1a6403",
-                "sha256:11940e9e3056576ac3244baef2fedade891977bcc1cb7e5cc8f8cc7d603edc89",
-                "sha256:1ba21b1d5c0e43416218db63037dbe1a01fc101dc6e6024bcad08123e48004ab",
-                "sha256:4aa1d8707812a658debf03824016bf5ea0d516afdea29b7dc14cf687bc4d4ec6",
-                "sha256:4acf681325ee1c7f950d058b05a820441075b0dd9a2adf5c4835b9bc056bf4fb",
-                "sha256:53680dc9b2519cbf609c62db3ed7c0b499077c7fefda564e330286e619ff0dd9",
-                "sha256:739cc70eaf17d57608639e74d63387b0d8594ce02f69e7a0b046f117974b3019",
-                "sha256:76f27a809cda87e07f192f001d11adc2b930e93a2b0c4a236fde5429527423be",
-                "sha256:7d2ed41c319c9fb4fd454fe25372028dfa417aacb9790f68171b2e3f06eae8cd",
-                "sha256:88ecb5c01bb9ca927ebc4098136038519aa5d66b44671861ffab754cae75102c",
-                "sha256:8df8612be9cd1c7797c93e1c5df861b2ddda0b48b08f2c3eaa0702cf88fb5f88",
-                "sha256:94e714a1cca63e4f5939cdce5f29ba8d415d85166be3441165edd427dc9f6bc0",
-                "sha256:9bd8a40f47080825af4317d0340c656744f2bfdb6819f818e6ba3cd24c0e1397",
-                "sha256:9d1265545f579edf3f8f0cb6f89f234f5e44ba725a34d86535b1a1d38decbccc",
-                "sha256:a935a90a76c44fe170d01e90a3594beef9e9a6220021acfb26053d01426f7dc2",
-                "sha256:af5d3c00557d657c8773ef9ee702c61dd13b9d7426794c9dfeb1dc4a0bf0ebc7",
-                "sha256:c2ce852e1cf2509a69e98358e8458775f89599566ac3775e70419b98615f4b06",
-                "sha256:c38ce92cb22a4bea4e3929429aa1067a454dcc9c335799af93ba9be21b6beb51",
-                "sha256:c391f594aae2fd9f679d419e9a4d5ba4bce5bb13f6a989195656e7dc4b95c8f0",
-                "sha256:c70e00c2d894cb230e5c15e4b1e1e6b2b478e09cf27cc593a11ef955b9ecc81a",
-                "sha256:df0c37ebd19e11d089ceba66eba59a168242fc6b7155cba4ffffa6eccdfb8f16",
-                "sha256:e97fbb5387c69209f134893abc788a6486dbf2f9e511070ca05eed4b930b1b02",
-                "sha256:f02a3a6c83df4026e55b63c1f06476c9aa3ed6af3d89b4f04ea656ccdaaaa359",
-                "sha256:f821213d48f4ab353d20ebc24e4faf94ba40d76680642fb7ce2ea31a3ad94f9b",
-                "sha256:f9d3558d263073ed95e46f4650becff0c5e1ffe0fc3a015de3c79283dfbdb3df"
+                "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863",
+                "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2",
+                "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1",
+                "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad",
+                "sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db",
+                "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76",
+                "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51",
+                "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32",
+                "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08",
+                "sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b",
+                "sha256:640cef9aa381b60e296db324337a554aeeb883ead99dc8f6c18e81a93942f5f4",
+                "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921",
+                "sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288",
+                "sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee",
+                "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0",
+                "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24",
+                "sha256:8e5a0b00e1e56a842f922e7fae8ae4077aee4af0acb5ae3622bd4b4c30aedf99",
+                "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151",
+                "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce",
+                "sha256:92fd6b027924a7e178ac202cfbe25e53368db90d56872d20ffae94b96c7acc57",
+                "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef",
+                "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54",
+                "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a",
+                "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23",
+                "sha256:ddf818e4e6c7c6f4f7c8a12709696d193976b591cc7dc50588d3d1a6b5dc8772",
+                "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce",
+                "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==2.2.1"
+            "version": "==2.2.2"
         },
         "pathspec": {
             "hashes": [
@@ -870,12 +871,12 @@
         },
         "pyfakefs": {
             "hashes": [
-                "sha256:969096d84b5b986f4f84399d03f4900381a3880d03adcdbd609566a4baf39bf9",
-                "sha256:96e52554621a3af7b8171f8660debb65781bcd0cb0bdddea8b12e1b7871c33f3"
+                "sha256:20cb51e860c2f3ff83859162ad5134bb8b0a1e7a81df0a18cfccc4862d0d9dcc",
+                "sha256:21d6a3276d9c964510c85cef0c568920d53ec9033da9b2a2c616489cedbe700a"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.7'",
-            "version": "==5.4.0"
+            "version": "==5.4.1"
         },
         "pyflakes": {
             "hashes": [
@@ -1049,31 +1050,31 @@
         },
         "scikit-learn": {
             "hashes": [
-                "sha256:0df87de9ce1c0140f2818beef310fb2e2afdc1e66fc9ad587965577f17733649",
-                "sha256:14e4c88436ac96bf69eb6d746ac76a574c314a23c6961b7d344b38877f20fee1",
-                "sha256:1754b0c2409d6ed5a3380512d0adcf182a01363c669033a2b55cca429ed86a81",
-                "sha256:1afed6951bc9d2053c6ee9a518a466cbc9b07c6a3f9d43bfe734192b6125d508",
-                "sha256:1d491ef66e37f4e812db7e6c8286520c2c3fc61b34bf5e59b67b4ce528de93af",
-                "sha256:234b6bda70fdcae9e4abbbe028582ce99c280458665a155eed0b820599377d25",
-                "sha256:2a3ee19211ded1a52ee37b0a7b373a8bfc66f95353af058a210b692bd4cda0dd",
-                "sha256:4310bff71aa98b45b46cd26fa641309deb73a5d1c0461d181587ad4f30ea3c36",
-                "sha256:4ba516fcdc73d60e7f48cbb0bccb9acbdb21807de3651531208aac73c758e3ab",
-                "sha256:6145dfd9605b0b50ae72cdf72b61a2acd87501369a763b0d73d004710ebb76b5",
-                "sha256:629e09f772ad42f657ca60a1a52342eef786218dd20cf1369a3b8d085e55ef8f",
-                "sha256:712c1c69c45b58ef21635360b3d0a680ff7d83ac95b6f9b82cf9294070cda710",
-                "sha256:78cd27b4669513b50db4f683ef41ea35b5dddc797bd2bbd990d49897fd1c8a46",
-                "sha256:93d3d496ff1965470f9977d05e5ec3376fb1e63b10e4fda5e39d23c2d8969a30",
-                "sha256:9f43dd527dabff5521af2786a2f8de5ba381e182ec7292663508901cf6ceaf6e",
-                "sha256:a1e289f33f613cefe6707dead50db31930530dc386b6ccff176c786335a7b01c",
-                "sha256:aa0029b78ef59af22cfbd833e8ace8526e4df90212db7ceccbea582ebb5d6794",
-                "sha256:c02e27d65b0c7dc32f2c5eb601aaf5530b7a02bfbe92438188624524878336f2",
-                "sha256:c540aaf44729ab5cd4bd5e394f2b375e65ceaea9cdd8c195788e70433d91bbc5",
-                "sha256:ce03506ccf5f96b7e9030fea7eb148999b254c44c10182ac55857bc9b5d4815f",
-                "sha256:d7cd3a77c32879311f2aa93466d3c288c955ef71d191503cf0677c3340ae8ae0"
+                "sha256:1d0b25d9c651fd050555aadd57431b53d4cf664e749069da77f3d52c5ad14b3b",
+                "sha256:36f0ea5d0f693cb247a073d21a4123bdf4172e470e6d163c12b74cbb1536cf38",
+                "sha256:426d258fddac674fdf33f3cb2d54d26f49406e2599dbf9a32b4d1696091d4256",
+                "sha256:44c62f2b124848a28fd695db5bc4da019287abf390bfce602ddc8aa1ec186aae",
+                "sha256:45dee87ac5309bb82e3ea633955030df9bbcb8d2cdb30383c6cd483691c546cc",
+                "sha256:49d64ef6cb8c093d883e5a36c4766548d974898d378e395ba41a806d0e824db8",
+                "sha256:5460a1a5b043ae5ae4596b3126a4ec33ccba1b51e7ca2c5d36dac2169f62ab1d",
+                "sha256:5cd7b524115499b18b63f0c96f4224eb885564937a0b3477531b2b63ce331904",
+                "sha256:671e2f0c3f2c15409dae4f282a3a619601fa824d2c820e5b608d9d775f91780c",
+                "sha256:68b8404841f944a4a1459b07198fa2edd41a82f189b44f3e1d55c104dbc2e40c",
+                "sha256:81bf5d8bbe87643103334032dd82f7419bc8c8d02a763643a6b9a5c7288c5054",
+                "sha256:8539a41b3d6d1af82eb629f9c57f37428ff1481c1e34dddb3b9d7af8ede67ac5",
+                "sha256:87440e2e188c87db80ea4023440923dccbd56fbc2d557b18ced00fef79da0727",
+                "sha256:90378e1747949f90c8f385898fff35d73193dfcaec3dd75d6b542f90c4e89755",
+                "sha256:b0203c368058ab92efc6168a1507d388d41469c873e96ec220ca8e74079bf62e",
+                "sha256:c97a50b05c194be9146d61fe87dbf8eac62b203d9e87a3ccc6ae9aed2dfaf361",
+                "sha256:d36d0bc983336bbc1be22f9b686b50c964f593c8a9a913a792442af9bf4f5e68",
+                "sha256:d762070980c17ba3e9a4a1e043ba0518ce4c55152032f1af0ca6f39b376b5928",
+                "sha256:d9993d5e78a8148b1d0fdf5b15ed92452af5581734129998c26f481c46586d68",
+                "sha256:daa1c471d95bad080c6e44b4946c9390a4842adc3082572c20e4f8884e39e959",
+                "sha256:ff4effe5a1d4e8fed260a83a163f7dbf4f6087b54528d8880bab1d1377bd78be"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.9'",
-            "version": "==1.4.1.post1"
+            "version": "==1.4.2"
         },
         "scipy": {
             "hashes": [
@@ -1464,11 +1465,11 @@
         },
         "filelock": {
             "hashes": [
-                "sha256:5ffa845303983e7a0b7ae17636509bc97997d58afeafa72fb141a17b152284cb",
-                "sha256:a79895a25bbefdf55d1a2a0a80968f7dbb28edcd6d4234a0afb3f37ecde4b546"
+                "sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f",
+                "sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.13.3"
+            "version": "==3.13.4"
         },
         "frozenlist": {
             "hashes": [
@@ -1555,19 +1556,19 @@
         },
         "idna": {
             "hashes": [
-                "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca",
-                "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"
+                "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc",
+                "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==3.6"
+            "version": "==3.7"
         },
         "joblib": {
             "hashes": [
-                "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1",
-                "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9"
+                "sha256:1eb0dc091919cd384490de890cb5dfd538410a6d4b3b54eef09fb8c50b409b1c",
+                "sha256:42942470d4062537be4d54c83511186da1fc14ba354961a2114da91efa9a4ed7"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.3.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.4.0"
         },
         "jsonschema": {
             "hashes": [
@@ -1754,38 +1755,36 @@
         },
         "pandas": {
             "hashes": [
-                "sha256:04f6ec3baec203c13e3f8b139fb0f9f86cd8c0b94603ae3ae8ce9a422e9f5bee",
-                "sha256:06cf591dbaefb6da9de8472535b185cba556d0ce2e6ed28e21d919704fef1a9e",
-                "sha256:0ab90f87093c13f3e8fa45b48ba9f39181046e8f3317d3aadb2fffbb1b978572",
-                "sha256:0f573ab277252ed9aaf38240f3b54cfc90fff8e5cab70411ee1d03f5d51f3944",
-                "sha256:101d0eb9c5361aa0146f500773395a03839a5e6ecde4d4b6ced88b7e5a1a6403",
-                "sha256:11940e9e3056576ac3244baef2fedade891977bcc1cb7e5cc8f8cc7d603edc89",
-                "sha256:1ba21b1d5c0e43416218db63037dbe1a01fc101dc6e6024bcad08123e48004ab",
-                "sha256:4aa1d8707812a658debf03824016bf5ea0d516afdea29b7dc14cf687bc4d4ec6",
-                "sha256:4acf681325ee1c7f950d058b05a820441075b0dd9a2adf5c4835b9bc056bf4fb",
-                "sha256:53680dc9b2519cbf609c62db3ed7c0b499077c7fefda564e330286e619ff0dd9",
-                "sha256:739cc70eaf17d57608639e74d63387b0d8594ce02f69e7a0b046f117974b3019",
-                "sha256:76f27a809cda87e07f192f001d11adc2b930e93a2b0c4a236fde5429527423be",
-                "sha256:7d2ed41c319c9fb4fd454fe25372028dfa417aacb9790f68171b2e3f06eae8cd",
-                "sha256:88ecb5c01bb9ca927ebc4098136038519aa5d66b44671861ffab754cae75102c",
-                "sha256:8df8612be9cd1c7797c93e1c5df861b2ddda0b48b08f2c3eaa0702cf88fb5f88",
-                "sha256:94e714a1cca63e4f5939cdce5f29ba8d415d85166be3441165edd427dc9f6bc0",
-                "sha256:9bd8a40f47080825af4317d0340c656744f2bfdb6819f818e6ba3cd24c0e1397",
-                "sha256:9d1265545f579edf3f8f0cb6f89f234f5e44ba725a34d86535b1a1d38decbccc",
-                "sha256:a935a90a76c44fe170d01e90a3594beef9e9a6220021acfb26053d01426f7dc2",
-                "sha256:af5d3c00557d657c8773ef9ee702c61dd13b9d7426794c9dfeb1dc4a0bf0ebc7",
-                "sha256:c2ce852e1cf2509a69e98358e8458775f89599566ac3775e70419b98615f4b06",
-                "sha256:c38ce92cb22a4bea4e3929429aa1067a454dcc9c335799af93ba9be21b6beb51",
-                "sha256:c391f594aae2fd9f679d419e9a4d5ba4bce5bb13f6a989195656e7dc4b95c8f0",
-                "sha256:c70e00c2d894cb230e5c15e4b1e1e6b2b478e09cf27cc593a11ef955b9ecc81a",
-                "sha256:df0c37ebd19e11d089ceba66eba59a168242fc6b7155cba4ffffa6eccdfb8f16",
-                "sha256:e97fbb5387c69209f134893abc788a6486dbf2f9e511070ca05eed4b930b1b02",
-                "sha256:f02a3a6c83df4026e55b63c1f06476c9aa3ed6af3d89b4f04ea656ccdaaaa359",
-                "sha256:f821213d48f4ab353d20ebc24e4faf94ba40d76680642fb7ce2ea31a3ad94f9b",
-                "sha256:f9d3558d263073ed95e46f4650becff0c5e1ffe0fc3a015de3c79283dfbdb3df"
+                "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863",
+                "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2",
+                "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1",
+                "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad",
+                "sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db",
+                "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76",
+                "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51",
+                "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32",
+                "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08",
+                "sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b",
+                "sha256:640cef9aa381b60e296db324337a554aeeb883ead99dc8f6c18e81a93942f5f4",
+                "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921",
+                "sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288",
+                "sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee",
+                "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0",
+                "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24",
+                "sha256:8e5a0b00e1e56a842f922e7fae8ae4077aee4af0acb5ae3622bd4b4c30aedf99",
+                "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151",
+                "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce",
+                "sha256:92fd6b027924a7e178ac202cfbe25e53368db90d56872d20ffae94b96c7acc57",
+                "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef",
+                "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54",
+                "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a",
+                "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23",
+                "sha256:ddf818e4e6c7c6f4f7c8a12709696d193976b591cc7dc50588d3d1a6b5dc8772",
+                "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce",
+                "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==2.2.1"
+            "version": "==2.2.2"
         },
         "protobuf": {
             "hashes": [
@@ -2026,31 +2025,31 @@
         },
         "scikit-learn": {
             "hashes": [
-                "sha256:0df87de9ce1c0140f2818beef310fb2e2afdc1e66fc9ad587965577f17733649",
-                "sha256:14e4c88436ac96bf69eb6d746ac76a574c314a23c6961b7d344b38877f20fee1",
-                "sha256:1754b0c2409d6ed5a3380512d0adcf182a01363c669033a2b55cca429ed86a81",
-                "sha256:1afed6951bc9d2053c6ee9a518a466cbc9b07c6a3f9d43bfe734192b6125d508",
-                "sha256:1d491ef66e37f4e812db7e6c8286520c2c3fc61b34bf5e59b67b4ce528de93af",
-                "sha256:234b6bda70fdcae9e4abbbe028582ce99c280458665a155eed0b820599377d25",
-                "sha256:2a3ee19211ded1a52ee37b0a7b373a8bfc66f95353af058a210b692bd4cda0dd",
-                "sha256:4310bff71aa98b45b46cd26fa641309deb73a5d1c0461d181587ad4f30ea3c36",
-                "sha256:4ba516fcdc73d60e7f48cbb0bccb9acbdb21807de3651531208aac73c758e3ab",
-                "sha256:6145dfd9605b0b50ae72cdf72b61a2acd87501369a763b0d73d004710ebb76b5",
-                "sha256:629e09f772ad42f657ca60a1a52342eef786218dd20cf1369a3b8d085e55ef8f",
-                "sha256:712c1c69c45b58ef21635360b3d0a680ff7d83ac95b6f9b82cf9294070cda710",
-                "sha256:78cd27b4669513b50db4f683ef41ea35b5dddc797bd2bbd990d49897fd1c8a46",
-                "sha256:93d3d496ff1965470f9977d05e5ec3376fb1e63b10e4fda5e39d23c2d8969a30",
-                "sha256:9f43dd527dabff5521af2786a2f8de5ba381e182ec7292663508901cf6ceaf6e",
-                "sha256:a1e289f33f613cefe6707dead50db31930530dc386b6ccff176c786335a7b01c",
-                "sha256:aa0029b78ef59af22cfbd833e8ace8526e4df90212db7ceccbea582ebb5d6794",
-                "sha256:c02e27d65b0c7dc32f2c5eb601aaf5530b7a02bfbe92438188624524878336f2",
-                "sha256:c540aaf44729ab5cd4bd5e394f2b375e65ceaea9cdd8c195788e70433d91bbc5",
-                "sha256:ce03506ccf5f96b7e9030fea7eb148999b254c44c10182ac55857bc9b5d4815f",
-                "sha256:d7cd3a77c32879311f2aa93466d3c288c955ef71d191503cf0677c3340ae8ae0"
+                "sha256:1d0b25d9c651fd050555aadd57431b53d4cf664e749069da77f3d52c5ad14b3b",
+                "sha256:36f0ea5d0f693cb247a073d21a4123bdf4172e470e6d163c12b74cbb1536cf38",
+                "sha256:426d258fddac674fdf33f3cb2d54d26f49406e2599dbf9a32b4d1696091d4256",
+                "sha256:44c62f2b124848a28fd695db5bc4da019287abf390bfce602ddc8aa1ec186aae",
+                "sha256:45dee87ac5309bb82e3ea633955030df9bbcb8d2cdb30383c6cd483691c546cc",
+                "sha256:49d64ef6cb8c093d883e5a36c4766548d974898d378e395ba41a806d0e824db8",
+                "sha256:5460a1a5b043ae5ae4596b3126a4ec33ccba1b51e7ca2c5d36dac2169f62ab1d",
+                "sha256:5cd7b524115499b18b63f0c96f4224eb885564937a0b3477531b2b63ce331904",
+                "sha256:671e2f0c3f2c15409dae4f282a3a619601fa824d2c820e5b608d9d775f91780c",
+                "sha256:68b8404841f944a4a1459b07198fa2edd41a82f189b44f3e1d55c104dbc2e40c",
+                "sha256:81bf5d8bbe87643103334032dd82f7419bc8c8d02a763643a6b9a5c7288c5054",
+                "sha256:8539a41b3d6d1af82eb629f9c57f37428ff1481c1e34dddb3b9d7af8ede67ac5",
+                "sha256:87440e2e188c87db80ea4023440923dccbd56fbc2d557b18ced00fef79da0727",
+                "sha256:90378e1747949f90c8f385898fff35d73193dfcaec3dd75d6b542f90c4e89755",
+                "sha256:b0203c368058ab92efc6168a1507d388d41469c873e96ec220ca8e74079bf62e",
+                "sha256:c97a50b05c194be9146d61fe87dbf8eac62b203d9e87a3ccc6ae9aed2dfaf361",
+                "sha256:d36d0bc983336bbc1be22f9b686b50c964f593c8a9a913a792442af9bf4f5e68",
+                "sha256:d762070980c17ba3e9a4a1e043ba0518ce4c55152032f1af0ca6f39b376b5928",
+                "sha256:d9993d5e78a8148b1d0fdf5b15ed92452af5581734129998c26f481c46586d68",
+                "sha256:daa1c471d95bad080c6e44b4946c9390a4842adc3082572c20e4f8884e39e959",
+                "sha256:ff4effe5a1d4e8fed260a83a163f7dbf4f6087b54528d8880bab1d1377bd78be"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.9'",
-            "version": "==1.4.1.post1"
+            "version": "==1.4.2"
         },
         "scipy": {
             "hashes": [
diff --git a/hiclass/Pipeline.py b/hiclass/Pipeline.py
index f1202b79..a1d25ada 100644
--- a/hiclass/Pipeline.py
+++ b/hiclass/Pipeline.py
@@ -1,4 +1,5 @@
 """Custom Pipeline class that supports the `calibrate` method."""
+
 from sklearn.pipeline import Pipeline as skPipeline
 
 
diff --git a/hiclass/probability_combiner/ArithmeticMeanCombiner.py b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
index 01419bfc..8679e895 100644
--- a/hiclass/probability_combiner/ArithmeticMeanCombiner.py
+++ b/hiclass/probability_combiner/ArithmeticMeanCombiner.py
@@ -1,4 +1,5 @@
 """Defines the ArithmeticMeanCombiner."""
+
 import numpy as np
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
 from typing import List
diff --git a/hiclass/probability_combiner/GeometricMeanCombiner.py b/hiclass/probability_combiner/GeometricMeanCombiner.py
index 2a700983..9f643c63 100644
--- a/hiclass/probability_combiner/GeometricMeanCombiner.py
+++ b/hiclass/probability_combiner/GeometricMeanCombiner.py
@@ -1,4 +1,5 @@
 """Defines the GeometricMeanCombiner."""
+
 import numpy as np
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
 from typing import List
diff --git a/hiclass/probability_combiner/MultiplyCombiner.py b/hiclass/probability_combiner/MultiplyCombiner.py
index 36badb82..e693f940 100644
--- a/hiclass/probability_combiner/MultiplyCombiner.py
+++ b/hiclass/probability_combiner/MultiplyCombiner.py
@@ -1,4 +1,5 @@
 """Defines the MultiplyCombiner."""
+
 import numpy as np
 from hiclass.probability_combiner.ProbabilityCombiner import ProbabilityCombiner
 from typing import List
diff --git a/hiclass/probability_combiner/ProbabilityCombiner.py b/hiclass/probability_combiner/ProbabilityCombiner.py
index be9569f8..a2f2ecfc 100644
--- a/hiclass/probability_combiner/ProbabilityCombiner.py
+++ b/hiclass/probability_combiner/ProbabilityCombiner.py
@@ -1,4 +1,5 @@
 """Abstract class defining the structure of a probability combiner."""
+
 import abc
 import numpy as np
 from typing import List
diff --git a/tests/test_calibration.py b/tests/test_calibration.py
index 2f655c86..3d9844ff 100644
--- a/tests/test_calibration.py
+++ b/tests/test_calibration.py
@@ -82,11 +82,12 @@ def binary_cal_X():
 @pytest.fixture
 def binary_mock_estimator(binary_calibration_data, binary_test_scores):
     # return calibration scores or test scores depending on input size
-    side_effect = (
-        lambda X: binary_calibration_data[0]
-        if len(X) == len(binary_calibration_data[0])
-        else binary_test_scores
-    )
+    def side_effect(X):
+        return (
+            binary_calibration_data[0]
+            if len(X) == len(binary_calibration_data[0])
+            else binary_test_scores
+        )
 
     lr = LogisticRegression()
     binary_estimator = Mock(spec=lr)
@@ -140,11 +141,12 @@ def multiclass_test_scores():
 @pytest.fixture
 def multiclass_mock_estimator(multiclass_calibration_data, multiclass_test_scores):
     # return calibration scores or test scores depending on input size
-    side_effect = (
-        lambda X: multiclass_calibration_data[0]
-        if len(X) == len(multiclass_calibration_data[0])
-        else multiclass_test_scores
-    )
+    def side_effect(X):
+        return (
+            multiclass_calibration_data[0]
+            if len(X) == len(multiclass_calibration_data[0])
+            else multiclass_test_scores
+        )
 
     multiclass_estimator = Mock(spec=LogisticRegression)
     multiclass_estimator.predict_proba.side_effect = side_effect

From ad2f735a97f7d4e0f3c464731afa3c0521ffdf06 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Fri, 12 Apr 2024 11:31:09 +0200
Subject: [PATCH 51/65] correct test

---
 tests/test_calibration.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/test_calibration.py b/tests/test_calibration.py
index 3d9844ff..6f4440f2 100644
--- a/tests/test_calibration.py
+++ b/tests/test_calibration.py
@@ -277,7 +277,10 @@ def test_beta_calibration(binary_calibration_data, binary_test_scores):
     assert proba.shape == (len(binary_test_scores),)
     assert_array_almost_equal(
         proba,
-        np.array([0.526125, 0.423743, 0.363907, 0.785855, 0.323201, 0.417089, 0.0]),
+        np.array(
+            [0.526125, 0.423743, 0.363907, 0.785855, 0.323201, 0.417089, 0.0],
+        ),
+        decimal=3,
     )
 
 

From 6f9a1f588c7aee325ba3e7f42799c02737ae4622 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Wed, 17 Apr 2024 12:50:05 +0200
Subject: [PATCH 52/65] enable BetaCalibrator to handle null values;add test

---
 hiclass/_calibration/BetaCalibrator.py |  8 ++++++
 tests/test_calibration.py              | 36 ++++++++++++++++++++++++++
 2 files changed, 44 insertions(+)

diff --git a/hiclass/_calibration/BetaCalibrator.py b/hiclass/_calibration/BetaCalibrator.py
index 22c36dde..00bbb6e9 100644
--- a/hiclass/_calibration/BetaCalibrator.py
+++ b/hiclass/_calibration/BetaCalibrator.py
@@ -19,7 +19,15 @@ def fit(self, y: np.ndarray, scores: np.ndarray, X: np.ndarray = None):
             return self
 
         scores_1 = np.log(scores)
+        # replace negative infinity with limit for log(n), n -> -inf
+        replace_negative_inf = np.log(1e-300)
+        scores_1 = np.nan_to_num(scores_1, neginf=replace_negative_inf)
+
         scores_2 = -np.log(1 - scores)
+        # replace positive infinity with limit for log(n), n -> inf
+        replace_positive_inf = np.log(1e300)
+        scores_2 = np.nan_to_num(scores_2, posinf=replace_positive_inf)
+
         feature_matrix = np.column_stack((scores_1, scores_2))
 
         lr = LogisticRegression()
diff --git a/tests/test_calibration.py b/tests/test_calibration.py
index 6f4440f2..6e419895 100644
--- a/tests/test_calibration.py
+++ b/tests/test_calibration.py
@@ -284,6 +284,42 @@ def test_beta_calibration(binary_calibration_data, binary_test_scores):
     )
 
 
+def test_calibration_methods_can_handle_zeros(binary_test_scores):
+    cal_scores = np.array(
+        [
+            [0, 1],
+            [0, 1],
+            [0, 1],
+            [1, 0],
+            [1, 0],
+            [0, 1],
+            [0, 1],
+            [1, 0],
+            [1, 0],
+            [1, 0],
+            [1, 0],
+            [1, 0],
+        ],
+        dtype=np.float32,
+    )
+
+    cal_labels = np.array([1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0])
+    assert_array_equal(np.sum(cal_scores, axis=1), np.ones(len(cal_scores)))
+
+    for calibrator in [
+        _PlattScaling(),
+        _IsotonicRegression(),
+        _BetaCalibrator(),
+        _InductiveVennAbersCalibrator(),
+        _CrossVennAbersCalibrator(LogisticRegression()),
+    ]:
+        try:
+            calibrator.fit(cal_labels, cal_scores[:, 1], cal_scores)
+            calibrator.predict_proba(binary_test_scores[:, 1])
+        except ValueError as e:
+            pytest.fail(repr(e))
+
+
 def test_illegal_calibration_method_raises_error(binary_mock_estimator):
     with pytest.raises(ValueError, match="abc is not a valid calibration method."):
         _Calibrator(binary_mock_estimator, method="abc")

From 49ca50118f05f469a5dd712c619c5b9ae7cb5d18 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Fri, 19 Apr 2024 20:19:11 +0200
Subject: [PATCH 53/65] add method to normalize probabilities

---
 hiclass/LocalClassifierPerNode.py                   |  5 +++++
 hiclass/LocalClassifierPerParentNode.py             |  5 +++++
 hiclass/_calibration/Calibrator.py                  |  4 +++-
 hiclass/_calibration/VennAbersCalibrator.py         |  4 +++-
 hiclass/_hiclass_utils.py                           | 12 ++++++++++++
 hiclass/probability_combiner/ProbabilityCombiner.py |  8 ++------
 6 files changed, 30 insertions(+), 8 deletions(-)
 create mode 100644 hiclass/_hiclass_utils.py

diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index ba02edbc..9cb3b390 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -23,6 +23,8 @@
     init_strings as probability_combiner_init_strings,
 )
 
+from hiclass._hiclass_utils import _normalize_probabilities
+
 
 class LocalClassifierPerNode(BaseEstimator, HierarchicalClassifier):
     """
@@ -320,12 +322,15 @@ def predict_proba(self, X):
         self._remove_separator(y)
 
         # normalize probabilities
+        level_probability_list = _normalize_probabilities(level_probability_list)
+        """
         level_probability_list = [
             np.nan_to_num(
                 level_probabilities / level_probabilities.sum(axis=1, keepdims=True)
             )
             for level_probabilities in level_probability_list
         ]
+        """
 
         # combine probabilities horizontally
         level_probability_list = self._combine_and_reorder(level_probability_list)
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index 67dfec7a..74a16061 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -22,6 +22,8 @@
     init_strings as probability_combiner_init_strings,
 )
 
+from hiclass._hiclass_utils import _normalize_probabilities
+
 
 class LocalClassifierPerParentNode(BaseEstimator, HierarchicalClassifier):
     """
@@ -305,12 +307,15 @@ def _predict_proba_remaining_levels(self, X, y):
             level_probability_list.append(cur_level_probabilities)
 
         # normalize probabilities
+        level_probability_list = _normalize_probabilities(level_probability_list)
+        """
         level_probability_list = [
             np.nan_to_num(
                 level_probabilities / level_probabilities.sum(axis=1, keepdims=True)
             )
             for level_probabilities in level_probability_list
         ]
+        """
 
         return level_probability_list
 
diff --git a/hiclass/_calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
index aacacae6..a751c0bb 100644
--- a/hiclass/_calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -10,6 +10,7 @@
 from hiclass._calibration.PlattScaling import _PlattScaling
 from hiclass._calibration.BetaCalibrator import _BetaCalibrator
 from hiclass._calibration.calibration_utils import _one_vs_rest_split
+from hiclass._hiclass_utils import _normalize_probabilities
 
 
 class _Calibrator(BaseEstimator):
@@ -102,7 +103,8 @@ def predict_proba(self, X: np.ndarray):
                 for idx, split in enumerate(score_splits):
                     probabilities[:, idx] = self.calibrators[idx].predict_proba(split)
 
-                probabilities /= probabilities.sum(axis=1, keepdims=True)
+                # probabilities /= probabilities.sum(axis=1, keepdims=True)
+                probabilities = _normalize_probabilities(probabilities)
 
         else:
             probabilities = np.zeros((X.shape[0], 2))
diff --git a/hiclass/_calibration/VennAbersCalibrator.py b/hiclass/_calibration/VennAbersCalibrator.py
index 6a799ee8..b865908e 100644
--- a/hiclass/_calibration/VennAbersCalibrator.py
+++ b/hiclass/_calibration/VennAbersCalibrator.py
@@ -6,6 +6,7 @@
 from collections import defaultdict
 from sklearn.utils.validation import check_is_fitted
 from sklearn.base import BaseEstimator
+from hiclass._hiclass_utils import _normalize_probabilities
 
 
 class _InductiveVennAbersCalibrator(_BinaryCalibrator):
@@ -303,7 +304,8 @@ def predict_proba(self, scores: np.ndarray):
                     probabilities[:, idx] = self.ivaps[idx].predict_proba(scores)
 
             # normalize
-            probabilities /= probabilities.sum(axis=1, keepdims=True)
+            # probabilities /= probabilities.sum(axis=1, keepdims=True)
+            probabilities = _normalize_probabilities(probabilities)
             return probabilities
 
         else:
diff --git a/hiclass/_hiclass_utils.py b/hiclass/_hiclass_utils.py
new file mode 100644
index 00000000..4658973e
--- /dev/null
+++ b/hiclass/_hiclass_utils.py
@@ -0,0 +1,12 @@
+import numpy as np
+
+
+def _normalize_probabilities(proba):
+    if isinstance(proba, np.ndarray):
+        return np.nan_to_num(proba / proba.sum(axis=1, keepdims=True))
+    return [
+        np.nan_to_num(
+            level_probabilities / level_probabilities.sum(axis=1, keepdims=True)
+        )
+        for level_probabilities in proba
+    ]
diff --git a/hiclass/probability_combiner/ProbabilityCombiner.py b/hiclass/probability_combiner/ProbabilityCombiner.py
index a2f2ecfc..bd5016b0 100644
--- a/hiclass/probability_combiner/ProbabilityCombiner.py
+++ b/hiclass/probability_combiner/ProbabilityCombiner.py
@@ -6,6 +6,7 @@
 from collections import defaultdict
 from networkx.exception import NetworkXError
 from hiclass import HierarchicalClassifier
+from hiclass._hiclass_utils import _normalize_probabilities
 
 
 class ProbabilityCombiner(abc.ABC):
@@ -24,12 +25,7 @@ def combine(self, proba: List[np.ndarray]) -> List[np.ndarray]:
         ...
 
     def _normalize(self, proba: List[np.ndarray]):
-        return [
-            np.nan_to_num(
-                level_probabilities / level_probabilities.sum(axis=1, keepdims=True)
-            )
-            for level_probabilities in proba
-        ]
+        return _normalize_probabilities(proba)
 
     def _find_predecessors(self, level: int):
         predecessors = defaultdict(list)

From f6fe9486e984c8e9783987e8037669a7b883c1fe Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Fri, 19 Apr 2024 20:28:39 +0200
Subject: [PATCH 54/65] remove unused code

---
 hiclass/LocalClassifierPerNode.py           | 8 --------
 hiclass/LocalClassifierPerParentNode.py     | 8 --------
 hiclass/_calibration/Calibrator.py          | 1 -
 hiclass/_calibration/VennAbersCalibrator.py | 1 -
 4 files changed, 18 deletions(-)

diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index 9cb3b390..68c23400 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -323,14 +323,6 @@ def predict_proba(self, X):
 
         # normalize probabilities
         level_probability_list = _normalize_probabilities(level_probability_list)
-        """
-        level_probability_list = [
-            np.nan_to_num(
-                level_probabilities / level_probabilities.sum(axis=1, keepdims=True)
-            )
-            for level_probabilities in level_probability_list
-        ]
-        """
 
         # combine probabilities horizontally
         level_probability_list = self._combine_and_reorder(level_probability_list)
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index 74a16061..7c90fa4b 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -308,14 +308,6 @@ def _predict_proba_remaining_levels(self, X, y):
 
         # normalize probabilities
         level_probability_list = _normalize_probabilities(level_probability_list)
-        """
-        level_probability_list = [
-            np.nan_to_num(
-                level_probabilities / level_probabilities.sum(axis=1, keepdims=True)
-            )
-            for level_probabilities in level_probability_list
-        ]
-        """
 
         return level_probability_list
 
diff --git a/hiclass/_calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
index a751c0bb..59df2ea1 100644
--- a/hiclass/_calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -103,7 +103,6 @@ def predict_proba(self, X: np.ndarray):
                 for idx, split in enumerate(score_splits):
                     probabilities[:, idx] = self.calibrators[idx].predict_proba(split)
 
-                # probabilities /= probabilities.sum(axis=1, keepdims=True)
                 probabilities = _normalize_probabilities(probabilities)
 
         else:
diff --git a/hiclass/_calibration/VennAbersCalibrator.py b/hiclass/_calibration/VennAbersCalibrator.py
index b865908e..31e16a7c 100644
--- a/hiclass/_calibration/VennAbersCalibrator.py
+++ b/hiclass/_calibration/VennAbersCalibrator.py
@@ -304,7 +304,6 @@ def predict_proba(self, scores: np.ndarray):
                     probabilities[:, idx] = self.ivaps[idx].predict_proba(scores)
 
             # normalize
-            # probabilities /= probabilities.sum(axis=1, keepdims=True)
             probabilities = _normalize_probabilities(probabilities)
             return probabilities
 

From d1a73accb0dc7c32fa05760d64758cadc3f3afcf Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sat, 4 May 2024 17:29:48 +0200
Subject: [PATCH 55/65] fix error when calculating log loss

---
 hiclass/metrics.py    | 2 +-
 tests/test_metrics.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/hiclass/metrics.py b/hiclass/metrics.py
index d05940e1..b5c36d35 100644
--- a/hiclass/metrics.py
+++ b/hiclass/metrics.py
@@ -393,7 +393,7 @@ def log_loss(
         for level in range(make_leveled(y_true).shape[1]):
             scores.append(_log_loss(classifier, y_true, y_prob[level], level))
         return _aggregate_scores(scores, agg)
-    return _multiclass_brier_score(classifier, y_true, y_prob, level)
+    return _log_loss(classifier, y_true, y_prob, level)
 
 
 def expected_calibration_error(
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index 75b5dba9..5125df40 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -478,8 +478,8 @@ def test_log_loss_single_level(uncertainty_data_multi_level):
 
     log_loss_1 = log_loss(classifier, y_true, prob, level=1)
     log_loss_2 = log_loss(classifier, y_true, [prob], level=1)
-    assert math.isclose(log_loss_1, 0.48793, abs_tol=1e-4)
-    assert math.isclose(log_loss_2, 0.48793, abs_tol=1e-4)
+    assert math.isclose(log_loss_1, 0.81349, abs_tol=1e-4)
+    assert math.isclose(log_loss_2, 0.81348, abs_tol=1e-4)
 
 
 def test_local_expected_calibration_error(uncertainty_data):

From 5f4fca90ef354b0caa6bc99d736955c48d20863c Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sun, 5 May 2024 14:42:56 +0200
Subject: [PATCH 56/65] fix bug when calculating metrics for a single level

---
 hiclass/metrics.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/hiclass/metrics.py b/hiclass/metrics.py
index b5c36d35..499acd27 100644
--- a/hiclass/metrics.py
+++ b/hiclass/metrics.py
@@ -348,6 +348,8 @@ def multiclass_brier_score(
     """
     y_prob = _validate_args(agg, y_prob, level)
     if isinstance(y_prob, list):
+        if level:
+            return _multiclass_brier_score(classifier, y_true, y_prob[level], level)
         scores = []
         for level in range(make_leveled(y_true).shape[1]):
             scores.append(
@@ -389,6 +391,8 @@ def log_loss(
     """
     y_prob = _validate_args(agg, y_prob, level)
     if isinstance(y_prob, list):
+        if level:
+            return _log_loss(classifier, y_true, y_prob[level], level)
         scores = []
         for level in range(make_leveled(y_true).shape[1]):
             scores.append(_log_loss(classifier, y_true, y_prob[level], level))
@@ -434,6 +438,10 @@ def expected_calibration_error(
     """
     y_prob = _validate_args(agg, y_prob, level)
     if isinstance(y_prob, list):
+        if level:
+            return _expected_calibration_error(
+                classifier, y_true, y_prob[level], y_pred, level, n_bins
+            )
         scores = []
         for level in range(make_leveled(y_true).shape[1]):
             scores.append(
@@ -485,6 +493,10 @@ def static_calibration_error(
     """
     y_prob = _validate_args(agg, y_prob, level)
     if isinstance(y_prob, list):
+        if level:
+            return _static_calibration_error(
+                classifier, y_true, y_prob[level], y_pred, level, n_bins=n_bins
+            )
         scores = []
         for level in range(make_leveled(y_true).shape[1]):
             scores.append(
@@ -536,6 +548,10 @@ def adaptive_calibration_error(
     """
     y_prob = _validate_args(agg, y_prob, level)
     if isinstance(y_prob, list):
+        if level:
+            return _adaptive_calibration_error(
+                classifier, y_true, y_prob[level], y_pred, level, n_ranges=n_ranges
+            )
         scores = []
         for level in range(make_leveled(y_true).shape[1]):
             scores.append(

From db15547826b2237e1eaaae74c91c5a759eefaff3 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sun, 8 Sep 2024 15:35:23 +0200
Subject: [PATCH 57/65] remove redundant import

---
 hiclass/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hiclass/__init__.py b/hiclass/__init__.py
index 6d7cb941..57778bac 100644
--- a/hiclass/__init__.py
+++ b/hiclass/__init__.py
@@ -5,7 +5,6 @@
 from .LocalClassifierPerLevel import LocalClassifierPerLevel
 from .LocalClassifierPerNode import LocalClassifierPerNode
 from .LocalClassifierPerParentNode import LocalClassifierPerParentNode
-from .LocalClassifierPerLevel import LocalClassifierPerLevel
 from .Pipeline import Pipeline
 from .FlatClassifier import FlatClassifier
 from .MultiLabelLocalClassifierPerNode import MultiLabelLocalClassifierPerNode

From 9e0417fc90ba919fcb2e66502338b292bee9faf2 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sun, 8 Sep 2024 15:55:29 +0200
Subject: [PATCH 58/65] fix spelling error

---
 hiclass/_calibration/Calibrator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hiclass/_calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
index 59df2ea1..e39bdab2 100644
--- a/hiclass/_calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -39,7 +39,7 @@ def fit(self, X: np.ndarray, y: np.ndarray):
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The calibration input samples. Internally, its dtype will be converted
             to ``dtype=np.float32``. If a sparse matrix is provided, it will be
-            converted into a sparse ``csc_matrix``.
+            converted into a sparse ``csr_matrix``.
         y : array-like of shape (n_samples, n_levels)
             The target values, i.e., hierarchical class labels for classification.
 

From 38c011a59164465dc84a05e0347a0607e19f6d1e Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sun, 8 Sep 2024 16:51:28 +0200
Subject: [PATCH 59/65] update Pipfile to contain docs requirements

---
 Pipfile      |   14 +-
 Pipfile.lock | 1754 ++++++++++++++++++++++++++++++++------------------
 2 files changed, 1150 insertions(+), 618 deletions(-)

diff --git a/Pipfile b/Pipfile
index 0c8fd579..70f6ab34 100644
--- a/Pipfile
+++ b/Pipfile
@@ -6,7 +6,7 @@ name = "pypi"
 [packages]
 networkx = "*"
 numpy = "*"
-scikit-learn = "*"
+scikit-learn = "1.4.2"
 scipy = "1.11.4"
 
 [dev-packages]
@@ -17,13 +17,19 @@ pydocstyle = "6.1.1"
 pytest-pydocstyle = "2.3.0"
 pytest-cov = "3.0.0"
 twine = "*"
-sphinx = "4.1.1"
-sphinx-rtd-theme = "0.5.2"
+sphinx = "5.0.0"
+sphinx-rtd-theme = "1.0.0"
+readthedocs-sphinx-search = "0.1.2"
+sphinx_code_tabs = "0.5.3"
+sphinx-gallery = "0.10.1"
+matplotlib = "3.9.2"
+pandas = "1.4.2"
+bert-sklearn = {git = "https://github.com/charles9n/bert-sklearn.git@master", editable = true}
 black = {version = "24.3.0", extras = ["colorama"]}
 pre-commit = "2.20.0"
 pyfakefs = "*"
 shap = "0.44.1"
-xarray = "*"
+xarray = "2023.1.0"
 
 [extras]
 ray = "*"
diff --git a/Pipfile.lock b/Pipfile.lock
index 29c3959b..2cbf59e1 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "3c7fbdc64a7f72c8b0ab1fb9b85784f94a53e91c6dffdf2a3eb4814fa4352aac"
+            "sha256": "e04b6696c83be3a0dd3d4593c19f027b9a703a7e9e51bdb336d8eea0fd2458e6"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -16,11 +16,11 @@
     "default": {
         "joblib": {
             "hashes": [
-                "sha256:1eb0dc091919cd384490de890cb5dfd538410a6d4b3b54eef09fb8c50b409b1c",
-                "sha256:42942470d4062537be4d54c83511186da1fc14ba354961a2114da91efa9a4ed7"
+                "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6",
+                "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.4.0"
+            "version": "==1.4.2"
         },
         "networkx": {
             "hashes": [
@@ -136,11 +136,11 @@
         },
         "threadpoolctl": {
             "hashes": [
-                "sha256:8f4c689a65b23e5ed825c8436a92b818aac005e0f3715f6a1664d7c7ee29d262",
-                "sha256:f11b491a03661d6dd7ef692dd422ab34185d982466c49c8f98c8f716b5c93196"
+                "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107",
+                "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.4.0"
+            "version": "==3.5.0"
         }
     },
     "develop": {
@@ -161,27 +161,32 @@
         },
         "attrs": {
             "hashes": [
-                "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30",
-                "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"
+                "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346",
+                "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==23.2.0"
+            "version": "==24.2.0"
         },
         "babel": {
             "hashes": [
-                "sha256:6919867db036398ba21eb5c7a0f6b28ab8cbc3ae7a73a44ebe34ae74a4e7d363",
-                "sha256:efb1a25b7118e67ce3a259bed20545c29cb68be8ad2c784c83689981b7a57287"
+                "sha256:368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b",
+                "sha256:d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.14.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.16.0"
         },
         "backports.tarfile": {
             "hashes": [
-                "sha256:2688f159c21afd56a07b75f01306f9f52c79aebcc5f4a117fb8fbb4445352c75",
-                "sha256:bcd36290d9684beb524d3fe74f4a2db056824c47746583f090b8e55daf0776e4"
+                "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34",
+                "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"
             ],
             "markers": "python_version < '3.12'",
-            "version": "==1.0.0"
+            "version": "==1.2.0"
+        },
+        "bert-sklearn": {
+            "editable": true,
+            "git": "https://github.com/charles9n/bert-sklearn.git@master",
+            "ref": "9cb510ae16209c1cb26b078e0e5037e1344600af"
         },
         "black": {
             "extras": [
@@ -214,13 +219,29 @@
             "markers": "python_version >= '3.8'",
             "version": "==24.3.0"
         },
+        "boto3": {
+            "hashes": [
+                "sha256:7bc78d7140c353b10a637927fe4bc4c4d95a464d1b8f515d5844def2ee52cbd5",
+                "sha256:c3e138e9041d59cd34cdc28a587dfdc899dba02ea26ebc3e10fb4bc88e5cf31b"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.35.14"
+        },
+        "botocore": {
+            "hashes": [
+                "sha256:24823135232f88266b66ae8e1d0f3d40872c14cd976781f7fe52b8f0d79035a0",
+                "sha256:8515a2fc7ca5bcf0b10016ba05ccf2d642b7cb77d8773026ff2fa5aa3bf38d2e"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.35.14"
+        },
         "certifi": {
             "hashes": [
-                "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f",
-                "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"
+                "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8",
+                "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2024.2.2"
+            "version": "==2024.8.30"
         },
         "cfgv": {
             "hashes": [
@@ -350,66 +371,165 @@
             "markers": "sys_platform == 'win32'",
             "version": "==0.4.6"
         },
+        "contourpy": {
+            "hashes": [
+                "sha256:00ccd0dbaad6d804ab259820fa7cb0b8036bda0686ef844d24125d8287178ce0",
+                "sha256:0be4d8425bfa755e0fd76ee1e019636ccc7c29f77a7c86b4328a9eb6a26d0639",
+                "sha256:0dce35502151b6bd35027ac39ba6e5a44be13a68f55735c3612c568cac3805fd",
+                "sha256:0fa4c02abe6c446ba70d96ece336e621efa4aecae43eaa9b030ae5fb92b309ad",
+                "sha256:14e262f67bd7e6eb6880bc564dcda30b15e351a594657e55b7eec94b6ef72843",
+                "sha256:167d6c890815e1dac9536dca00828b445d5d0df4d6a8c6adb4a7ec3166812fa8",
+                "sha256:1ec4dc6bf570f5b22ed0d7efba0dfa9c5b9e0431aeea7581aa217542d9e809a4",
+                "sha256:303c252947ab4b14c08afeb52375b26781ccd6a5ccd81abcdfc1fafd14cf93c1",
+                "sha256:31cd3a85dbdf1fc002280c65caa7e2b5f65e4a973fcdf70dd2fdcb9868069294",
+                "sha256:32b238b3b3b649e09ce9aaf51f0c261d38644bdfa35cbaf7b263457850957a84",
+                "sha256:33c92cdae89ec5135d036e7218e69b0bb2851206077251f04a6c4e0e21f03927",
+                "sha256:345af746d7766821d05d72cb8f3845dfd08dd137101a2cb9b24de277d716def8",
+                "sha256:3634b5385c6716c258d0419c46d05c8aa7dc8cb70326c9a4fb66b69ad2b52e09",
+                "sha256:364174c2a76057feef647c802652f00953b575723062560498dc7930fc9b1cb7",
+                "sha256:36e0cff201bcb17a0a8ecc7f454fe078437fa6bda730e695a92f2d9932bd507f",
+                "sha256:36f965570cff02b874773c49bfe85562b47030805d7d8360748f3eca570f4cab",
+                "sha256:3bb3808858a9dc68f6f03d319acd5f1b8a337e6cdda197f02f4b8ff67ad2057b",
+                "sha256:3e1c7fa44aaae40a2247e2e8e0627f4bea3dd257014764aa644f319a5f8600e3",
+                "sha256:3faeb2998e4fcb256542e8a926d08da08977f7f5e62cf733f3c211c2a5586223",
+                "sha256:420d39daa61aab1221567b42eecb01112908b2cab7f1b4106a52caaec8d36973",
+                "sha256:4553c421929ec95fb07b3aaca0fae668b2eb5a5203d1217ca7c34c063c53d087",
+                "sha256:4865cd1d419e0c7a7bf6de1777b185eebdc51470800a9f42b9e9decf17762081",
+                "sha256:4cfb5c62ce023dfc410d6059c936dcf96442ba40814aefbfa575425a3a7f19dc",
+                "sha256:4d63ee447261e963af02642ffcb864e5a2ee4cbfd78080657a9880b8b1868e18",
+                "sha256:570ef7cf892f0afbe5b2ee410c507ce12e15a5fa91017a0009f79f7d93a1268f",
+                "sha256:637f674226be46f6ba372fd29d9523dd977a291f66ab2a74fbeb5530bb3f445d",
+                "sha256:68a32389b06b82c2fdd68276148d7b9275b5f5cf13e5417e4252f6d1a34f72a2",
+                "sha256:69375194457ad0fad3a839b9e29aa0b0ed53bb54db1bfb6c3ae43d111c31ce41",
+                "sha256:6cb6cc968059db9c62cb35fbf70248f40994dfcd7aa10444bbf8b3faeb7c2d67",
+                "sha256:710a26b3dc80c0e4febf04555de66f5fd17e9cf7170a7b08000601a10570bda6",
+                "sha256:732896af21716b29ab3e988d4ce14bc5133733b85956316fb0c56355f398099b",
+                "sha256:75ee7cb1a14c617f34a51d11fa7524173e56551646828353c4af859c56b766e2",
+                "sha256:76a896b2f195b57db25d6b44e7e03f221d32fe318d03ede41f8b4d9ba1bff53c",
+                "sha256:76c905ef940a4474a6289c71d53122a4f77766eef23c03cd57016ce19d0f7b42",
+                "sha256:7a52040312b1a858b5e31ef28c2e865376a386c60c0e248370bbea2d3f3b760d",
+                "sha256:7ffa0db17717a8ffb127efd0c95a4362d996b892c2904db72428d5b52e1938a4",
+                "sha256:81cb5ed4952aae6014bc9d0421dec7c5835c9c8c31cdf51910b708f548cf58e5",
+                "sha256:834e0cfe17ba12f79963861e0f908556b2cedd52e1f75e6578801febcc6a9f49",
+                "sha256:87ddffef1dbe5e669b5c2440b643d3fdd8622a348fe1983fad7a0f0ccb1cd67b",
+                "sha256:880ea32e5c774634f9fcd46504bf9f080a41ad855f4fef54f5380f5133d343c7",
+                "sha256:8ca947601224119117f7c19c9cdf6b3ab54c5726ef1d906aa4a69dfb6dd58102",
+                "sha256:90f73a5116ad1ba7174341ef3ea5c3150ddf20b024b98fb0c3b29034752c8aeb",
+                "sha256:92f8557cbb07415a4d6fa191f20fd9d2d9eb9c0b61d1b2f52a8926e43c6e9af7",
+                "sha256:94e848a6b83da10898cbf1311a815f770acc9b6a3f2d646f330d57eb4e87592e",
+                "sha256:9c0da700bf58f6e0b65312d0a5e695179a71d0163957fa381bb3c1f72972537c",
+                "sha256:a11077e395f67ffc2c44ec2418cfebed032cd6da3022a94fc227b6faf8e2acb8",
+                "sha256:aea348f053c645100612b333adc5983d87be69acdc6d77d3169c090d3b01dc35",
+                "sha256:b11b39aea6be6764f84360fce6c82211a9db32a7c7de8fa6dd5397cf1d079c3b",
+                "sha256:c6c7c2408b7048082932cf4e641fa3b8ca848259212f51c8c59c45aa7ac18f14",
+                "sha256:c6ec93afeb848a0845a18989da3beca3eec2c0f852322efe21af1931147d12cb",
+                "sha256:cacd81e2d4b6f89c9f8a5b69b86490152ff39afc58a95af002a398273e5ce589",
+                "sha256:d402880b84df3bec6eab53cd0cf802cae6a2ef9537e70cf75e91618a3801c20c",
+                "sha256:d51fca85f9f7ad0b65b4b9fe800406d0d77017d7270d31ec3fb1cc07358fdea0",
+                "sha256:d73f659398a0904e125280836ae6f88ba9b178b2fed6884f3b1f95b989d2c8da",
+                "sha256:d78ab28a03c854a873787a0a42254a0ccb3cb133c672f645c9f9c8f3ae9d0800",
+                "sha256:da84c537cb8b97d153e9fb208c221c45605f73147bd4cadd23bdae915042aad6",
+                "sha256:dbc4c3217eee163fa3984fd1567632b48d6dfd29216da3ded3d7b844a8014a66",
+                "sha256:e12968fdfd5bb45ffdf6192a590bd8ddd3ba9e58360b29683c6bb71a7b41edca",
+                "sha256:e1fd23e9d01591bab45546c089ae89d926917a66dceb3abcf01f6105d927e2cb",
+                "sha256:e8134301d7e204c88ed7ab50028ba06c683000040ede1d617298611f9dc6240c",
+                "sha256:eb8b141bb00fa977d9122636b16aa67d37fd40a3d8b52dd837e536d64b9a4d06",
+                "sha256:eca7e17a65f72a5133bdbec9ecf22401c62bcf4821361ef7811faee695799779",
+                "sha256:f317576606de89da6b7e0861cf6061f6146ead3528acabff9236458a6ba467f8",
+                "sha256:fd2a0fc506eccaaa7595b7e1418951f213cf8255be2600f1ea1b61e46a60c55f",
+                "sha256:fe41b41505a5a33aeaed2a613dccaeaa74e0e3ead6dd6fd3a118fb471644fd6c"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==1.3.0"
+        },
         "coverage": {
             "extras": [
                 "toml"
             ],
             "hashes": [
-                "sha256:00838a35b882694afda09f85e469c96367daa3f3f2b097d846a7216993d37f4c",
-                "sha256:0513b9508b93da4e1716744ef6ebc507aff016ba115ffe8ecff744d1322a7b63",
-                "sha256:09c3255458533cb76ef55da8cc49ffab9e33f083739c8bd4f58e79fecfe288f7",
-                "sha256:09ef9199ed6653989ebbcaacc9b62b514bb63ea2f90256e71fea3ed74bd8ff6f",
-                "sha256:09fa497a8ab37784fbb20ab699c246053ac294d13fc7eb40ec007a5043ec91f8",
-                "sha256:0f9f50e7ef2a71e2fae92774c99170eb8304e3fdf9c8c3c7ae9bab3e7229c5cf",
-                "sha256:137eb07173141545e07403cca94ab625cc1cc6bc4c1e97b6e3846270e7e1fea0",
-                "sha256:1f384c3cc76aeedce208643697fb3e8437604b512255de6d18dae3f27655a384",
-                "sha256:201bef2eea65e0e9c56343115ba3814e896afe6d36ffd37bab783261db430f76",
-                "sha256:38dd60d7bf242c4ed5b38e094baf6401faa114fc09e9e6632374388a404f98e7",
-                "sha256:3b799445b9f7ee8bf299cfaed6f5b226c0037b74886a4e11515e569b36fe310d",
-                "sha256:3ea79bb50e805cd6ac058dfa3b5c8f6c040cb87fe83de10845857f5535d1db70",
-                "sha256:40209e141059b9370a2657c9b15607815359ab3ef9918f0196b6fccce8d3230f",
-                "sha256:41c9c5f3de16b903b610d09650e5e27adbfa7f500302718c9ffd1c12cf9d6818",
-                "sha256:54eb8d1bf7cacfbf2a3186019bcf01d11c666bd495ed18717162f7eb1e9dd00b",
-                "sha256:598825b51b81c808cb6f078dcb972f96af96b078faa47af7dfcdf282835baa8d",
-                "sha256:5fc1de20b2d4a061b3df27ab9b7c7111e9a710f10dc2b84d33a4ab25065994ec",
-                "sha256:623512f8ba53c422fcfb2ce68362c97945095b864cda94a92edbaf5994201083",
-                "sha256:690db6517f09336559dc0b5f55342df62370a48f5469fabf502db2c6d1cffcd2",
-                "sha256:69eb372f7e2ece89f14751fbcbe470295d73ed41ecd37ca36ed2eb47512a6ab9",
-                "sha256:73bfb9c09951125d06ee473bed216e2c3742f530fc5acc1383883125de76d9cd",
-                "sha256:742a76a12aa45b44d236815d282b03cfb1de3b4323f3e4ec933acfae08e54ade",
-                "sha256:7c95949560050d04d46b919301826525597f07b33beba6187d04fa64d47ac82e",
-                "sha256:8130a2aa2acb8788e0b56938786c33c7c98562697bf9f4c7d6e8e5e3a0501e4a",
-                "sha256:8a2b2b78c78293782fd3767d53e6474582f62443d0504b1554370bde86cc8227",
-                "sha256:8ce1415194b4a6bd0cdcc3a1dfbf58b63f910dcb7330fe15bdff542c56949f87",
-                "sha256:9ca28a302acb19b6af89e90f33ee3e1906961f94b54ea37de6737b7ca9d8827c",
-                "sha256:a4cdc86d54b5da0df6d3d3a2f0b710949286094c3a6700c21e9015932b81447e",
-                "sha256:aa5b1c1bfc28384f1f53b69a023d789f72b2e0ab1b3787aae16992a7ca21056c",
-                "sha256:aadacf9a2f407a4688d700e4ebab33a7e2e408f2ca04dbf4aef17585389eff3e",
-                "sha256:ae71e7ddb7a413dd60052e90528f2f65270aad4b509563af6d03d53e979feafd",
-                "sha256:b14706df8b2de49869ae03a5ccbc211f4041750cd4a66f698df89d44f4bd30ec",
-                "sha256:b1a93009cb80730c9bca5d6d4665494b725b6e8e157c1cb7f2db5b4b122ea562",
-                "sha256:b2991665420a803495e0b90a79233c1433d6ed77ef282e8e152a324bbbc5e0c8",
-                "sha256:b2c5edc4ac10a7ef6605a966c58929ec6c1bd0917fb8c15cb3363f65aa40e677",
-                "sha256:b4d33f418f46362995f1e9d4f3a35a1b6322cb959c31d88ae56b0298e1c22357",
-                "sha256:b91cbc4b195444e7e258ba27ac33769c41b94967919f10037e6355e998af255c",
-                "sha256:c74880fc64d4958159fbd537a091d2a585448a8f8508bf248d72112723974cbd",
-                "sha256:c901df83d097649e257e803be22592aedfd5182f07b3cc87d640bbb9afd50f49",
-                "sha256:cac99918c7bba15302a2d81f0312c08054a3359eaa1929c7e4b26ebe41e9b286",
-                "sha256:cc4f1358cb0c78edef3ed237ef2c86056206bb8d9140e73b6b89fbcfcbdd40e1",
-                "sha256:ccd341521be3d1b3daeb41960ae94a5e87abe2f46f17224ba5d6f2b8398016cf",
-                "sha256:ce4b94265ca988c3f8e479e741693d143026632672e3ff924f25fab50518dd51",
-                "sha256:cf271892d13e43bc2b51e6908ec9a6a5094a4df1d8af0bfc360088ee6c684409",
-                "sha256:d5ae728ff3b5401cc320d792866987e7e7e880e6ebd24433b70a33b643bb0384",
-                "sha256:d71eec7d83298f1af3326ce0ff1d0ea83c7cb98f72b577097f9083b20bdaf05e",
-                "sha256:d898fe162d26929b5960e4e138651f7427048e72c853607f2b200909794ed978",
-                "sha256:d89d7b2974cae412400e88f35d86af72208e1ede1a541954af5d944a8ba46c57",
-                "sha256:dfa8fe35a0bb90382837b238fff375de15f0dcdb9ae68ff85f7a63649c98527e",
-                "sha256:e0be5efd5127542ef31f165de269f77560d6cdef525fffa446de6f7e9186cfb2",
-                "sha256:fdfafb32984684eb03c2d83e1e51f64f0906b11e64482df3c5db936ce3839d48",
-                "sha256:ff7687ca3d7028d8a5f0ebae95a6e4827c5616b31a4ee1192bdfde697db110d4"
+                "sha256:06a737c882bd26d0d6ee7269b20b12f14a8704807a01056c80bb881a4b2ce6ca",
+                "sha256:07e2ca0ad381b91350c0ed49d52699b625aab2b44b65e1b4e02fa9df0e92ad2d",
+                "sha256:0c0420b573964c760df9e9e86d1a9a622d0d27f417e1a949a8a66dd7bcee7bc6",
+                "sha256:0dbde0f4aa9a16fa4d754356a8f2e36296ff4d83994b2c9d8398aa32f222f989",
+                "sha256:1125ca0e5fd475cbbba3bb67ae20bd2c23a98fac4e32412883f9bcbaa81c314c",
+                "sha256:13b0a73a0896988f053e4fbb7de6d93388e6dd292b0d87ee51d106f2c11b465b",
+                "sha256:166811d20dfea725e2e4baa71fffd6c968a958577848d2131f39b60043400223",
+                "sha256:170d444ab405852903b7d04ea9ae9b98f98ab6d7e63e1115e82620807519797f",
+                "sha256:1f4aa8219db826ce6be7099d559f8ec311549bfc4046f7f9fe9b5cea5c581c56",
+                "sha256:225667980479a17db1048cb2bf8bfb39b8e5be8f164b8f6628b64f78a72cf9d3",
+                "sha256:260933720fdcd75340e7dbe9060655aff3af1f0c5d20f46b57f262ab6c86a5e8",
+                "sha256:2bdb062ea438f22d99cba0d7829c2ef0af1d768d1e4a4f528087224c90b132cb",
+                "sha256:2c09f4ce52cb99dd7505cd0fc8e0e37c77b87f46bc9c1eb03fe3bc9991085388",
+                "sha256:3115a95daa9bdba70aea750db7b96b37259a81a709223c8448fa97727d546fe0",
+                "sha256:3e0cadcf6733c09154b461f1ca72d5416635e5e4ec4e536192180d34ec160f8a",
+                "sha256:3f1156e3e8f2872197af3840d8ad307a9dd18e615dc64d9ee41696f287c57ad8",
+                "sha256:4421712dbfc5562150f7554f13dde997a2e932a6b5f352edcce948a815efee6f",
+                "sha256:44df346d5215a8c0e360307d46ffaabe0f5d3502c8a1cefd700b34baf31d411a",
+                "sha256:502753043567491d3ff6d08629270127e0c31d4184c4c8d98f92c26f65019962",
+                "sha256:547f45fa1a93154bd82050a7f3cddbc1a7a4dd2a9bf5cb7d06f4ae29fe94eaf8",
+                "sha256:5621a9175cf9d0b0c84c2ef2b12e9f5f5071357c4d2ea6ca1cf01814f45d2391",
+                "sha256:609b06f178fe8e9f89ef676532760ec0b4deea15e9969bf754b37f7c40326dbc",
+                "sha256:645786266c8f18a931b65bfcefdbf6952dd0dea98feee39bd188607a9d307ed2",
+                "sha256:6878ef48d4227aace338d88c48738a4258213cd7b74fd9a3d4d7582bb1d8a155",
+                "sha256:6a89ecca80709d4076b95f89f308544ec8f7b4727e8a547913a35f16717856cb",
+                "sha256:6db04803b6c7291985a761004e9060b2bca08da6d04f26a7f2294b8623a0c1a0",
+                "sha256:6e2cd258d7d927d09493c8df1ce9174ad01b381d4729a9d8d4e38670ca24774c",
+                "sha256:6e81d7a3e58882450ec4186ca59a3f20a5d4440f25b1cff6f0902ad890e6748a",
+                "sha256:702855feff378050ae4f741045e19a32d57d19f3e0676d589df0575008ea5004",
+                "sha256:78b260de9790fd81e69401c2dc8b17da47c8038176a79092a89cb2b7d945d060",
+                "sha256:7bb65125fcbef8d989fa1dd0e8a060999497629ca5b0efbca209588a73356232",
+                "sha256:7dea0889685db8550f839fa202744652e87c60015029ce3f60e006f8c4462c93",
+                "sha256:8284cf8c0dd272a247bc154eb6c95548722dce90d098c17a883ed36e67cdb129",
+                "sha256:877abb17e6339d96bf08e7a622d05095e72b71f8afd8a9fefc82cf30ed944163",
+                "sha256:8929543a7192c13d177b770008bc4e8119f2e1f881d563fc6b6305d2d0ebe9de",
+                "sha256:8ae539519c4c040c5ffd0632784e21b2f03fc1340752af711f33e5be83a9d6c6",
+                "sha256:8f59d57baca39b32db42b83b2a7ba6f47ad9c394ec2076b084c3f029b7afca23",
+                "sha256:9054a0754de38d9dbd01a46621636689124d666bad1936d76c0341f7d71bf569",
+                "sha256:953510dfb7b12ab69d20135a0662397f077c59b1e6379a768e97c59d852ee51d",
+                "sha256:95cae0efeb032af8458fc27d191f85d1717b1d4e49f7cb226cf526ff28179778",
+                "sha256:9bc572be474cafb617672c43fe989d6e48d3c83af02ce8de73fff1c6bb3c198d",
+                "sha256:9c56863d44bd1c4fe2abb8a4d6f5371d197f1ac0ebdee542f07f35895fc07f36",
+                "sha256:9e0b2df163b8ed01d515807af24f63de04bebcecbd6c3bfeff88385789fdf75a",
+                "sha256:a09ece4a69cf399510c8ab25e0950d9cf2b42f7b3cb0374f95d2e2ff594478a6",
+                "sha256:a1ac0ae2b8bd743b88ed0502544847c3053d7171a3cff9228af618a068ed9c34",
+                "sha256:a318d68e92e80af8b00fa99609796fdbcdfef3629c77c6283566c6f02c6d6704",
+                "sha256:a4acd025ecc06185ba2b801f2de85546e0b8ac787cf9d3b06e7e2a69f925b106",
+                "sha256:a6d3adcf24b624a7b778533480e32434a39ad8fa30c315208f6d3e5542aeb6e9",
+                "sha256:a78d169acd38300060b28d600344a803628c3fd585c912cacc9ea8790fe96862",
+                "sha256:a95324a9de9650a729239daea117df21f4b9868ce32e63f8b650ebe6cef5595b",
+                "sha256:abd5fd0db5f4dc9289408aaf34908072f805ff7792632250dcb36dc591d24255",
+                "sha256:b06079abebbc0e89e6163b8e8f0e16270124c154dc6e4a47b413dd538859af16",
+                "sha256:b43c03669dc4618ec25270b06ecd3ee4fa94c7f9b3c14bae6571ca00ef98b0d3",
+                "sha256:b48f312cca9621272ae49008c7f613337c53fadca647d6384cc129d2996d1133",
+                "sha256:b5d7b556859dd85f3a541db6a4e0167b86e7273e1cdc973e5b175166bb634fdb",
+                "sha256:b9f222de8cded79c49bf184bdbc06630d4c58eec9459b939b4a690c82ed05657",
+                "sha256:c3c02d12f837d9683e5ab2f3d9844dc57655b92c74e286c262e0fc54213c216d",
+                "sha256:c44fee9975f04b33331cb8eb272827111efc8930cfd582e0320613263ca849ca",
+                "sha256:cf4b19715bccd7ee27b6b120e7e9dd56037b9c0681dcc1adc9ba9db3d417fa36",
+                "sha256:d0c212c49b6c10e6951362f7c6df3329f04c2b1c28499563d4035d964ab8e08c",
+                "sha256:d3296782ca4eab572a1a4eca686d8bfb00226300dcefdf43faa25b5242ab8a3e",
+                "sha256:d85f5e9a5f8b73e2350097c3756ef7e785f55bd71205defa0bfdaf96c31616ff",
+                "sha256:da511e6ad4f7323ee5702e6633085fb76c2f893aaf8ce4c51a0ba4fc07580ea7",
+                "sha256:e05882b70b87a18d937ca6768ff33cc3f72847cbc4de4491c8e73880766718e5",
+                "sha256:e61c0abb4c85b095a784ef23fdd4aede7a2628478e7baba7c5e3deba61070a02",
+                "sha256:e6a08c0be454c3b3beb105c0596ebdc2371fab6bb90c0c0297f4e58fd7e1012c",
+                "sha256:e9a6e0eb86070e8ccaedfbd9d38fec54864f3125ab95419970575b42af7541df",
+                "sha256:ed37bd3c3b063412f7620464a9ac1314d33100329f39799255fb8d3027da50d3",
+                "sha256:f1adfc8ac319e1a348af294106bc6a8458a0f1633cc62a1446aebc30c5fa186a",
+                "sha256:f5796e664fe802da4f57a168c85359a8fbf3eab5e55cd4e4569fbacecc903959",
+                "sha256:fc5a77d0c516700ebad189b587de289a20a78324bc54baee03dd486f0855d234",
+                "sha256:fd21f6ae3f08b41004dfb433fa895d858f3f5979e7762d052b12aef444e29afc"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==7.6.1"
+        },
+        "cycler": {
+            "hashes": [
+                "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30",
+                "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==7.4.4"
+            "version": "==0.12.1"
         },
         "distlib": {
             "hashes": [
@@ -420,19 +540,19 @@
         },
         "docutils": {
             "hashes": [
-                "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af",
-                "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"
+                "sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125",
+                "sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==0.16"
+            "version": "==0.17.1"
         },
         "filelock": {
             "hashes": [
-                "sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f",
-                "sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4"
+                "sha256:81de9eb8453c769b63369f87f11131a7ab04e367f8d97ad39dc230daa07e3bec",
+                "sha256:f6ed4c963184f4c84dd5557ce8fece759a3724b37b80c6c4f20a2f63a4dc6609"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.13.4"
+            "version": "==3.16.0"
         },
         "flake8": {
             "hashes": [
@@ -443,21 +563,77 @@
             "markers": "python_version >= '3.6'",
             "version": "==4.0.1"
         },
+        "fonttools": {
+            "hashes": [
+                "sha256:02569e9a810f9d11f4ae82c391ebc6fb5730d95a0657d24d754ed7763fb2d122",
+                "sha256:0679a30b59d74b6242909945429dbddb08496935b82f91ea9bf6ad240ec23397",
+                "sha256:10f5e6c3510b79ea27bb1ebfcc67048cde9ec67afa87c7dd7efa5c700491ac7f",
+                "sha256:2af40ae9cdcb204fc1d8f26b190aa16534fcd4f0df756268df674a270eab575d",
+                "sha256:32f029c095ad66c425b0ee85553d0dc326d45d7059dbc227330fc29b43e8ba60",
+                "sha256:35250099b0cfb32d799fb5d6c651220a642fe2e3c7d2560490e6f1d3f9ae9169",
+                "sha256:3b3c8ebafbee8d9002bd8f1195d09ed2bd9ff134ddec37ee8f6a6375e6a4f0e8",
+                "sha256:4824c198f714ab5559c5be10fd1adf876712aa7989882a4ec887bf1ef3e00e31",
+                "sha256:5ff7e5e9bad94e3a70c5cd2fa27f20b9bb9385e10cddab567b85ce5d306ea923",
+                "sha256:651390c3b26b0c7d1f4407cad281ee7a5a85a31a110cbac5269de72a51551ba2",
+                "sha256:6e08f572625a1ee682115223eabebc4c6a2035a6917eac6f60350aba297ccadb",
+                "sha256:6ed170b5e17da0264b9f6fae86073be3db15fa1bd74061c8331022bca6d09bab",
+                "sha256:73379d3ffdeecb376640cd8ed03e9d2d0e568c9d1a4e9b16504a834ebadc2dfb",
+                "sha256:75a157d8d26c06e64ace9df037ee93a4938a4606a38cb7ffaf6635e60e253b7a",
+                "sha256:791b31ebbc05197d7aa096bbc7bd76d591f05905d2fd908bf103af4488e60670",
+                "sha256:7b6b35e52ddc8fb0db562133894e6ef5b4e54e1283dff606fda3eed938c36fc8",
+                "sha256:84ec3fb43befb54be490147b4a922b5314e16372a643004f182babee9f9c3407",
+                "sha256:8959a59de5af6d2bec27489e98ef25a397cfa1774b375d5787509c06659b3671",
+                "sha256:9dfdae43b7996af46ff9da520998a32b105c7f098aeea06b2226b30e74fbba88",
+                "sha256:9e6ceba2a01b448e36754983d376064730690401da1dd104ddb543519470a15f",
+                "sha256:9efd176f874cb6402e607e4cc9b4a9cd584d82fc34a4b0c811970b32ba62501f",
+                "sha256:a1c7c5aa18dd3b17995898b4a9b5929d69ef6ae2af5b96d585ff4005033d82f0",
+                "sha256:aae7bd54187e8bf7fd69f8ab87b2885253d3575163ad4d669a262fe97f0136cb",
+                "sha256:b21952c092ffd827504de7e66b62aba26fdb5f9d1e435c52477e6486e9d128b2",
+                "sha256:b96cd370a61f4d083c9c0053bf634279b094308d52fdc2dd9a22d8372fdd590d",
+                "sha256:becc5d7cb89c7b7afa8321b6bb3dbee0eec2b57855c90b3e9bf5fb816671fa7c",
+                "sha256:bee32ea8765e859670c4447b0817514ca79054463b6b79784b08a8df3a4d78e3",
+                "sha256:c6e7170d675d12eac12ad1a981d90f118c06cf680b42a2d74c6c931e54b50719",
+                "sha256:c818c058404eb2bba05e728d38049438afd649e3c409796723dfc17cd3f08749",
+                "sha256:c8696544c964500aa9439efb6761947393b70b17ef4e82d73277413f291260a4",
+                "sha256:c9cd19cf4fe0595ebdd1d4915882b9440c3a6d30b008f3cc7587c1da7b95be5f",
+                "sha256:d4d0096cb1ac7a77b3b41cd78c9b6bc4a400550e21dc7a92f2b5ab53ed74eb02",
+                "sha256:d92d3c2a1b39631a6131c2fa25b5406855f97969b068e7e08413325bc0afba58",
+                "sha256:da33440b1413bad53a8674393c5d29ce64d8c1a15ef8a77c642ffd900d07bfe1",
+                "sha256:e013aae589c1c12505da64a7d8d023e584987e51e62006e1bb30d72f26522c41",
+                "sha256:e128778a8e9bc11159ce5447f76766cefbd876f44bd79aff030287254e4752c4",
+                "sha256:e54f1bba2f655924c1138bbc7fa91abd61f45c68bd65ab5ed985942712864bbb",
+                "sha256:e5b708073ea3d684235648786f5f6153a48dc8762cdfe5563c57e80787c29fbb",
+                "sha256:e8bf06b94694251861ba7fdeea15c8ec0967f84c3d4143ae9daf42bbc7717fe3",
+                "sha256:f08df60fbd8d289152079a65da4e66a447efc1d5d5a4d3f299cdd39e3b2e4a7d",
+                "sha256:f1f8758a2ad110bd6432203a344269f445a2907dc24ef6bccfd0ac4e14e0d71d",
+                "sha256:f677ce218976496a587ab17140da141557beb91d2a5c1a14212c994093f2eae2"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==4.53.1"
+        },
+        "fsspec": {
+            "hashes": [
+                "sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8",
+                "sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2024.9.0"
+        },
         "identify": {
             "hashes": [
-                "sha256:10a7ca245cfcd756a554a7288159f72ff105ad233c7c4b9c6f0f4d108f5f6791",
-                "sha256:c4de0081837b211594f8e877a6b4fad7ca32bbfc1a9307fdd61c28bfe923f13e"
+                "sha256:cb171c685bdc31bcc4c1734698736a7d5b6c8bf2e0c15117f4d469c8640ae5cf",
+                "sha256:e79ae4406387a9d300332b5fd366d8994f1525e8414984e1a59e058b2eda2dd0"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.5.35"
+            "version": "==2.6.0"
         },
         "idna": {
             "hashes": [
-                "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc",
-                "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"
+                "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac",
+                "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==3.7"
+            "markers": "python_version >= '3.6'",
+            "version": "==3.8"
         },
         "imagesize": {
             "hashes": [
@@ -469,11 +645,11 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570",
-                "sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2"
+                "sha256:66f342cc6ac9818fc6ff340576acd24d65ba0b3efabb2b4ac08b598965a4a2f1",
+                "sha256:9a547d3bc3608b025f93d403fdd1aae741c24fbb8314df4b155675742ce303c5"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==7.1.0"
+            "version": "==8.4.0"
         },
         "iniconfig": {
             "hashes": [
@@ -493,70 +669,198 @@
         },
         "jaraco.context": {
             "hashes": [
-                "sha256:3e16388f7da43d384a1a7cd3452e72e14732ac9fe459678773a3608a812bf266",
-                "sha256:c2f67165ce1f9be20f32f650f25d8edfc1646a8aeee48ae06fb35f90763576d2"
+                "sha256:9bae4ea555cf0b14938dc0aee7c9f32ed303aa20a3b73e7dc80111628792d1b3",
+                "sha256:f797fc481b490edb305122c9181830a3a5b76d84ef6d1aef2fb9b47ab956f9e4"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.3.0"
+            "version": "==6.0.1"
         },
         "jaraco.functools": {
             "hashes": [
-                "sha256:c279cb24c93d694ef7270f970d499cab4d3813f4e08273f95398651a634f0925",
-                "sha256:daf276ddf234bea897ef14f43c4e1bf9eefeac7b7a82a4dd69228ac20acff68d"
+                "sha256:3460c74cd0d32bf82b9576bbb3527c4364d5b27a21f5158a62aed6c4b42e23f5",
+                "sha256:c9d16a3ed4ccb5a889ad8e0b7a343401ee5b2a71cee6ed192d3f68bc351e94e3"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.0.0"
+            "version": "==4.0.2"
         },
         "jinja2": {
             "hashes": [
-                "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa",
-                "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"
+                "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369",
+                "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.1.3"
+            "version": "==3.1.4"
+        },
+        "jmespath": {
+            "hashes": [
+                "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980",
+                "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.0.1"
         },
         "joblib": {
             "hashes": [
-                "sha256:1eb0dc091919cd384490de890cb5dfd538410a6d4b3b54eef09fb8c50b409b1c",
-                "sha256:42942470d4062537be4d54c83511186da1fc14ba354961a2114da91efa9a4ed7"
+                "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6",
+                "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.4.0"
+            "version": "==1.4.2"
         },
         "keyring": {
             "hashes": [
-                "sha256:26fc12e6a329d61d24aa47b22a7c5c3f35753df7d8f2860973cf94f4e1fb3427",
-                "sha256:7230ea690525133f6ad536a9b5def74a4bd52642abe594761028fc044d7c7893"
+                "sha256:8d85a1ea5d6db8515b59e1c5d1d1678b03cf7fc8b8dcfb1651e8c4a524eb42ef",
+                "sha256:8d963da00ccdf06e356acd9bf3b743208878751032d8599c6cc89eb51310ffae"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==25.3.0"
+        },
+        "kiwisolver": {
+            "hashes": [
+                "sha256:073a36c8273647592ea332e816e75ef8da5c303236ec0167196793eb1e34657a",
+                "sha256:08471d4d86cbaec61f86b217dd938a83d85e03785f51121e791a6e6689a3be95",
+                "sha256:0c18ec74c0472de033e1bebb2911c3c310eef5649133dd0bedf2a169a1b269e5",
+                "sha256:0c6c43471bc764fad4bc99c5c2d6d16a676b1abf844ca7c8702bdae92df01ee0",
+                "sha256:10849fb2c1ecbfae45a693c070e0320a91b35dd4bcf58172c023b994283a124d",
+                "sha256:18077b53dc3bb490e330669a99920c5e6a496889ae8c63b58fbc57c3d7f33a18",
+                "sha256:18e0cca3e008e17fe9b164b55735a325140a5a35faad8de92dd80265cd5eb80b",
+                "sha256:22f499f6157236c19f4bbbd472fa55b063db77a16cd74d49afe28992dff8c258",
+                "sha256:2a8781ac3edc42ea4b90bc23e7d37b665d89423818e26eb6df90698aa2287c95",
+                "sha256:2e6039dcbe79a8e0f044f1c39db1986a1b8071051efba3ee4d74f5b365f5226e",
+                "sha256:34ea1de54beef1c104422d210c47c7d2a4999bdecf42c7b5718fbe59a4cac383",
+                "sha256:3ab58c12a2cd0fc769089e6d38466c46d7f76aced0a1f54c77652446733d2d02",
+                "sha256:3abc5b19d24af4b77d1598a585b8a719beb8569a71568b66f4ebe1fb0449460b",
+                "sha256:3bf1ed55088f214ba6427484c59553123fdd9b218a42bbc8c6496d6754b1e523",
+                "sha256:3ce6b2b0231bda412463e152fc18335ba32faf4e8c23a754ad50ffa70e4091ee",
+                "sha256:3da53da805b71e41053dc670f9a820d1157aae77b6b944e08024d17bcd51ef88",
+                "sha256:3f9362ecfca44c863569d3d3c033dbe8ba452ff8eed6f6b5806382741a1334bd",
+                "sha256:409afdfe1e2e90e6ee7fc896f3df9a7fec8e793e58bfa0d052c8a82f99c37abb",
+                "sha256:40fa14dbd66b8b8f470d5fc79c089a66185619d31645f9b0773b88b19f7223c4",
+                "sha256:4322872d5772cae7369f8351da1edf255a604ea7087fe295411397d0cfd9655e",
+                "sha256:44756f9fd339de0fb6ee4f8c1696cfd19b2422e0d70b4cefc1cc7f1f64045a8c",
+                "sha256:46707a10836894b559e04b0fd143e343945c97fd170d69a2d26d640b4e297935",
+                "sha256:48b571ecd8bae15702e4f22d3ff6a0f13e54d3d00cd25216d5e7f658242065ee",
+                "sha256:48be928f59a1f5c8207154f935334d374e79f2b5d212826307d072595ad76a2e",
+                "sha256:4bfa75a048c056a411f9705856abfc872558e33c055d80af6a380e3658766038",
+                "sha256:4c00336b9dd5ad96d0a558fd18a8b6f711b7449acce4c157e7343ba92dd0cf3d",
+                "sha256:4c26ed10c4f6fa6ddb329a5120ba3b6db349ca192ae211e882970bfc9d91420b",
+                "sha256:4d05d81ecb47d11e7f8932bd8b61b720bf0b41199358f3f5e36d38e28f0532c5",
+                "sha256:4e77f2126c3e0b0d055f44513ed349038ac180371ed9b52fe96a32aa071a5107",
+                "sha256:5337ec7809bcd0f424c6b705ecf97941c46279cf5ed92311782c7c9c2026f07f",
+                "sha256:5360cc32706dab3931f738d3079652d20982511f7c0ac5711483e6eab08efff2",
+                "sha256:58370b1ffbd35407444d57057b57da5d6549d2d854fa30249771775c63b5fe17",
+                "sha256:58cb20602b18f86f83a5c87d3ee1c766a79c0d452f8def86d925e6c60fbf7bfb",
+                "sha256:599b5c873c63a1f6ed7eead644a8a380cfbdf5db91dcb6f85707aaab213b1674",
+                "sha256:5b7dfa3b546da08a9f622bb6becdb14b3e24aaa30adba66749d38f3cc7ea9706",
+                "sha256:5b9c3f4ee0b9a439d2415012bd1b1cc2df59e4d6a9939f4d669241d30b414327",
+                "sha256:5d34eb8494bea691a1a450141ebb5385e4b69d38bb8403b5146ad279f4b30fa3",
+                "sha256:5d5abf8f8ec1f4e22882273c423e16cae834c36856cac348cfbfa68e01c40f3a",
+                "sha256:5e3bc157fed2a4c02ec468de4ecd12a6e22818d4f09cde2c31ee3226ffbefab2",
+                "sha256:612a10bdae23404a72941a0fc8fa2660c6ea1217c4ce0dbcab8a8f6543ea9e7f",
+                "sha256:657a05857bda581c3656bfc3b20e353c232e9193eb167766ad2dc58b56504948",
+                "sha256:65e720d2ab2b53f1f72fb5da5fb477455905ce2c88aaa671ff0a447c2c80e8e3",
+                "sha256:693902d433cf585133699972b6d7c42a8b9f8f826ebcaf0132ff55200afc599e",
+                "sha256:6af936f79086a89b3680a280c47ea90b4df7047b5bdf3aa5c524bbedddb9e545",
+                "sha256:71bb308552200fb2c195e35ef05de12f0c878c07fc91c270eb3d6e41698c3bcc",
+                "sha256:764202cc7e70f767dab49e8df52c7455e8de0df5d858fa801a11aa0d882ccf3f",
+                "sha256:76c8094ac20ec259471ac53e774623eb62e6e1f56cd8690c67ce6ce4fcb05650",
+                "sha256:78a42513018c41c2ffd262eb676442315cbfe3c44eed82385c2ed043bc63210a",
+                "sha256:79849239c39b5e1fd906556c474d9b0439ea6792b637511f3fe3a41158d89ca8",
+                "sha256:7ab9ccab2b5bd5702ab0803676a580fffa2aa178c2badc5557a84cc943fcf750",
+                "sha256:7bbfcb7165ce3d54a3dfbe731e470f65739c4c1f85bb1018ee912bae139e263b",
+                "sha256:7c06a4c7cf15ec739ce0e5971b26c93638730090add60e183530d70848ebdd34",
+                "sha256:801fa7802e5cfabe3ab0c81a34c323a319b097dfb5004be950482d882f3d7225",
+                "sha256:803b8e1459341c1bb56d1c5c010406d5edec8a0713a0945851290a7930679b51",
+                "sha256:82a5c2f4b87c26bb1a0ef3d16b5c4753434633b83d365cc0ddf2770c93829e3c",
+                "sha256:84ec80df401cfee1457063732d90022f93951944b5b58975d34ab56bb150dfb3",
+                "sha256:8705f17dfeb43139a692298cb6637ee2e59c0194538153e83e9ee0c75c2eddde",
+                "sha256:88a9ca9c710d598fd75ee5de59d5bda2684d9db36a9f50b6125eaea3969c2599",
+                "sha256:88f17c5ffa8e9462fb79f62746428dd57b46eb931698e42e990ad63103f35e6c",
+                "sha256:8a3ec5aa8e38fc4c8af308917ce12c536f1c88452ce554027e55b22cbbfbff76",
+                "sha256:8a9c83f75223d5e48b0bc9cb1bf2776cf01563e00ade8775ffe13b0b6e1af3a6",
+                "sha256:8b01aac285f91ca889c800042c35ad3b239e704b150cfd3382adfc9dcc780e39",
+                "sha256:8d53103597a252fb3ab8b5845af04c7a26d5e7ea8122303dd7a021176a87e8b9",
+                "sha256:8e045731a5416357638d1700927529e2b8ab304811671f665b225f8bf8d8f933",
+                "sha256:8f0ea6da6d393d8b2e187e6a5e3fb81f5862010a40c3945e2c6d12ae45cfb2ad",
+                "sha256:90da3b5f694b85231cf93586dad5e90e2d71b9428f9aad96952c99055582f520",
+                "sha256:913983ad2deb14e66d83c28b632fd35ba2b825031f2fa4ca29675e665dfecbe1",
+                "sha256:9242795d174daa40105c1d86aba618e8eab7bf96ba8c3ee614da8302a9f95503",
+                "sha256:929e294c1ac1e9f615c62a4e4313ca1823ba37326c164ec720a803287c4c499b",
+                "sha256:933d4de052939d90afbe6e9d5273ae05fb836cc86c15b686edd4b3560cc0ee36",
+                "sha256:942216596dc64ddb25adb215c3c783215b23626f8d84e8eff8d6d45c3f29f75a",
+                "sha256:94252291e3fe68001b1dd747b4c0b3be12582839b95ad4d1b641924d68fd4643",
+                "sha256:9893ff81bd7107f7b685d3017cc6583daadb4fc26e4a888350df530e41980a60",
+                "sha256:9e838bba3a3bac0fe06d849d29772eb1afb9745a59710762e4ba3f4cb8424483",
+                "sha256:a0f64a48bb81af7450e641e3fe0b0394d7381e342805479178b3d335d60ca7cf",
+                "sha256:a17f6a29cf8935e587cc8a4dbfc8368c55edc645283db0ce9801016f83526c2d",
+                "sha256:a1ecf0ac1c518487d9d23b1cd7139a6a65bc460cd101ab01f1be82ecf09794b6",
+                "sha256:a79ae34384df2b615eefca647a2873842ac3b596418032bef9a7283675962644",
+                "sha256:a91b5f9f1205845d488c928e8570dcb62b893372f63b8b6e98b863ebd2368ff2",
+                "sha256:aa0abdf853e09aff551db11fce173e2177d00786c688203f52c87ad7fcd91ef9",
+                "sha256:ac542bf38a8a4be2dc6b15248d36315ccc65f0743f7b1a76688ffb6b5129a5c2",
+                "sha256:ad42ba922c67c5f219097b28fae965e10045ddf145d2928bfac2eb2e17673640",
+                "sha256:aeb3531b196ef6f11776c21674dba836aeea9d5bd1cf630f869e3d90b16cfade",
+                "sha256:b38ac83d5f04b15e515fd86f312479d950d05ce2368d5413d46c088dda7de90a",
+                "sha256:b7d755065e4e866a8086c9bdada157133ff466476a2ad7861828e17b6026e22c",
+                "sha256:bd3de6481f4ed8b734da5df134cd5a6a64fe32124fe83dde1e5b5f29fe30b1e6",
+                "sha256:bfa1acfa0c54932d5607e19a2c24646fb4c1ae2694437789129cf099789a3b00",
+                "sha256:c619b101e6de2222c1fcb0531e1b17bbffbe54294bfba43ea0d411d428618c27",
+                "sha256:ce8be0466f4c0d585cdb6c1e2ed07232221df101a4c6f28821d2aa754ca2d9e2",
+                "sha256:cf0438b42121a66a3a667de17e779330fc0f20b0d97d59d2f2121e182b0505e4",
+                "sha256:cf8bcc23ceb5a1b624572a1623b9f79d2c3b337c8c455405ef231933a10da379",
+                "sha256:d2b0e12a42fb4e72d509fc994713d099cbb15ebf1103545e8a45f14da2dfca54",
+                "sha256:d83db7cde68459fc803052a55ace60bea2bae361fc3b7a6d5da07e11954e4b09",
+                "sha256:dda56c24d869b1193fcc763f1284b9126550eaf84b88bbc7256e15028f19188a",
+                "sha256:dea0bf229319828467d7fca8c7c189780aa9ff679c94539eed7532ebe33ed37c",
+                "sha256:e1631290ee9271dffe3062d2634c3ecac02c83890ada077d225e081aca8aab89",
+                "sha256:e28c7fea2196bf4c2f8d46a0415c77a1c480cc0724722f23d7410ffe9842c407",
+                "sha256:e2e6c39bd7b9372b0be21456caab138e8e69cc0fc1190a9dfa92bd45a1e6e904",
+                "sha256:e33e8fbd440c917106b237ef1a2f1449dfbb9b6f6e1ce17c94cd6a1e0d438376",
+                "sha256:e8df2eb9b2bac43ef8b082e06f750350fbbaf2887534a5be97f6cf07b19d9583",
+                "sha256:e968b84db54f9d42046cf154e02911e39c0435c9801681e3fc9ce8a3c4130278",
+                "sha256:eb542fe7933aa09d8d8f9d9097ef37532a7df6497819d16efe4359890a2f417a",
+                "sha256:edcfc407e4eb17e037bca59be0e85a2031a2ac87e4fed26d3e9df88b4165f92d",
+                "sha256:eee3ea935c3d227d49b4eb85660ff631556841f6e567f0f7bda972df6c2c9935",
+                "sha256:ef97b8df011141c9b0f6caf23b29379f87dd13183c978a30a3c546d2c47314cb",
+                "sha256:f106407dda69ae456dd1227966bf445b157ccc80ba0dff3802bb63f30b74e895",
+                "sha256:f3160309af4396e0ed04db259c3ccbfdc3621b5559b5453075e5de555e1f3a1b",
+                "sha256:f32d6edbc638cde7652bd690c3e728b25332acbadd7cad670cc4a02558d9c417",
+                "sha256:f37cfe618a117e50d8c240555331160d73d0411422b59b5ee217843d7b693608",
+                "sha256:f4c9aee212bc89d4e13f58be11a56cc8036cabad119259d12ace14b34476fd07",
+                "sha256:f4d742cb7af1c28303a51b7a27aaee540e71bb8e24f68c736f6f2ffc82f2bf05",
+                "sha256:f5a8b53bdc0b3961f8b6125e198617c40aeed638b387913bf1ce78afb1b0be2a",
+                "sha256:f816dd2277f8d63d79f9c8473a79fe54047bc0467754962840782c575522224d",
+                "sha256:f9a9e8a507420fe35992ee9ecb302dab68550dedc0da9e2880dd88071c5fb052"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==25.1.0"
+            "version": "==1.4.7"
         },
         "llvmlite": {
             "hashes": [
-                "sha256:05cb7e9b6ce69165ce4d1b994fbdedca0c62492e537b0cc86141b6e2c78d5888",
-                "sha256:08fa9ab02b0d0179c688a4216b8939138266519aaa0aa94f1195a8542faedb56",
-                "sha256:3366938e1bf63d26c34fbfb4c8e8d2ded57d11e0567d5bb243d89aab1eb56098",
-                "sha256:43d65cc4e206c2e902c1004dd5418417c4efa6c1d04df05c6c5675a27e8ca90e",
-                "sha256:70f44ccc3c6220bd23e0ba698a63ec2a7d3205da0d848804807f37fc243e3f77",
-                "sha256:763f8d8717a9073b9e0246998de89929071d15b47f254c10eef2310b9aac033d",
-                "sha256:7e0c4c11c8c2aa9b0701f91b799cb9134a6a6de51444eff5a9087fc7c1384275",
-                "sha256:81e674c2fe85576e6c4474e8c7e7aba7901ac0196e864fe7985492b737dbab65",
-                "sha256:8d90edf400b4ceb3a0e776b6c6e4656d05c7187c439587e06f86afceb66d2be5",
-                "sha256:a78ab89f1924fc11482209f6799a7a3fc74ddc80425a7a3e0e8174af0e9e2301",
-                "sha256:ae511caed28beaf1252dbaf5f40e663f533b79ceb408c874c01754cafabb9cbf",
-                "sha256:b2fce7d355068494d1e42202c7aff25d50c462584233013eb4470c33b995e3ee",
-                "sha256:bb3975787f13eb97629052edb5017f6c170eebc1c14a0433e8089e5db43bcce6",
-                "sha256:bdd3888544538a94d7ec99e7c62a0cdd8833609c85f0c23fcb6c5c591aec60ad",
-                "sha256:c35da49666a21185d21b551fc3caf46a935d54d66969d32d72af109b5e7d2b6f",
-                "sha256:c5bece0cdf77f22379f19b1959ccd7aee518afa4afbd3656c6365865f84903f9",
-                "sha256:d0936c2067a67fb8816c908d5457d63eba3e2b17e515c5fe00e5ee2bace06040",
-                "sha256:d47494552559e00d81bfb836cf1c4d5a5062e54102cc5767d5aa1e77ccd2505c",
-                "sha256:d7599b65c7af7abbc978dbf345712c60fd596aa5670496561cc10e8a71cebfb2",
-                "sha256:ebe66a86dc44634b59a3bc860c7b20d26d9aaffcd30364ebe8ba79161a9121f4",
-                "sha256:f92b09243c0cc3f457da8b983f67bd8e1295d0f5b3746c7a1861d7a99403854a"
+                "sha256:14f0e4bf2fd2d9a75a3534111e8ebeb08eda2f33e9bdd6dfa13282afacdde0ed",
+                "sha256:18e9953c748b105668487b7c81a3e97b046d8abf95c4ddc0cd3c94f4e4651ae8",
+                "sha256:35d80d61d0cda2d767f72de99450766250560399edc309da16937b93d3b676e7",
+                "sha256:3e8d0618cb9bfe40ac38a9633f2493d4d4e9fcc2f438d39a4e854f39cc0f5f98",
+                "sha256:47e147cdda9037f94b399bf03bfd8a6b6b1f2f90be94a454e3386f006455a9b4",
+                "sha256:6912a87782acdff6eb8bf01675ed01d60ca1f2551f8176a300a886f09e836a6a",
+                "sha256:6d4fd101f571a31acb1559ae1af30f30b1dc4b3186669f92ad780e17c81e91bc",
+                "sha256:74937acd22dc11b33946b67dca7680e6d103d6e90eeaaaf932603bec6fe7b03a",
+                "sha256:7a2872ee80dcf6b5dbdc838763d26554c2a18aa833d31a2635bff16aafefb9c9",
+                "sha256:7d434ec7e2ce3cc8f452d1cd9a28591745de022f931d67be688a737320dfcead",
+                "sha256:977525a1e5f4059316b183fb4fd34fa858c9eade31f165427a3977c95e3ee749",
+                "sha256:9cd2a7376f7b3367019b664c21f0c61766219faa3b03731113ead75107f3b66c",
+                "sha256:a289af9a1687c6cf463478f0fa8e8aa3b6fb813317b0d70bf1ed0759eab6f761",
+                "sha256:ae2b5b5c3ef67354824fb75517c8db5fbe93bc02cd9671f3c62271626bc041d5",
+                "sha256:bc9efc739cc6ed760f795806f67889923f7274276f0eb45092a1473e40d9b867",
+                "sha256:c1da416ab53e4f7f3bc8d4eeba36d801cc1894b9fbfbf2022b29b6bad34a7df2",
+                "sha256:d5bd550001d26450bd90777736c69d68c487d17bf371438f975229b2b8241a91",
+                "sha256:df6509e1507ca0760787a199d19439cc887bfd82226f5af746d6977bd9f66844",
+                "sha256:e0a9a1a39d4bf3517f2af9d23d479b4175ead205c592ceeb8b89af48a327ea57",
+                "sha256:eccce86bba940bae0d8d48ed925f21dbb813519169246e2ab292b5092aba121f",
+                "sha256:f99b600aa7f65235a5a05d0b9a9f31150c390f31261f2a0ba678e26823ec38f7"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==0.42.0"
+            "version": "==0.43.0"
         },
         "markdown-it-py": {
             "hashes": [
@@ -632,6 +936,53 @@
             "markers": "python_version >= '3.7'",
             "version": "==2.1.5"
         },
+        "matplotlib": {
+            "hashes": [
+                "sha256:039082812cacd6c6bec8e17a9c1e6baca230d4116d522e81e1f63a74d01d2e21",
+                "sha256:03ba9c1299c920964e8d3857ba27173b4dbb51ca4bab47ffc2c2ba0eb5e2cbc5",
+                "sha256:050598c2b29e0b9832cde72bcf97627bf00262adbc4a54e2b856426bb2ef0697",
+                "sha256:18128cc08f0d3cfff10b76baa2f296fc28c4607368a8402de61bb3f2eb33c7d9",
+                "sha256:1cd93b91ab47a3616b4d3c42b52f8363b88ca021e340804c6ab2536344fad9ca",
+                "sha256:1d94ff717eb2bd0b58fe66380bd8b14ac35f48a98e7c6765117fe67fb7684e64",
+                "sha256:306c8dfc73239f0e72ac50e5a9cf19cc4e8e331dd0c54f5e69ca8758550f1e1e",
+                "sha256:37e51dd1c2db16ede9cfd7b5cabdfc818b2c6397c83f8b10e0e797501c963a03",
+                "sha256:3fd595f34aa8a55b7fc8bf9ebea8aa665a84c82d275190a61118d33fbc82ccae",
+                "sha256:4876d7d40219e8ae8bb70f9263bcbe5714415acfdf781086601211335e24f8aa",
+                "sha256:5413401594cfaff0052f9d8b1aafc6d305b4bd7c4331dccd18f561ff7e1d3bd3",
+                "sha256:5816b1e1fe8c192cbc013f8f3e3368ac56fbecf02fb41b8f8559303f24c5015e",
+                "sha256:65aacf95b62272d568044531e41de26285d54aec8cb859031f511f84bd8b495a",
+                "sha256:6758baae2ed64f2331d4fd19be38b7b4eae3ecec210049a26b6a4f3ae1c85dcc",
+                "sha256:6d1ce5ed2aefcdce11904fc5bbea7d9c21fff3d5f543841edf3dea84451a09ea",
+                "sha256:6d9f07a80deab4bb0b82858a9e9ad53d1382fd122be8cde11080f4e7dfedb38b",
+                "sha256:7741f26a58a240f43bee74965c4882b6c93df3e7eb3de160126d8c8f53a6ae6e",
+                "sha256:8912ef7c2362f7193b5819d17dae8629b34a95c58603d781329712ada83f9447",
+                "sha256:909645cce2dc28b735674ce0931a4ac94e12f5b13f6bb0b5a5e65e7cea2c192b",
+                "sha256:96ab43906269ca64a6366934106fa01534454a69e471b7bf3d79083981aaab92",
+                "sha256:9d78bbc0cbc891ad55b4f39a48c22182e9bdaea7fc0e5dbd364f49f729ca1bbb",
+                "sha256:ab68d50c06938ef28681073327795c5db99bb4666214d2d5f880ed11aeaded66",
+                "sha256:ac43031375a65c3196bee99f6001e7fa5bdfb00ddf43379d3c0609bdca042df9",
+                "sha256:ae82a14dab96fbfad7965403c643cafe6515e386de723e498cf3eeb1e0b70cc7",
+                "sha256:b2696efdc08648536efd4e1601b5fd491fd47f4db97a5fbfd175549a7365c1b2",
+                "sha256:b82c5045cebcecd8496a4d694d43f9cc84aeeb49fe2133e036b207abe73f4d30",
+                "sha256:be0fc24a5e4531ae4d8e858a1a548c1fe33b176bb13eff7f9d0d38ce5112a27d",
+                "sha256:bf81de2926c2db243c9b2cbc3917619a0fc85796c6ba4e58f541df814bbf83c7",
+                "sha256:c375cc72229614632c87355366bdf2570c2dac01ac66b8ad048d2dabadf2d0d4",
+                "sha256:c797dac8bb9c7a3fd3382b16fe8f215b4cf0f22adccea36f1545a6d7be310b41",
+                "sha256:cef2a73d06601437be399908cf13aee74e86932a5ccc6ccdf173408ebc5f6bb2",
+                "sha256:d52a3b618cb1cbb769ce2ee1dcdb333c3ab6e823944e9a2d36e37253815f9556",
+                "sha256:d719465db13267bcef19ea8954a971db03b9f48b4647e3860e4bc8e6ed86610f",
+                "sha256:d8dd059447824eec055e829258ab092b56bb0579fc3164fa09c64f3acd478772",
+                "sha256:dbe196377a8248972f5cede786d4c5508ed5f5ca4a1e09b44bda889958b33f8c",
+                "sha256:e0830e188029c14e891fadd99702fd90d317df294c3298aad682739c5533721a",
+                "sha256:f053c40f94bc51bc03832a41b4f153d83f2062d88c72b5e79997072594e97e51",
+                "sha256:f32c7410c7f246838a77d6d1eff0c0f87f3cb0e7c4247aebea71a6d5a68cab49",
+                "sha256:f6ee45bc4245533111ced13f1f2cace1e7f89d1c793390392a80c139d6cf0e6c",
+                "sha256:f7c0410f181a531ec4e93bbc27692f2c71a15c2da16766f5ba9761e7ae518413"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.9'",
+            "version": "==3.9.2"
+        },
         "mccabe": {
             "hashes": [
                 "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
@@ -649,11 +1000,18 @@
         },
         "more-itertools": {
             "hashes": [
-                "sha256:686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684",
-                "sha256:8fccb480c43d3e99a00087634c06dd02b0d50fbf088b380de5a41a015ec239e1"
+                "sha256:037b0d3203ce90cca8ab1defbbdac29d5f993fc20131f3664dc8d6acfa872aef",
+                "sha256:5482bfef7849c25dc3c6dd53a6173ae4795da2a41a80faea6700d9f5846c5da6"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==10.2.0"
+            "version": "==10.5.0"
+        },
+        "mpmath": {
+            "hashes": [
+                "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f",
+                "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"
+            ],
+            "version": "==1.3.0"
         },
         "mypy-extensions": {
             "hashes": [
@@ -663,61 +1021,70 @@
             "markers": "python_version >= '3.5'",
             "version": "==1.0.0"
         },
+        "networkx": {
+            "hashes": [
+                "sha256:0c127d8b2f4865f59ae9cb8aafcd60b5c70f3241ebd66f7defad7c4ab90126c9",
+                "sha256:28575580c6ebdaf4505b22c6256a2b9de86b316dc63ba9e93abde3d78dfdbcf2"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.10'",
+            "version": "==3.3"
+        },
         "nh3": {
             "hashes": [
-                "sha256:0316c25b76289cf23be6b66c77d3608a4fdf537b35426280032f432f14291b9a",
-                "sha256:1a814dd7bba1cb0aba5bcb9bebcc88fd801b63e21e2450ae6c52d3b3336bc911",
-                "sha256:1aa52a7def528297f256de0844e8dd680ee279e79583c76d6fa73a978186ddfb",
-                "sha256:22c26e20acbb253a5bdd33d432a326d18508a910e4dcf9a3316179860d53345a",
-                "sha256:40015514022af31975c0b3bca4014634fa13cb5dc4dbcbc00570acc781316dcc",
-                "sha256:40d0741a19c3d645e54efba71cb0d8c475b59135c1e3c580f879ad5514cbf028",
-                "sha256:551672fd71d06cd828e282abdb810d1be24e1abb7ae2543a8fa36a71c1006fe9",
-                "sha256:66f17d78826096291bd264f260213d2b3905e3c7fae6dfc5337d49429f1dc9f3",
-                "sha256:85cdbcca8ef10733bd31f931956f7fbb85145a4d11ab9e6742bbf44d88b7e351",
-                "sha256:a3f55fabe29164ba6026b5ad5c3151c314d136fd67415a17660b4aaddacf1b10",
-                "sha256:b4427ef0d2dfdec10b641ed0bdaf17957eb625b2ec0ea9329b3d28806c153d71",
-                "sha256:ba73a2f8d3a1b966e9cdba7b211779ad8a2561d2dba9674b8a19ed817923f65f",
-                "sha256:c21bac1a7245cbd88c0b0e4a420221b7bfa838a2814ee5bb924e9c2f10a1120b",
-                "sha256:c551eb2a3876e8ff2ac63dff1585236ed5dfec5ffd82216a7a174f7c5082a78a",
-                "sha256:c790769152308421283679a142dbdb3d1c46c79c823008ecea8e8141db1a2062",
-                "sha256:d7a25fd8c86657f5d9d576268e3b3767c5cd4f42867c9383618be8517f0f022a"
-            ],
-            "version": "==0.2.17"
+                "sha256:0411beb0589eacb6734f28d5497ca2ed379eafab8ad8c84b31bb5c34072b7164",
+                "sha256:14c5a72e9fe82aea5fe3072116ad4661af5cf8e8ff8fc5ad3450f123e4925e86",
+                "sha256:19aaba96e0f795bd0a6c56291495ff59364f4300d4a39b29a0abc9cb3774a84b",
+                "sha256:34c03fa78e328c691f982b7c03d4423bdfd7da69cd707fe572f544cf74ac23ad",
+                "sha256:36c95d4b70530b320b365659bb5034341316e6a9b30f0b25fa9c9eff4c27a204",
+                "sha256:3a157ab149e591bb638a55c8c6bcb8cdb559c8b12c13a8affaba6cedfe51713a",
+                "sha256:42c64511469005058cd17cc1537578eac40ae9f7200bedcfd1fc1a05f4f8c200",
+                "sha256:5f36b271dae35c465ef5e9090e1fdaba4a60a56f0bb0ba03e0932a66f28b9189",
+                "sha256:6955369e4d9f48f41e3f238a9e60f9410645db7e07435e62c6a9ea6135a4907f",
+                "sha256:7b7c2a3c9eb1a827d42539aa64091640bd275b81e097cd1d8d82ef91ffa2e811",
+                "sha256:8ce0f819d2f1933953fca255db2471ad58184a60508f03e6285e5114b6254844",
+                "sha256:94a166927e53972a9698af9542ace4e38b9de50c34352b962f4d9a7d4c927af4",
+                "sha256:a7f1b5b2c15866f2db413a3649a8fe4fd7b428ae58be2c0f6bca5eefd53ca2be",
+                "sha256:c8b3a1cebcba9b3669ed1a84cc65bf005728d2f0bc1ed2a6594a992e817f3a50",
+                "sha256:de3ceed6e661954871d6cd78b410213bdcb136f79aafe22aa7182e028b8c7307",
+                "sha256:f0eca9ca8628dbb4e916ae2491d72957fdd35f7a5d326b7032a345f111ac07fe"
+            ],
+            "version": "==0.2.18"
         },
         "nodeenv": {
             "hashes": [
-                "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2",
-                "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"
+                "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f",
+                "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'",
-            "version": "==1.8.0"
+            "version": "==1.9.1"
         },
         "numba": {
             "hashes": [
-                "sha256:0594b3dfb369fada1f8bb2e3045cd6c61a564c62e50cf1f86b4666bc721b3450",
-                "sha256:0b77aecf52040de2a1eb1d7e314497b9e56fba17466c80b457b971a25bb1576d",
-                "sha256:0f68589740a8c38bb7dc1b938b55d1145244c8353078eea23895d4f82c8b9ec1",
-                "sha256:1cce206a3b92836cdf26ef39d3a3242fec25e07f020cc4feec4c4a865e340569",
-                "sha256:2801003caa263d1e8497fb84829a7ecfb61738a95f62bc05693fcf1733e978e4",
-                "sha256:3476a4f641bfd58f35ead42f4dcaf5f132569c4647c6f1360ccf18ee4cda3990",
-                "sha256:411df625372c77959570050e861981e9d196cc1da9aa62c3d6a836b5cc338966",
-                "sha256:43727e7ad20b3ec23ee4fc642f5b61845c71f75dd2825b3c234390c6d8d64051",
-                "sha256:4e0318ae729de6e5dbe64c75ead1a95eb01fabfe0e2ebed81ebf0344d32db0ae",
-                "sha256:525ef3f820931bdae95ee5379c670d5c97289c6520726bc6937a4a7d4230ba24",
-                "sha256:5bf68f4d69dd3a9f26a9b23548fa23e3bcb9042e2935257b471d2a8d3c424b7f",
-                "sha256:649913a3758891c77c32e2d2a3bcbedf4a69f5fea276d11f9119677c45a422e8",
-                "sha256:76f69132b96028d2774ed20415e8c528a34e3299a40581bae178f0994a2f370b",
-                "sha256:7d80bce4ef7e65bf895c29e3889ca75a29ee01da80266a01d34815918e365835",
-                "sha256:8c8b4477763cb1fbd86a3be7050500229417bf60867c93e131fd2626edb02238",
-                "sha256:8d51ccd7008a83105ad6a0082b6a2b70f1142dc7cfd76deb8c5a862367eb8c86",
-                "sha256:9712808e4545270291d76b9a264839ac878c5eb7d8b6e02c970dc0ac29bc8187",
-                "sha256:97385a7f12212c4f4bc28f648720a92514bee79d7063e40ef66c2d30600fd18e",
-                "sha256:990e395e44d192a12105eca3083b61307db7da10e093972ca285c85bef0963d6",
-                "sha256:dd2842fac03be4e5324ebbbd4d2d0c8c0fc6e0df75c09477dd45b288a0777389",
-                "sha256:f7ad1d217773e89a9845886401eaaab0a156a90aa2f179fdc125261fd1105096"
+                "sha256:01ef4cd7d83abe087d644eaa3d95831b777aa21d441a23703d649e06b8e06b74",
+                "sha256:0b983bd6ad82fe868493012487f34eae8bf7dd94654951404114f23c3466d34b",
+                "sha256:0ebaa91538e996f708f1ab30ef4d3ddc344b64b5227b67a57aa74f401bb68b9d",
+                "sha256:1527dc578b95c7c4ff248792ec33d097ba6bef9eda466c948b68dfc995c25781",
+                "sha256:159e618ef213fba758837f9837fb402bbe65326e60ba0633dbe6c7f274d42c1b",
+                "sha256:19407ced081d7e2e4b8d8c36aa57b7452e0283871c296e12d798852bc7d7f198",
+                "sha256:3031547a015710140e8c87226b4cfe927cac199835e5bf7d4fe5cb64e814e3ab",
+                "sha256:38d6ea4c1f56417076ecf8fc327c831ae793282e0ff51080c5094cb726507b1c",
+                "sha256:3fb02b344a2a80efa6f677aa5c40cd5dd452e1b35f8d1c2af0dfd9ada9978e4b",
+                "sha256:4142d7ac0210cc86432b818338a2bc368dc773a2f5cf1e32ff7c5b378bd63ee8",
+                "sha256:5d761de835cd38fb400d2c26bb103a2726f548dc30368853121d66201672e651",
+                "sha256:5df6158e5584eece5fc83294b949fd30b9f1125df7708862205217e068aabf16",
+                "sha256:5f4fde652ea604ea3c86508a3fb31556a6157b2c76c8b51b1d45eb40c8598703",
+                "sha256:62908d29fb6a3229c242e981ca27e32a6e606cc253fc9e8faeb0e48760de241e",
+                "sha256:819a3dfd4630d95fd574036f99e47212a1af41cbcb019bf8afac63ff56834449",
+                "sha256:a17b70fc9e380ee29c42717e8cc0bfaa5556c416d94f9aa96ba13acb41bdece8",
+                "sha256:c151748cd269ddeab66334bd754817ffc0cabd9433acb0f551697e5151917d25",
+                "sha256:cac02c041e9b5bc8cf8f2034ff6f0dbafccd1ae9590dc146b3a02a45e53af4e2",
+                "sha256:d7da4098db31182fc5ffe4bc42c6f24cd7d1cb8a14b59fd755bfee32e34b8404",
+                "sha256:f75262e8fe7fa96db1dca93d53a194a38c46da28b112b8a4aca168f0df860347",
+                "sha256:fe0b28abb8d70f8160798f4de9d486143200f34458d34c4a214114e445d7124e"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==0.59.1"
+            "version": "==0.60.0"
         },
         "numpy": {
             "hashes": [
@@ -764,44 +1131,39 @@
         },
         "packaging": {
             "hashes": [
-                "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5",
-                "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"
+                "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002",
+                "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==24.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==24.1"
         },
         "pandas": {
             "hashes": [
-                "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863",
-                "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2",
-                "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1",
-                "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad",
-                "sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db",
-                "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76",
-                "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51",
-                "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32",
-                "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08",
-                "sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b",
-                "sha256:640cef9aa381b60e296db324337a554aeeb883ead99dc8f6c18e81a93942f5f4",
-                "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921",
-                "sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288",
-                "sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee",
-                "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0",
-                "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24",
-                "sha256:8e5a0b00e1e56a842f922e7fae8ae4077aee4af0acb5ae3622bd4b4c30aedf99",
-                "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151",
-                "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce",
-                "sha256:92fd6b027924a7e178ac202cfbe25e53368db90d56872d20ffae94b96c7acc57",
-                "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef",
-                "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54",
-                "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a",
-                "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23",
-                "sha256:ddf818e4e6c7c6f4f7c8a12709696d193976b591cc7dc50588d3d1a6b5dc8772",
-                "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce",
-                "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"
+                "sha256:0010771bd9223f7afe5f051eb47c4a49534345dfa144f2f5470b27189a4dd3b5",
+                "sha256:061609334a8182ab500a90fe66d46f6f387de62d3a9cb9aa7e62e3146c712167",
+                "sha256:09d8be7dd9e1c4c98224c4dfe8abd60d145d934e9fc1f5f411266308ae683e6a",
+                "sha256:295872bf1a09758aba199992c3ecde455f01caf32266d50abc1a073e828a7b9d",
+                "sha256:3228198333dd13c90b6434ddf61aa6d57deaca98cf7b654f4ad68a2db84f8cfe",
+                "sha256:385c52e85aaa8ea6a4c600a9b2821181a51f8be0aee3af6f2dcb41dafc4fc1d0",
+                "sha256:51649ef604a945f781105a6d2ecf88db7da0f4868ac5d45c51cb66081c4d9c73",
+                "sha256:5586cc95692564b441f4747c47c8a9746792e87b40a4680a2feb7794defb1ce3",
+                "sha256:5a206afa84ed20e07603f50d22b5f0db3fb556486d8c2462d8bc364831a4b417",
+                "sha256:5b79af3a69e5175c6fa7b4e046b21a646c8b74e92c6581a9d825687d92071b51",
+                "sha256:5c54ea4ef3823108cd4ec7fb27ccba4c3a775e0f83e39c5e17f5094cb17748bc",
+                "sha256:8c5bf555b6b0075294b73965adaafb39cf71c312e38c5935c93d78f41c19828a",
+                "sha256:92bc1fc585f1463ca827b45535957815b7deb218c549b7c18402c322c7549a12",
+                "sha256:95c1e422ced0199cf4a34385ff124b69412c4bc912011ce895582bee620dfcaa",
+                "sha256:b8134651258bce418cb79c71adeff0a44090c98d955f6953168ba16cc285d9f7",
+                "sha256:be67c782c4f1b1f24c2f16a157e12c2693fd510f8df18e3287c77f33d124ed07",
+                "sha256:c072c7f06b9242c855ed8021ff970c0e8f8b10b35e2640c657d2a541c5950f59",
+                "sha256:d0d4f13e4be7ce89d7057a786023c461dd9370040bdb5efa0a7fe76b556867a0",
+                "sha256:df82739e00bb6daf4bba4479a40f38c718b598a84654cbd8bb498fd6b0aa8c16",
+                "sha256:f549097993744ff8c41b5e8f2f0d3cbfaabe89b4ae32c8c08ead6cc535b80139",
+                "sha256:ff08a14ef21d94cdf18eef7c569d66f2e24e0bc89350bcd7d243dd804e3b5eb2"
             ],
-            "markers": "python_version >= '3.9'",
-            "version": "==2.2.2"
+            "index": "pypi",
+            "markers": "python_version >= '3.8'",
+            "version": "==1.4.2"
         },
         "pathspec": {
             "hashes": [
@@ -811,6 +1173,92 @@
             "markers": "python_version >= '3.8'",
             "version": "==0.12.1"
         },
+        "pillow": {
+            "hashes": [
+                "sha256:02a2be69f9c9b8c1e97cf2713e789d4e398c751ecfd9967c18d0ce304efbf885",
+                "sha256:030abdbe43ee02e0de642aee345efa443740aa4d828bfe8e2eb11922ea6a21ea",
+                "sha256:06b2f7898047ae93fad74467ec3d28fe84f7831370e3c258afa533f81ef7f3df",
+                "sha256:0755ffd4a0c6f267cccbae2e9903d95477ca2f77c4fcf3a3a09570001856c8a5",
+                "sha256:0a9ec697746f268507404647e531e92889890a087e03681a3606d9b920fbee3c",
+                "sha256:0ae24a547e8b711ccaaf99c9ae3cd975470e1a30caa80a6aaee9a2f19c05701d",
+                "sha256:134ace6dc392116566980ee7436477d844520a26a4b1bd4053f6f47d096997fd",
+                "sha256:166c1cd4d24309b30d61f79f4a9114b7b2313d7450912277855ff5dfd7cd4a06",
+                "sha256:1b5dea9831a90e9d0721ec417a80d4cbd7022093ac38a568db2dd78363b00908",
+                "sha256:1d846aea995ad352d4bdcc847535bd56e0fd88d36829d2c90be880ef1ee4668a",
+                "sha256:1ef61f5dd14c300786318482456481463b9d6b91ebe5ef12f405afbba77ed0be",
+                "sha256:297e388da6e248c98bc4a02e018966af0c5f92dfacf5a5ca22fa01cb3179bca0",
+                "sha256:298478fe4f77a4408895605f3482b6cc6222c018b2ce565c2b6b9c354ac3229b",
+                "sha256:29dbdc4207642ea6aad70fbde1a9338753d33fb23ed6956e706936706f52dd80",
+                "sha256:2db98790afc70118bd0255c2eeb465e9767ecf1f3c25f9a1abb8ffc8cfd1fe0a",
+                "sha256:32cda9e3d601a52baccb2856b8ea1fc213c90b340c542dcef77140dfa3278a9e",
+                "sha256:37fb69d905be665f68f28a8bba3c6d3223c8efe1edf14cc4cfa06c241f8c81d9",
+                "sha256:416d3a5d0e8cfe4f27f574362435bc9bae57f679a7158e0096ad2beb427b8696",
+                "sha256:43efea75eb06b95d1631cb784aa40156177bf9dd5b4b03ff38979e048258bc6b",
+                "sha256:4b35b21b819ac1dbd1233317adeecd63495f6babf21b7b2512d244ff6c6ce309",
+                "sha256:4d9667937cfa347525b319ae34375c37b9ee6b525440f3ef48542fcf66f2731e",
+                "sha256:5161eef006d335e46895297f642341111945e2c1c899eb406882a6c61a4357ab",
+                "sha256:543f3dc61c18dafb755773efc89aae60d06b6596a63914107f75459cf984164d",
+                "sha256:551d3fd6e9dc15e4c1eb6fc4ba2b39c0c7933fa113b220057a34f4bb3268a060",
+                "sha256:59291fb29317122398786c2d44427bbd1a6d7ff54017075b22be9d21aa59bd8d",
+                "sha256:5b001114dd152cfd6b23befeb28d7aee43553e2402c9f159807bf55f33af8a8d",
+                "sha256:5b4815f2e65b30f5fbae9dfffa8636d992d49705723fe86a3661806e069352d4",
+                "sha256:5dc6761a6efc781e6a1544206f22c80c3af4c8cf461206d46a1e6006e4429ff3",
+                "sha256:5e84b6cc6a4a3d76c153a6b19270b3526a5a8ed6b09501d3af891daa2a9de7d6",
+                "sha256:6209bb41dc692ddfee4942517c19ee81b86c864b626dbfca272ec0f7cff5d9fb",
+                "sha256:673655af3eadf4df6b5457033f086e90299fdd7a47983a13827acf7459c15d94",
+                "sha256:6c762a5b0997f5659a5ef2266abc1d8851ad7749ad9a6a5506eb23d314e4f46b",
+                "sha256:7086cc1d5eebb91ad24ded9f58bec6c688e9f0ed7eb3dbbf1e4800280a896496",
+                "sha256:73664fe514b34c8f02452ffb73b7a92c6774e39a647087f83d67f010eb9a0cf0",
+                "sha256:76a911dfe51a36041f2e756b00f96ed84677cdeb75d25c767f296c1c1eda1319",
+                "sha256:780c072c2e11c9b2c7ca37f9a2ee8ba66f44367ac3e5c7832afcfe5104fd6d1b",
+                "sha256:7928ecbf1ece13956b95d9cbcfc77137652b02763ba384d9ab508099a2eca856",
+                "sha256:7970285ab628a3779aecc35823296a7869f889b8329c16ad5a71e4901a3dc4ef",
+                "sha256:7a8d4bade9952ea9a77d0c3e49cbd8b2890a399422258a77f357b9cc9be8d680",
+                "sha256:7c1ee6f42250df403c5f103cbd2768a28fe1a0ea1f0f03fe151c8741e1469c8b",
+                "sha256:7dfecdbad5c301d7b5bde160150b4db4c659cee2b69589705b6f8a0c509d9f42",
+                "sha256:812f7342b0eee081eaec84d91423d1b4650bb9828eb53d8511bcef8ce5aecf1e",
+                "sha256:866b6942a92f56300012f5fbac71f2d610312ee65e22f1aa2609e491284e5597",
+                "sha256:86dcb5a1eb778d8b25659d5e4341269e8590ad6b4e8b44d9f4b07f8d136c414a",
+                "sha256:87dd88ded2e6d74d31e1e0a99a726a6765cda32d00ba72dc37f0651f306daaa8",
+                "sha256:8bc1a764ed8c957a2e9cacf97c8b2b053b70307cf2996aafd70e91a082e70df3",
+                "sha256:8d4d5063501b6dd4024b8ac2f04962d661222d120381272deea52e3fc52d3736",
+                "sha256:8f0aef4ef59694b12cadee839e2ba6afeab89c0f39a3adc02ed51d109117b8da",
+                "sha256:930044bb7679ab003b14023138b50181899da3f25de50e9dbee23b61b4de2126",
+                "sha256:950be4d8ba92aca4b2bb0741285a46bfae3ca699ef913ec8416c1b78eadd64cd",
+                "sha256:961a7293b2457b405967af9c77dcaa43cc1a8cd50d23c532e62d48ab6cdd56f5",
+                "sha256:9b885f89040bb8c4a1573566bbb2f44f5c505ef6e74cec7ab9068c900047f04b",
+                "sha256:9f4727572e2918acaa9077c919cbbeb73bd2b3ebcfe033b72f858fc9fbef0026",
+                "sha256:a02364621fe369e06200d4a16558e056fe2805d3468350df3aef21e00d26214b",
+                "sha256:a985e028fc183bf12a77a8bbf36318db4238a3ded7fa9df1b9a133f1cb79f8fc",
+                "sha256:ac1452d2fbe4978c2eec89fb5a23b8387aba707ac72810d9490118817d9c0b46",
+                "sha256:b15e02e9bb4c21e39876698abf233c8c579127986f8207200bc8a8f6bb27acf2",
+                "sha256:b2724fdb354a868ddf9a880cb84d102da914e99119211ef7ecbdc613b8c96b3c",
+                "sha256:bbc527b519bd3aa9d7f429d152fea69f9ad37c95f0b02aebddff592688998abe",
+                "sha256:bcd5e41a859bf2e84fdc42f4edb7d9aba0a13d29a2abadccafad99de3feff984",
+                "sha256:bd2880a07482090a3bcb01f4265f1936a903d70bc740bfcb1fd4e8a2ffe5cf5a",
+                "sha256:bee197b30783295d2eb680b311af15a20a8b24024a19c3a26431ff83eb8d1f70",
+                "sha256:bf2342ac639c4cf38799a44950bbc2dfcb685f052b9e262f446482afaf4bffca",
+                "sha256:c76e5786951e72ed3686e122d14c5d7012f16c8303a674d18cdcd6d89557fc5b",
+                "sha256:cbed61494057c0f83b83eb3a310f0bf774b09513307c434d4366ed64f4128a91",
+                "sha256:cfdd747216947628af7b259d274771d84db2268ca062dd5faf373639d00113a3",
+                "sha256:d7480af14364494365e89d6fddc510a13e5a2c3584cb19ef65415ca57252fb84",
+                "sha256:dbc6ae66518ab3c5847659e9988c3b60dc94ffb48ef9168656e0019a93dbf8a1",
+                "sha256:dc3e2db6ba09ffd7d02ae9141cfa0ae23393ee7687248d46a7507b75d610f4f5",
+                "sha256:dfe91cb65544a1321e631e696759491ae04a2ea11d36715eca01ce07284738be",
+                "sha256:e4d49b85c4348ea0b31ea63bc75a9f3857869174e2bf17e7aba02945cd218e6f",
+                "sha256:e4db64794ccdf6cb83a59d73405f63adbe2a1887012e308828596100a0b2f6cc",
+                "sha256:e553cad5179a66ba15bb18b353a19020e73a7921296a7979c4a2b7f6a5cd57f9",
+                "sha256:e88d5e6ad0d026fba7bdab8c3f225a69f063f116462c49892b0149e21b6c0a0e",
+                "sha256:ecd85a8d3e79cd7158dec1c9e5808e821feea088e2f69a974db5edf84dc53141",
+                "sha256:f5b92f4d70791b4a67157321c4e8225d60b119c5cc9aee8ecf153aace4aad4ef",
+                "sha256:f5f0c3e969c8f12dd2bb7e0b15d5c468b51e5017e01e2e867335c81903046a22",
+                "sha256:f7baece4ce06bade126fb84b8af1c33439a76d8a6fd818970215e0560ca28c27",
+                "sha256:ff25afb18123cea58a591ea0244b92eb1e61a1fd497bf6d6384f09bc3262ec3e",
+                "sha256:ff337c552345e95702c5fde3158acb0625111017d0e5f24bf3acdb9cc16b90d1"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==10.4.0"
+        },
         "pkginfo": {
             "hashes": [
                 "sha256:5df73835398d10db79f8eecd5cd86b1f6d29317589ea70796994d49399af6297",
@@ -821,19 +1269,19 @@
         },
         "platformdirs": {
             "hashes": [
-                "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068",
-                "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"
+                "sha256:63b79589009fa8159973601dd4563143396b35c5f93a58b36f9049ff046949b1",
+                "sha256:facaa5a3c57aa1e053e3da7b49e0cc31fe0113ca42a4659d5c2e98e545624afe"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.2.0"
+            "version": "==4.3.1"
         },
         "pluggy": {
             "hashes": [
-                "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981",
-                "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"
+                "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1",
+                "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.4.0"
+            "version": "==1.5.0"
         },
         "pre-commit": {
             "hashes": [
@@ -871,12 +1319,12 @@
         },
         "pyfakefs": {
             "hashes": [
-                "sha256:20cb51e860c2f3ff83859162ad5134bb8b0a1e7a81df0a18cfccc4862d0d9dcc",
-                "sha256:21d6a3276d9c964510c85cef0c568920d53ec9033da9b2a2c616489cedbe700a"
+                "sha256:1a45bba8615323ec29d65929d32dc66d7b59a1e60a02109950440edb0486c539",
+                "sha256:7a549b32865aa97d8ba6538285a93816941d9b7359be2954ac60ec36b277e879"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.7'",
-            "version": "==5.4.1"
+            "version": "==5.6.0"
         },
         "pyflakes": {
             "hashes": [
@@ -888,11 +1336,19 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c",
-                "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"
+                "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199",
+                "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.17.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.18.0"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c",
+                "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032"
+            ],
+            "markers": "python_full_version >= '3.6.8'",
+            "version": "==3.1.4"
         },
         "pytest": {
             "hashes": [
@@ -933,7 +1389,7 @@
                 "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3",
                 "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.9.0.post0"
         },
         "pytz": {
@@ -945,68 +1401,70 @@
         },
         "pywin32-ctypes": {
             "hashes": [
-                "sha256:3426e063bdd5fd4df74a14fa3cf80a0b42845a87e1d1e81f6549f9daec593a60",
-                "sha256:bf490a1a709baf35d688fe0ecf980ed4de11d2b3e37b51e5442587a75d9957e7"
+                "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8",
+                "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755"
             ],
             "markers": "sys_platform == 'win32'",
-            "version": "==0.2.2"
+            "version": "==0.2.3"
         },
         "pyyaml": {
             "hashes": [
-                "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5",
-                "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc",
-                "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df",
-                "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741",
-                "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206",
-                "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27",
-                "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595",
-                "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62",
-                "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98",
-                "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696",
-                "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290",
-                "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9",
-                "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d",
-                "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6",
-                "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867",
-                "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47",
-                "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486",
-                "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6",
-                "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3",
-                "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007",
-                "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938",
-                "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0",
-                "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c",
-                "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735",
-                "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d",
-                "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28",
-                "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4",
-                "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
-                "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
-                "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef",
-                "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
-                "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd",
-                "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3",
-                "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0",
-                "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515",
-                "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c",
-                "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c",
-                "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924",
-                "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34",
-                "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43",
-                "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859",
-                "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673",
-                "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54",
-                "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a",
-                "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b",
-                "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab",
-                "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa",
-                "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c",
-                "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585",
-                "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d",
-                "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"
+                "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff",
+                "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48",
+                "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086",
+                "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e",
+                "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133",
+                "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5",
+                "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484",
+                "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee",
+                "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5",
+                "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68",
+                "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a",
+                "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf",
+                "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99",
+                "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8",
+                "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85",
+                "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19",
+                "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc",
+                "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a",
+                "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1",
+                "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317",
+                "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c",
+                "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631",
+                "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d",
+                "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652",
+                "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5",
+                "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e",
+                "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b",
+                "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8",
+                "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476",
+                "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706",
+                "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563",
+                "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237",
+                "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b",
+                "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083",
+                "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180",
+                "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425",
+                "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e",
+                "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f",
+                "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725",
+                "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183",
+                "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab",
+                "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774",
+                "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725",
+                "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e",
+                "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5",
+                "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d",
+                "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290",
+                "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44",
+                "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed",
+                "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4",
+                "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba",
+                "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12",
+                "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==6.0.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==6.0.2"
         },
         "readme-renderer": {
             "hashes": [
@@ -1016,13 +1474,22 @@
             "markers": "python_version >= '3.8'",
             "version": "==43.0"
         },
+        "readthedocs-sphinx-search": {
+            "hashes": [
+                "sha256:66d950fadcc044a082816763c766d3ebc7130cb84caeabbffa239368dc5124fe",
+                "sha256:f3eacbebf766c0b12342462959913d4c5b2de1763f790a46563f719cd910f0f7"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.6'",
+            "version": "==0.1.2"
+        },
         "requests": {
             "hashes": [
-                "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
-                "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
+                "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760",
+                "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.31.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.32.3"
         },
         "requests-toolbelt": {
             "hashes": [
@@ -1042,11 +1509,19 @@
         },
         "rich": {
             "hashes": [
-                "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222",
-                "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"
+                "sha256:2e85306a063b9492dffc86278197a60cbece75bcb766022f3436f567cae11bdc",
+                "sha256:a5ac1f1cd448ade0d59cc3356f7db7a7ccda2c8cbae9c7a90c28ff463d3e91f4"
             ],
             "markers": "python_full_version >= '3.7.0'",
-            "version": "==13.7.1"
+            "version": "==13.8.0"
+        },
+        "s3transfer": {
+            "hashes": [
+                "sha256:0711534e9356d3cc692fdde846b4a1e4b0cb6519971860796e6bc4c7aea00ef6",
+                "sha256:eca1c20de70a39daee580aef4986996620f365c4e0fda6a86100231d62f1bf69"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.10.2"
         },
         "scikit-learn": {
             "hashes": [
@@ -1108,14 +1583,6 @@
             "markers": "python_version >= '3.9'",
             "version": "==1.11.4"
         },
-        "setuptools": {
-            "hashes": [
-                "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e",
-                "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"
-            ],
-            "markers": "python_version >= '3.8'",
-            "version": "==69.2.0"
-        },
         "shap": {
             "hashes": [
                 "sha256:1f19a3c531c6ceb0d7adea8262385580b52ef187981606342cd03288fc48cbfa",
@@ -1152,7 +1619,7 @@
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.16.0"
         },
         "slicer": {
@@ -1172,44 +1639,61 @@
         },
         "sphinx": {
             "hashes": [
-                "sha256:23c846a1841af998cb736218539bb86d16f5eb95f5760b1966abcd2d584e62b8",
-                "sha256:3d513088236eef51e5b0adb78b0492eb22cc3b8ccdb0b36dd021173b365d4454"
+                "sha256:464d9c1bd5613bcebe76b46658763f3f3dbb184da7406e632a84596d3cd8ee90",
+                "sha256:af248b21e3282f847ff20feebe7a1985fb34773cbe3fc75bf206897f1a2199c4"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.6'",
-            "version": "==4.1.1"
+            "version": "==5.0.0"
+        },
+        "sphinx-code-tabs": {
+            "hashes": [
+                "sha256:11d9d1fa7eb5f21c5b24a023e3522bbd36b456392a74a5597ca0adde30dc27e3",
+                "sha256:a17f387a37e09b9deb64c5335023baa0f1f5b47ff451fee4fc0f02ea46a0bed3"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==0.5.3"
+        },
+        "sphinx-gallery": {
+            "hashes": [
+                "sha256:953f32b0833b0a689ff33516d0866865fb8601c0626811b95d2e844286d207e4"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.7'",
+            "version": "==0.10.1"
         },
         "sphinx-rtd-theme": {
             "hashes": [
-                "sha256:32bd3b5d13dc8186d7a42fc816a23d32e83a4827d7d9882948e7b837c232da5a",
-                "sha256:4a05bdbe8b1446d77a01e20a23ebc6777c74f43237035e76be89699308987d6f"
+                "sha256:4d35a56f4508cfee4c4fb604373ede6feae2a306731d533f409ef5c3496fdbd8",
+                "sha256:eec6d497e4c2195fa0e8b2016b337532b8a699a68bcb22a512870e16925c6a5c"
             ],
             "index": "pypi",
-            "version": "==0.5.2"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.0.0"
         },
         "sphinxcontrib-applehelp": {
             "hashes": [
-                "sha256:c40a4f96f3776c4393d933412053962fac2b84f4c99a7982ba42e09576a70619",
-                "sha256:cb61eb0ec1b61f349e5cc36b2028e9e7ca765be05e49641c97241274753067b4"
+                "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1",
+                "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==1.0.8"
+            "version": "==2.0.0"
         },
         "sphinxcontrib-devhelp": {
             "hashes": [
-                "sha256:6485d09629944511c893fa11355bda18b742b83a2b181f9a009f7e500595c90f",
-                "sha256:9893fd3f90506bc4b97bdb977ceb8fbd823989f4316b28c3841ec128544372d3"
+                "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad",
+                "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==1.0.6"
+            "version": "==2.0.0"
         },
         "sphinxcontrib-htmlhelp": {
             "hashes": [
-                "sha256:0dc87637d5de53dd5eec3a6a01753b1ccf99494bd756aafecd74b4fa9e729015",
-                "sha256:393f04f112b4d2f53d93448d4bce35842f62b307ccdc549ec1585e950bc35e04"
+                "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8",
+                "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==2.0.5"
+            "version": "==2.1.0"
         },
         "sphinxcontrib-jsmath": {
             "hashes": [
@@ -1221,34 +1705,42 @@
         },
         "sphinxcontrib-qthelp": {
             "hashes": [
-                "sha256:053dedc38823a80a7209a80860b16b722e9e0209e32fea98c90e4e6624588ed6",
-                "sha256:e2ae3b5c492d58fcbd73281fbd27e34b8393ec34a073c792642cd8e529288182"
+                "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab",
+                "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==1.0.7"
+            "version": "==2.0.0"
         },
         "sphinxcontrib-serializinghtml": {
             "hashes": [
-                "sha256:326369b8df80a7d2d8d7f99aa5ac577f51ea51556ed974e7716cfd4fca3f6cb7",
-                "sha256:93f3f5dc458b91b192fe10c397e324f262cf163d79f3282c158e8436a2c4511f"
+                "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331",
+                "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==1.1.10"
+            "version": "==2.0.0"
+        },
+        "sympy": {
+            "hashes": [
+                "sha256:401449d84d07be9d0c7a46a64bd54fe097667d5e7181bfe67ec777be9e01cb13",
+                "sha256:c51d75517712f1aed280d4ce58506a4a88d635d6b5dd48b39102a7ae1f3fcfe9"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.13.2"
         },
         "threadpoolctl": {
             "hashes": [
-                "sha256:8f4c689a65b23e5ed825c8436a92b818aac005e0f3715f6a1664d7c7ee29d262",
-                "sha256:f11b491a03661d6dd7ef692dd422ab34185d982466c49c8f98c8f716b5c93196"
+                "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107",
+                "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.4.0"
+            "version": "==3.5.0"
         },
         "toml": {
             "hashes": [
                 "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
                 "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
             ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==0.10.2"
         },
         "tomli": {
@@ -1259,63 +1751,89 @@
             "markers": "python_version >= '3.7'",
             "version": "==2.0.1"
         },
+        "torch": {
+            "hashes": [
+                "sha256:092e7c2280c860eff762ac08c4bdcd53d701677851670695e0c22d6d345b269c",
+                "sha256:0b5f88afdfa05a335d80351e3cea57d38e578c8689f751d35e0ff36bce872113",
+                "sha256:18835374f599207a9e82c262153c20ddf42ea49bc76b6eadad8e5f49729f6e4d",
+                "sha256:362f82e23a4cd46341daabb76fba08f04cd646df9bfaf5da50af97cb60ca4971",
+                "sha256:40f6d3fe3bae74efcf08cb7f8295eaddd8a838ce89e9d26929d4edd6d5e4329d",
+                "sha256:5fc1d4d7ed265ef853579caf272686d1ed87cebdcd04f2a498f800ffc53dab71",
+                "sha256:6bce130f2cd2d52ba4e2c6ada461808de7e5eccbac692525337cfb4c19421846",
+                "sha256:72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d",
+                "sha256:91e326e2ccfb1496e3bee58f70ef605aeb27bd26be07ba64f37dcaac3d070ada",
+                "sha256:a38de2803ee6050309aac032676536c3d3b6a9804248537e38e098d0e14817ec",
+                "sha256:b57f07e92858db78c5b72857b4f0b33a65b00dc5d68e7948a8494b0314efb880",
+                "sha256:c9299c16c9743001ecef515536ac45900247f4338ecdf70746f2461f9e4831db",
+                "sha256:c99e1db4bf0c5347107845d715b4aa1097e601bdc36343d758963055e9599d93",
+                "sha256:d36a8ef100f5bff3e9c3cea934b9e0d7ea277cb8210c7152d34a9a6c5830eadd",
+                "sha256:ddddbd8b066e743934a4200b3d54267a46db02106876d21cf31f7da7a96f98ea",
+                "sha256:e8ac1985c3ff0f60d85b991954cfc2cc25f79c84545aead422763148ed2759e3",
+                "sha256:ebea70ff30544fc021d441ce6b219a88b67524f01170b1c538d7d3ebb5e7f56c",
+                "sha256:ef503165f2341942bfdf2bd520152f19540d0c0e34961232f134dc59ad435be8",
+                "sha256:f18197f3f7c15cde2115892b64f17c80dbf01ed72b008020e7da339902742cf6",
+                "sha256:fdc4fe11db3eb93c1115d3e973a27ac7c1a8318af8934ffa36b0370efe28e042"
+            ],
+            "markers": "python_full_version >= '3.8.0'",
+            "version": "==2.4.1"
+        },
         "tqdm": {
             "hashes": [
-                "sha256:1ee4f8a893eb9bef51c6e35730cebf234d5d0b6bd112b0271e10ed7c24a02bd9",
-                "sha256:6cd52cdf0fef0e0f543299cfc96fec90d7b8a7e88745f411ec33eb44d5ed3531"
+                "sha256:90279a3770753eafc9194a0364852159802111925aa30eb3f9d85b0e805ac7cd",
+                "sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.66.2"
+            "version": "==4.66.5"
         },
         "twine": {
             "hashes": [
-                "sha256:89b0cc7d370a4b66421cc6102f269aa910fe0f1861c124f573cf2ddedbc10cf4",
-                "sha256:a262933de0b484c53408f9edae2e7821c1c45a3314ff2df9bdd343aa7ab8edc0"
+                "sha256:215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997",
+                "sha256:9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==5.0.0"
+            "version": "==5.1.1"
         },
-        "tzdata": {
+        "typing-extensions": {
             "hashes": [
-                "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd",
-                "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"
+                "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d",
+                "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"
             ],
-            "markers": "python_version >= '2'",
-            "version": "==2024.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==4.12.2"
         },
         "urllib3": {
             "hashes": [
-                "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d",
-                "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"
+                "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472",
+                "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.2.1"
+            "version": "==2.2.2"
         },
         "virtualenv": {
             "hashes": [
-                "sha256:961c026ac520bac5f69acb8ea063e8a4f071bcc9457b9c1f28f6b085c511583a",
-                "sha256:e08e13ecdca7a0bd53798f356d5831434afa5b07b93f0abdf0797b7a06ffe197"
+                "sha256:48f2695d9809277003f30776d155615ffc11328e6a0a8c1f0ec80188d7874a55",
+                "sha256:c17f4e0f3e6036e9f26700446f85c76ab11df65ff6d8a9cbfad9f71aabfcf23c"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==20.25.1"
+            "version": "==20.26.4"
         },
         "xarray": {
             "hashes": [
-                "sha256:5c1db19efdde61db7faedad8fc944f4e29698fb6fbd578d352668b63598bd1d8",
-                "sha256:ca2bc4da2bf2e7879e15862a7a7c3fc76ad19f6a08931d030220cef39a29118d"
+                "sha256:7bee552751ff1b29dab8b7715726e5ecb56691ac54593cf4881dff41978ce0cd",
+                "sha256:7e530b1deafdd43e5c2b577d0944e6b528fbe88045fd849e49a8d11871ecd522"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.9'",
-            "version": "==2024.3.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==2023.1.0"
         },
         "zipp": {
             "hashes": [
-                "sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b",
-                "sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715"
+                "sha256:9960cd8967c8f85a56f920d5d507274e74f9ff813a0ab8889a5b5be2daf44064",
+                "sha256:c22b14cc4763c5a5b04134207736c107db42e9d3ef2d9779d465f5f1bcba572b"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.18.1"
+            "version": "==3.20.1"
         }
     },
     "extras": {
@@ -1329,19 +1847,19 @@
         },
         "attrs": {
             "hashes": [
-                "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30",
-                "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"
+                "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346",
+                "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==23.2.0"
+            "version": "==24.2.0"
         },
         "certifi": {
             "hashes": [
-                "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f",
-                "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"
+                "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8",
+                "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2024.2.2"
+            "version": "==2024.8.30"
         },
         "charset-normalizer": {
             "hashes": [
@@ -1465,11 +1983,11 @@
         },
         "filelock": {
             "hashes": [
-                "sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f",
-                "sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4"
+                "sha256:81de9eb8453c769b63369f87f11131a7ab04e367f8d97ad39dc230daa07e3bec",
+                "sha256:f6ed4c963184f4c84dd5557ce8fece759a3724b37b80c6c4f20a2f63a4dc6609"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.13.4"
+            "version": "==3.16.0"
         },
         "frozenlist": {
             "hashes": [
@@ -1556,27 +2074,27 @@
         },
         "idna": {
             "hashes": [
-                "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc",
-                "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"
+                "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac",
+                "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==3.7"
+            "markers": "python_version >= '3.6'",
+            "version": "==3.8"
         },
         "joblib": {
             "hashes": [
-                "sha256:1eb0dc091919cd384490de890cb5dfd538410a6d4b3b54eef09fb8c50b409b1c",
-                "sha256:42942470d4062537be4d54c83511186da1fc14ba354961a2114da91efa9a4ed7"
+                "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6",
+                "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.4.0"
+            "version": "==1.4.2"
         },
         "jsonschema": {
             "hashes": [
-                "sha256:7996507afae316306f9e2290407761157c6f78002dcf7419acb99822143d1c6f",
-                "sha256:85727c00279f5fa6bedbe6238d2aa6403bedd8b4864ab11207d07df3cc1b2ee5"
+                "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4",
+                "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.21.1"
+            "version": "==4.23.0"
         },
         "jsonschema-specifications": {
             "hashes": [
@@ -1588,30 +2106,30 @@
         },
         "llvmlite": {
             "hashes": [
-                "sha256:05cb7e9b6ce69165ce4d1b994fbdedca0c62492e537b0cc86141b6e2c78d5888",
-                "sha256:08fa9ab02b0d0179c688a4216b8939138266519aaa0aa94f1195a8542faedb56",
-                "sha256:3366938e1bf63d26c34fbfb4c8e8d2ded57d11e0567d5bb243d89aab1eb56098",
-                "sha256:43d65cc4e206c2e902c1004dd5418417c4efa6c1d04df05c6c5675a27e8ca90e",
-                "sha256:70f44ccc3c6220bd23e0ba698a63ec2a7d3205da0d848804807f37fc243e3f77",
-                "sha256:763f8d8717a9073b9e0246998de89929071d15b47f254c10eef2310b9aac033d",
-                "sha256:7e0c4c11c8c2aa9b0701f91b799cb9134a6a6de51444eff5a9087fc7c1384275",
-                "sha256:81e674c2fe85576e6c4474e8c7e7aba7901ac0196e864fe7985492b737dbab65",
-                "sha256:8d90edf400b4ceb3a0e776b6c6e4656d05c7187c439587e06f86afceb66d2be5",
-                "sha256:a78ab89f1924fc11482209f6799a7a3fc74ddc80425a7a3e0e8174af0e9e2301",
-                "sha256:ae511caed28beaf1252dbaf5f40e663f533b79ceb408c874c01754cafabb9cbf",
-                "sha256:b2fce7d355068494d1e42202c7aff25d50c462584233013eb4470c33b995e3ee",
-                "sha256:bb3975787f13eb97629052edb5017f6c170eebc1c14a0433e8089e5db43bcce6",
-                "sha256:bdd3888544538a94d7ec99e7c62a0cdd8833609c85f0c23fcb6c5c591aec60ad",
-                "sha256:c35da49666a21185d21b551fc3caf46a935d54d66969d32d72af109b5e7d2b6f",
-                "sha256:c5bece0cdf77f22379f19b1959ccd7aee518afa4afbd3656c6365865f84903f9",
-                "sha256:d0936c2067a67fb8816c908d5457d63eba3e2b17e515c5fe00e5ee2bace06040",
-                "sha256:d47494552559e00d81bfb836cf1c4d5a5062e54102cc5767d5aa1e77ccd2505c",
-                "sha256:d7599b65c7af7abbc978dbf345712c60fd596aa5670496561cc10e8a71cebfb2",
-                "sha256:ebe66a86dc44634b59a3bc860c7b20d26d9aaffcd30364ebe8ba79161a9121f4",
-                "sha256:f92b09243c0cc3f457da8b983f67bd8e1295d0f5b3746c7a1861d7a99403854a"
+                "sha256:14f0e4bf2fd2d9a75a3534111e8ebeb08eda2f33e9bdd6dfa13282afacdde0ed",
+                "sha256:18e9953c748b105668487b7c81a3e97b046d8abf95c4ddc0cd3c94f4e4651ae8",
+                "sha256:35d80d61d0cda2d767f72de99450766250560399edc309da16937b93d3b676e7",
+                "sha256:3e8d0618cb9bfe40ac38a9633f2493d4d4e9fcc2f438d39a4e854f39cc0f5f98",
+                "sha256:47e147cdda9037f94b399bf03bfd8a6b6b1f2f90be94a454e3386f006455a9b4",
+                "sha256:6912a87782acdff6eb8bf01675ed01d60ca1f2551f8176a300a886f09e836a6a",
+                "sha256:6d4fd101f571a31acb1559ae1af30f30b1dc4b3186669f92ad780e17c81e91bc",
+                "sha256:74937acd22dc11b33946b67dca7680e6d103d6e90eeaaaf932603bec6fe7b03a",
+                "sha256:7a2872ee80dcf6b5dbdc838763d26554c2a18aa833d31a2635bff16aafefb9c9",
+                "sha256:7d434ec7e2ce3cc8f452d1cd9a28591745de022f931d67be688a737320dfcead",
+                "sha256:977525a1e5f4059316b183fb4fd34fa858c9eade31f165427a3977c95e3ee749",
+                "sha256:9cd2a7376f7b3367019b664c21f0c61766219faa3b03731113ead75107f3b66c",
+                "sha256:a289af9a1687c6cf463478f0fa8e8aa3b6fb813317b0d70bf1ed0759eab6f761",
+                "sha256:ae2b5b5c3ef67354824fb75517c8db5fbe93bc02cd9671f3c62271626bc041d5",
+                "sha256:bc9efc739cc6ed760f795806f67889923f7274276f0eb45092a1473e40d9b867",
+                "sha256:c1da416ab53e4f7f3bc8d4eeba36d801cc1894b9fbfbf2022b29b6bad34a7df2",
+                "sha256:d5bd550001d26450bd90777736c69d68c487d17bf371438f975229b2b8241a91",
+                "sha256:df6509e1507ca0760787a199d19439cc887bfd82226f5af746d6977bd9f66844",
+                "sha256:e0a9a1a39d4bf3517f2af9d23d479b4175ead205c592ceeb8b89af48a327ea57",
+                "sha256:eccce86bba940bae0d8d48ed925f21dbb813519169246e2ab292b5092aba121f",
+                "sha256:f99b600aa7f65235a5a05d0b9a9f31150c390f31261f2a0ba678e26823ec38f7"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==0.42.0"
+            "version": "==0.43.0"
         },
         "msgpack": {
             "hashes": [
@@ -1677,30 +2195,30 @@
         },
         "numba": {
             "hashes": [
-                "sha256:0594b3dfb369fada1f8bb2e3045cd6c61a564c62e50cf1f86b4666bc721b3450",
-                "sha256:0b77aecf52040de2a1eb1d7e314497b9e56fba17466c80b457b971a25bb1576d",
-                "sha256:0f68589740a8c38bb7dc1b938b55d1145244c8353078eea23895d4f82c8b9ec1",
-                "sha256:1cce206a3b92836cdf26ef39d3a3242fec25e07f020cc4feec4c4a865e340569",
-                "sha256:2801003caa263d1e8497fb84829a7ecfb61738a95f62bc05693fcf1733e978e4",
-                "sha256:3476a4f641bfd58f35ead42f4dcaf5f132569c4647c6f1360ccf18ee4cda3990",
-                "sha256:411df625372c77959570050e861981e9d196cc1da9aa62c3d6a836b5cc338966",
-                "sha256:43727e7ad20b3ec23ee4fc642f5b61845c71f75dd2825b3c234390c6d8d64051",
-                "sha256:4e0318ae729de6e5dbe64c75ead1a95eb01fabfe0e2ebed81ebf0344d32db0ae",
-                "sha256:525ef3f820931bdae95ee5379c670d5c97289c6520726bc6937a4a7d4230ba24",
-                "sha256:5bf68f4d69dd3a9f26a9b23548fa23e3bcb9042e2935257b471d2a8d3c424b7f",
-                "sha256:649913a3758891c77c32e2d2a3bcbedf4a69f5fea276d11f9119677c45a422e8",
-                "sha256:76f69132b96028d2774ed20415e8c528a34e3299a40581bae178f0994a2f370b",
-                "sha256:7d80bce4ef7e65bf895c29e3889ca75a29ee01da80266a01d34815918e365835",
-                "sha256:8c8b4477763cb1fbd86a3be7050500229417bf60867c93e131fd2626edb02238",
-                "sha256:8d51ccd7008a83105ad6a0082b6a2b70f1142dc7cfd76deb8c5a862367eb8c86",
-                "sha256:9712808e4545270291d76b9a264839ac878c5eb7d8b6e02c970dc0ac29bc8187",
-                "sha256:97385a7f12212c4f4bc28f648720a92514bee79d7063e40ef66c2d30600fd18e",
-                "sha256:990e395e44d192a12105eca3083b61307db7da10e093972ca285c85bef0963d6",
-                "sha256:dd2842fac03be4e5324ebbbd4d2d0c8c0fc6e0df75c09477dd45b288a0777389",
-                "sha256:f7ad1d217773e89a9845886401eaaab0a156a90aa2f179fdc125261fd1105096"
+                "sha256:01ef4cd7d83abe087d644eaa3d95831b777aa21d441a23703d649e06b8e06b74",
+                "sha256:0b983bd6ad82fe868493012487f34eae8bf7dd94654951404114f23c3466d34b",
+                "sha256:0ebaa91538e996f708f1ab30ef4d3ddc344b64b5227b67a57aa74f401bb68b9d",
+                "sha256:1527dc578b95c7c4ff248792ec33d097ba6bef9eda466c948b68dfc995c25781",
+                "sha256:159e618ef213fba758837f9837fb402bbe65326e60ba0633dbe6c7f274d42c1b",
+                "sha256:19407ced081d7e2e4b8d8c36aa57b7452e0283871c296e12d798852bc7d7f198",
+                "sha256:3031547a015710140e8c87226b4cfe927cac199835e5bf7d4fe5cb64e814e3ab",
+                "sha256:38d6ea4c1f56417076ecf8fc327c831ae793282e0ff51080c5094cb726507b1c",
+                "sha256:3fb02b344a2a80efa6f677aa5c40cd5dd452e1b35f8d1c2af0dfd9ada9978e4b",
+                "sha256:4142d7ac0210cc86432b818338a2bc368dc773a2f5cf1e32ff7c5b378bd63ee8",
+                "sha256:5d761de835cd38fb400d2c26bb103a2726f548dc30368853121d66201672e651",
+                "sha256:5df6158e5584eece5fc83294b949fd30b9f1125df7708862205217e068aabf16",
+                "sha256:5f4fde652ea604ea3c86508a3fb31556a6157b2c76c8b51b1d45eb40c8598703",
+                "sha256:62908d29fb6a3229c242e981ca27e32a6e606cc253fc9e8faeb0e48760de241e",
+                "sha256:819a3dfd4630d95fd574036f99e47212a1af41cbcb019bf8afac63ff56834449",
+                "sha256:a17b70fc9e380ee29c42717e8cc0bfaa5556c416d94f9aa96ba13acb41bdece8",
+                "sha256:c151748cd269ddeab66334bd754817ffc0cabd9433acb0f551697e5151917d25",
+                "sha256:cac02c041e9b5bc8cf8f2034ff6f0dbafccd1ae9590dc146b3a02a45e53af4e2",
+                "sha256:d7da4098db31182fc5ffe4bc42c6f24cd7d1cb8a14b59fd755bfee32e34b8404",
+                "sha256:f75262e8fe7fa96db1dca93d53a194a38c46da28b112b8a4aca168f0df860347",
+                "sha256:fe0b28abb8d70f8160798f4de9d486143200f34458d34c4a214114e445d7124e"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==0.59.1"
+            "version": "==0.60.0"
         },
         "numpy": {
             "hashes": [
@@ -1747,11 +2265,11 @@
         },
         "packaging": {
             "hashes": [
-                "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5",
-                "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"
+                "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002",
+                "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==24.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==24.1"
         },
         "pandas": {
             "hashes": [
@@ -1773,11 +2291,13 @@
                 "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24",
                 "sha256:8e5a0b00e1e56a842f922e7fae8ae4077aee4af0acb5ae3622bd4b4c30aedf99",
                 "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151",
+                "sha256:9057e6aa78a584bc93a13f0a9bf7e753a5e9770a30b4d758b8d5f2a62a9433cd",
                 "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce",
                 "sha256:92fd6b027924a7e178ac202cfbe25e53368db90d56872d20ffae94b96c7acc57",
                 "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef",
                 "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54",
                 "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a",
+                "sha256:c7adfc142dac335d8c1e0dcbd37eb8617eac386596eb9e1a1b77791cf2498238",
                 "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23",
                 "sha256:ddf818e4e6c7c6f4f7c8a12709696d193976b591cc7dc50588d3d1a6b5dc8772",
                 "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce",
@@ -1788,27 +2308,27 @@
         },
         "protobuf": {
             "hashes": [
-                "sha256:38aa5f535721d5bb99861166c445c4105c4e285c765fbb2ac10f116e32dcd46d",
-                "sha256:3c388ea6ddfe735f8cf69e3f7dc7611e73107b60bdfcf5d0f024c3ccd3794e23",
-                "sha256:7ee014c2c87582e101d6b54260af03b6596728505c79f17c8586e7523aaa8f8c",
-                "sha256:8ca2a1d97c290ec7b16e4e5dff2e5ae150cc1582f55b5ab300d45cb0dfa90e51",
-                "sha256:9b557c317ebe6836835ec4ef74ec3e994ad0894ea424314ad3552bc6e8835b4e",
-                "sha256:b9ba3ca83c2e31219ffbeb9d76b63aad35a3eb1544170c55336993d7a18ae72c",
-                "sha256:d693d2504ca96750d92d9de8a103102dd648fda04540495535f0fec7577ed8fc",
-                "sha256:da612f2720c0183417194eeaa2523215c4fcc1a1949772dc65f05047e08d5932",
-                "sha256:e6039957449cb918f331d32ffafa8eb9255769c96aa0560d9a5bf0b4e00a2a33",
-                "sha256:f7417703f841167e5a27d48be13389d52ad705ec09eade63dfc3180a959215d7",
-                "sha256:fbfe61e7ee8c1860855696e3ac6cfd1b01af5498facc6834fcc345c9684fb2ca"
+                "sha256:018db9056b9d75eb93d12a9d35120f97a84d9a919bcab11ed56ad2d399d6e8dd",
+                "sha256:510ed78cd0980f6d3218099e874714cdf0d8a95582e7b059b06cabad855ed0a0",
+                "sha256:532627e8fdd825cf8767a2d2b94d77e874d5ddb0adefb04b237f7cc296748681",
+                "sha256:6206afcb2d90181ae8722798dcb56dc76675ab67458ac24c0dd7d75d632ac9bd",
+                "sha256:66c3edeedb774a3508ae70d87b3a19786445fe9a068dd3585e0cefa8a77b83d0",
+                "sha256:6d7cc9e60f976cf3e873acb9a40fed04afb5d224608ed5c1a105db4a3f09c5b6",
+                "sha256:853db610214e77ee817ecf0514e0d1d052dff7f63a0c157aa6eabae98db8a8de",
+                "sha256:d001a73c8bc2bf5b5c1360d59dd7573744e163b3607fa92788b7f3d5fefbd9a5",
+                "sha256:dde74af0fa774fa98892209992295adbfb91da3fa98c8f67a88afe8f5a349add",
+                "sha256:dde9fcaa24e7a9654f4baf2a55250b13a5ea701493d904c54069776b99a8216b",
+                "sha256:eef7a8a2f4318e2cb2dee8666d26e58eaf437c14788f3a2911d0c3da40405ae8"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.26.1"
+            "version": "==5.28.0"
         },
         "python-dateutil": {
             "hashes": [
                 "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3",
                 "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.9.0.post0"
         },
         "pytz": {
@@ -1820,208 +2340,214 @@
         },
         "pyyaml": {
             "hashes": [
-                "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5",
-                "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc",
-                "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df",
-                "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741",
-                "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206",
-                "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27",
-                "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595",
-                "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62",
-                "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98",
-                "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696",
-                "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290",
-                "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9",
-                "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d",
-                "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6",
-                "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867",
-                "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47",
-                "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486",
-                "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6",
-                "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3",
-                "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007",
-                "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938",
-                "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0",
-                "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c",
-                "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735",
-                "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d",
-                "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28",
-                "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4",
-                "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
-                "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
-                "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef",
-                "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
-                "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd",
-                "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3",
-                "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0",
-                "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515",
-                "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c",
-                "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c",
-                "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924",
-                "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34",
-                "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43",
-                "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859",
-                "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673",
-                "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54",
-                "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a",
-                "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b",
-                "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab",
-                "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa",
-                "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c",
-                "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585",
-                "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d",
-                "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"
+                "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff",
+                "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48",
+                "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086",
+                "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e",
+                "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133",
+                "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5",
+                "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484",
+                "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee",
+                "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5",
+                "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68",
+                "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a",
+                "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf",
+                "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99",
+                "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8",
+                "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85",
+                "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19",
+                "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc",
+                "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a",
+                "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1",
+                "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317",
+                "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c",
+                "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631",
+                "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d",
+                "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652",
+                "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5",
+                "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e",
+                "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b",
+                "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8",
+                "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476",
+                "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706",
+                "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563",
+                "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237",
+                "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b",
+                "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083",
+                "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180",
+                "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425",
+                "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e",
+                "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f",
+                "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725",
+                "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183",
+                "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab",
+                "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774",
+                "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725",
+                "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e",
+                "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5",
+                "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d",
+                "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290",
+                "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44",
+                "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed",
+                "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4",
+                "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba",
+                "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12",
+                "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==6.0.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==6.0.2"
         },
         "ray": {
             "hashes": [
-                "sha256:31aa60373fc7291752ee89a5f5ad8effec682b1f165911f38ae95fc43bc668a9",
-                "sha256:32d97e5343578a3d37ab5f30148fa193dec46a21fa21f15b6f23fe48a420831a",
-                "sha256:44ab600fe0b5a12675d0d42d564994ac4e53286217c4de1c4eb00d74ae79ef24",
-                "sha256:5b4ec4b5707e18382685d0703ed04afd1602359a3056f6ae4b37588a0551eef3",
-                "sha256:5b7d41eb04f6b67c38170edc0406dc71537eabfd6e5d4e3399a36385ff8b0194",
-                "sha256:5fe8fb8847304dd3a6e435b95af9e5436309f2b3612c63c56bf4ac8dea73f9f4",
-                "sha256:6b49a8c2b40f02a56a2af2b6026c1eedd485747c6e4c2cf9ac433af6e572bdbb",
-                "sha256:77ba4120d694e7c3dc7d93a9d3cb33925827d04ad11af2d21fa0db66f227d27a",
-                "sha256:8a174268c7b6ca9826e4884b837395b695a45c17049927965d1b4cc370184ba2",
-                "sha256:8a44535e6266fa09e3eb4fc9035906decfc9f3aeda86fe66b1e738a01a51939a",
-                "sha256:8eb11aec8a65946f7546d0e703158c03a85a8be27332dbbf86d9411802700e7e",
-                "sha256:8eb606b7d247213b377ccca0f8d425f9c61a48b23e9b2e4566bc75f66d797bb5",
-                "sha256:917d081fc98500f244ebc0e8da836025e1e4fa52f21030b8336cb0a2c79e84e2",
-                "sha256:a3db89d22afc7a0a976249715dd90ffe69f7692d32cb599cd1afbc38482060f7",
-                "sha256:c193deed7e3f604cdb37047f5646cab14f4337693dd32add8bc902dfadb89f75",
-                "sha256:c7d1438cba8726ec9a59c96964e007b60a0728436647f48c383228692c2f2ee0",
-                "sha256:cb74f7d2aa5a21e5f9dcb315a4f9bde822328e76ba95cd0ba370cfda098a67f4",
-                "sha256:eceecea4133e63f5d607cc9f2a4278de51eeeeef552f694895e381aae9ff8522",
-                "sha256:f215eb704f2cb72e984d5a85fe435b4d74808c906950176789ba2101ce739082",
-                "sha256:fb92f2d6d4eca602dfb0d3d459a09be59668e1560ce4bd89b692892f25b1933b"
+                "sha256:1994aaf9996ffc45019856545e817d527ad572762f1af76ad669ae4e786fcfd6",
+                "sha256:1e7e2d2e987be728a81821b6fd2bccb23e4d8a6cca8417db08b24f06a08d8476",
+                "sha256:2ca1a0de41d4462fd764598a5981cf55fc955599f38f9a1ae10868e94c6dd80d",
+                "sha256:4e6314bfdb8c73abcac13f41cc3d935dd1a8ad94c65005a4bfdc4861dc8b070d",
+                "sha256:587af570cbe5f6cedca854f15107740e63c67207bee900713cb2ee38f6ebf20f",
+                "sha256:5e98d2bac394b806109782f316740c5b3c3f10a50117c8e28200a528df734928",
+                "sha256:70a154e3071cbb4d7a9b68f2dcf491b96b760be0ec6e2ef11a766071ac6acfef",
+                "sha256:7b746913268d5ea5e19bff0eb6bdc7e0538036892a8b57c08411787481195df2",
+                "sha256:8bd48be4c362004d31e5df072fd58b929efc67adfefc0adece41483b15f84539",
+                "sha256:8e406cce41679790146d4d2b1b0cb0b413ca35276e43b68ee796366169c1dbde",
+                "sha256:ac561e20a62ce941b74d02a0b92b7765c6ba87cc22e24f34f64ded2c454ba64e",
+                "sha256:c395b46efd0dd871424b1b8d6baf99f91983946fbe351ff66ea34e8919daff29",
+                "sha256:c5600f745bb0e4df840a5cd51e82b1acf517f73505df9869fe3e369966956129",
+                "sha256:d3b7a7d73f818e249064460ffa95402ebd852bf97d9ec6167b8b0d95be03da9f",
+                "sha256:d7a606c8ca53c64fc496703e9fd15d1a1ffb50e6b457a33d3622be2f13fc30a5",
+                "sha256:dd8bdf9d16989684486db9ebcd23679140e2d6769fcdaadc05e8cac6b373023e",
+                "sha256:e29754fac4b69a9cb0d089841af59ec6fb10b5d4a248b7c579d319ca2ed1c96f",
+                "sha256:e2ccfd144180f03d38b02a81afdac2b437f27e46736bf2653a1f0e8d67ea56cd",
+                "sha256:eb86355a3a0e794e2f1dbd5a84805dddfca64921ad0999b7fa5276e40d243692",
+                "sha256:ef41e9254f3e18a90a8cf13fac9e35ac086eb778079ab6c76a37d3a6059186c5"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==2.10.0"
+            "version": "==2.35.0"
         },
         "referencing": {
             "hashes": [
-                "sha256:5773bd84ef41799a5a8ca72dc34590c041eb01bf9aa02632b4a973fb0181a844",
-                "sha256:d53ae300ceddd3169f1ffa9caf2cb7b769e92657e4fafb23d34b93679116dfd4"
+                "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c",
+                "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.34.0"
+            "version": "==0.35.1"
         },
         "requests": {
             "hashes": [
-                "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
-                "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
+                "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760",
+                "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.31.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.32.3"
         },
         "rpds-py": {
             "hashes": [
-                "sha256:01e36a39af54a30f28b73096dd39b6802eddd04c90dbe161c1b8dbe22353189f",
-                "sha256:044a3e61a7c2dafacae99d1e722cc2d4c05280790ec5a05031b3876809d89a5c",
-                "sha256:08231ac30a842bd04daabc4d71fddd7e6d26189406d5a69535638e4dcb88fe76",
-                "sha256:08f9ad53c3f31dfb4baa00da22f1e862900f45908383c062c27628754af2e88e",
-                "sha256:0ab39c1ba9023914297dd88ec3b3b3c3f33671baeb6acf82ad7ce883f6e8e157",
-                "sha256:0af039631b6de0397ab2ba16eaf2872e9f8fca391b44d3d8cac317860a700a3f",
-                "sha256:0b8612cd233543a3781bc659c731b9d607de65890085098986dfd573fc2befe5",
-                "sha256:11a8c85ef4a07a7638180bf04fe189d12757c696eb41f310d2426895356dcf05",
-                "sha256:1374f4129f9bcca53a1bba0bb86bf78325a0374577cf7e9e4cd046b1e6f20e24",
-                "sha256:1d4acf42190d449d5e89654d5c1ed3a4f17925eec71f05e2a41414689cda02d1",
-                "sha256:1d9a5be316c15ffb2b3c405c4ff14448c36b4435be062a7f578ccd8b01f0c4d8",
-                "sha256:1df3659d26f539ac74fb3b0c481cdf9d725386e3552c6fa2974f4d33d78e544b",
-                "sha256:22806714311a69fd0af9b35b7be97c18a0fc2826e6827dbb3a8c94eac6cf7eeb",
-                "sha256:2644e47de560eb7bd55c20fc59f6daa04682655c58d08185a9b95c1970fa1e07",
-                "sha256:2e6d75ab12b0bbab7215e5d40f1e5b738aa539598db27ef83b2ec46747df90e1",
-                "sha256:30f43887bbae0d49113cbaab729a112251a940e9b274536613097ab8b4899cf6",
-                "sha256:34b18ba135c687f4dac449aa5157d36e2cbb7c03cbea4ddbd88604e076aa836e",
-                "sha256:36b3ee798c58ace201289024b52788161e1ea133e4ac93fba7d49da5fec0ef9e",
-                "sha256:39514da80f971362f9267c600b6d459bfbbc549cffc2cef8e47474fddc9b45b1",
-                "sha256:39f5441553f1c2aed4de4377178ad8ff8f9d733723d6c66d983d75341de265ab",
-                "sha256:3a96e0c6a41dcdba3a0a581bbf6c44bb863f27c541547fb4b9711fd8cf0ffad4",
-                "sha256:3f26b5bd1079acdb0c7a5645e350fe54d16b17bfc5e71f371c449383d3342e17",
-                "sha256:41ef53e7c58aa4ef281da975f62c258950f54b76ec8e45941e93a3d1d8580594",
-                "sha256:42821446ee7a76f5d9f71f9e33a4fb2ffd724bb3e7f93386150b61a43115788d",
-                "sha256:43fbac5f22e25bee1d482c97474f930a353542855f05c1161fd804c9dc74a09d",
-                "sha256:4457a94da0d5c53dc4b3e4de1158bdab077db23c53232f37a3cb7afdb053a4e3",
-                "sha256:465a3eb5659338cf2a9243e50ad9b2296fa15061736d6e26240e713522b6235c",
-                "sha256:482103aed1dfe2f3b71a58eff35ba105289b8d862551ea576bd15479aba01f66",
-                "sha256:4832d7d380477521a8c1644bbab6588dfedea5e30a7d967b5fb75977c45fd77f",
-                "sha256:4901165d170a5fde6f589acb90a6b33629ad1ec976d4529e769c6f3d885e3e80",
-                "sha256:5307def11a35f5ae4581a0b658b0af8178c65c530e94893345bebf41cc139d33",
-                "sha256:5417558f6887e9b6b65b4527232553c139b57ec42c64570569b155262ac0754f",
-                "sha256:56a737287efecafc16f6d067c2ea0117abadcd078d58721f967952db329a3e5c",
-                "sha256:586f8204935b9ec884500498ccc91aa869fc652c40c093bd9e1471fbcc25c022",
-                "sha256:5b4e7d8d6c9b2e8ee2d55c90b59c707ca59bc30058269b3db7b1f8df5763557e",
-                "sha256:5ddcba87675b6d509139d1b521e0c8250e967e63b5909a7e8f8944d0f90ff36f",
-                "sha256:618a3d6cae6ef8ec88bb76dd80b83cfe415ad4f1d942ca2a903bf6b6ff97a2da",
-                "sha256:635dc434ff724b178cb192c70016cc0ad25a275228f749ee0daf0eddbc8183b1",
-                "sha256:661d25cbffaf8cc42e971dd570d87cb29a665f49f4abe1f9e76be9a5182c4688",
-                "sha256:66e6a3af5a75363d2c9a48b07cb27c4ea542938b1a2e93b15a503cdfa8490795",
-                "sha256:67071a6171e92b6da534b8ae326505f7c18022c6f19072a81dcf40db2638767c",
-                "sha256:685537e07897f173abcf67258bee3c05c374fa6fff89d4c7e42fb391b0605e98",
-                "sha256:69e64831e22a6b377772e7fb337533c365085b31619005802a79242fee620bc1",
-                "sha256:6b0817e34942b2ca527b0e9298373e7cc75f429e8da2055607f4931fded23e20",
-                "sha256:6c81e5f372cd0dc5dc4809553d34f832f60a46034a5f187756d9b90586c2c307",
-                "sha256:6d7faa6f14017c0b1e69f5e2c357b998731ea75a442ab3841c0dbbbfe902d2c4",
-                "sha256:6ef0befbb5d79cf32d0266f5cff01545602344eda89480e1dd88aca964260b18",
-                "sha256:6ef687afab047554a2d366e112dd187b62d261d49eb79b77e386f94644363294",
-                "sha256:7223a2a5fe0d217e60a60cdae28d6949140dde9c3bcc714063c5b463065e3d66",
-                "sha256:77f195baa60a54ef9d2de16fbbfd3ff8b04edc0c0140a761b56c267ac11aa467",
-                "sha256:793968759cd0d96cac1e367afd70c235867831983f876a53389ad869b043c948",
-                "sha256:7bd339195d84439cbe5771546fe8a4e8a7a045417d8f9de9a368c434e42a721e",
-                "sha256:7cd863afe7336c62ec78d7d1349a2f34c007a3cc6c2369d667c65aeec412a5b1",
-                "sha256:7f2facbd386dd60cbbf1a794181e6aa0bd429bd78bfdf775436020172e2a23f0",
-                "sha256:84ffab12db93b5f6bad84c712c92060a2d321b35c3c9960b43d08d0f639d60d7",
-                "sha256:8c8370641f1a7f0e0669ddccca22f1da893cef7628396431eb445d46d893e5cd",
-                "sha256:8db715ebe3bb7d86d77ac1826f7d67ec11a70dbd2376b7cc214199360517b641",
-                "sha256:8e8916ae4c720529e18afa0b879473049e95949bf97042e938530e072fde061d",
-                "sha256:8f03bccbd8586e9dd37219bce4d4e0d3ab492e6b3b533e973fa08a112cb2ffc9",
-                "sha256:8f2fc11e8fe034ee3c34d316d0ad8808f45bc3b9ce5857ff29d513f3ff2923a1",
-                "sha256:923d39efa3cfb7279a0327e337a7958bff00cc447fd07a25cddb0a1cc9a6d2da",
-                "sha256:93df1de2f7f7239dc9cc5a4a12408ee1598725036bd2dedadc14d94525192fc3",
-                "sha256:998e33ad22dc7ec7e030b3df701c43630b5bc0d8fbc2267653577e3fec279afa",
-                "sha256:99f70b740dc04d09e6b2699b675874367885217a2e9f782bdf5395632ac663b7",
-                "sha256:9a00312dea9310d4cb7dbd7787e722d2e86a95c2db92fbd7d0155f97127bcb40",
-                "sha256:9d54553c1136b50fd12cc17e5b11ad07374c316df307e4cfd6441bea5fb68496",
-                "sha256:9dbbeb27f4e70bfd9eec1be5477517365afe05a9b2c441a0b21929ee61048124",
-                "sha256:a1ce3ba137ed54f83e56fb983a5859a27d43a40188ba798993812fed73c70836",
-                "sha256:a34d557a42aa28bd5c48a023c570219ba2593bcbbb8dc1b98d8cf5d529ab1434",
-                "sha256:a5f446dd5055667aabaee78487f2b5ab72e244f9bc0b2ffebfeec79051679984",
-                "sha256:ad36cfb355e24f1bd37cac88c112cd7730873f20fb0bdaf8ba59eedf8216079f",
-                "sha256:aec493917dd45e3c69d00a8874e7cbed844efd935595ef78a0f25f14312e33c6",
-                "sha256:b316144e85316da2723f9d8dc75bada12fa58489a527091fa1d5a612643d1a0e",
-                "sha256:b34ae4636dfc4e76a438ab826a0d1eed2589ca7d9a1b2d5bb546978ac6485461",
-                "sha256:b34b7aa8b261c1dbf7720b5d6f01f38243e9b9daf7e6b8bc1fd4657000062f2c",
-                "sha256:bc362ee4e314870a70f4ae88772d72d877246537d9f8cb8f7eacf10884862432",
-                "sha256:bed88b9a458e354014d662d47e7a5baafd7ff81c780fd91584a10d6ec842cb73",
-                "sha256:c0013fe6b46aa496a6749c77e00a3eb07952832ad6166bd481c74bda0dcb6d58",
-                "sha256:c0b5dcf9193625afd8ecc92312d6ed78781c46ecbf39af9ad4681fc9f464af88",
-                "sha256:c4325ff0442a12113a6379af66978c3fe562f846763287ef66bdc1d57925d337",
-                "sha256:c463ed05f9dfb9baebef68048aed8dcdc94411e4bf3d33a39ba97e271624f8f7",
-                "sha256:c8362467a0fdeccd47935f22c256bec5e6abe543bf0d66e3d3d57a8fb5731863",
-                "sha256:cd5bf1af8efe569654bbef5a3e0a56eca45f87cfcffab31dd8dde70da5982475",
-                "sha256:cf1ea2e34868f6fbf070e1af291c8180480310173de0b0c43fc38a02929fc0e3",
-                "sha256:d62dec4976954a23d7f91f2f4530852b0c7608116c257833922a896101336c51",
-                "sha256:d68c93e381010662ab873fea609bf6c0f428b6d0bb00f2c6939782e0818d37bf",
-                "sha256:d7c36232a90d4755b720fbd76739d8891732b18cf240a9c645d75f00639a9024",
-                "sha256:dd18772815d5f008fa03d2b9a681ae38d5ae9f0e599f7dda233c439fcaa00d40",
-                "sha256:ddc2f4dfd396c7bfa18e6ce371cba60e4cf9d2e5cdb71376aa2da264605b60b9",
-                "sha256:e003b002ec72c8d5a3e3da2989c7d6065b47d9eaa70cd8808b5384fbb970f4ec",
-                "sha256:e32a92116d4f2a80b629778280103d2a510a5b3f6314ceccd6e38006b5e92dcb",
-                "sha256:e4461d0f003a0aa9be2bdd1b798a041f177189c1a0f7619fe8c95ad08d9a45d7",
-                "sha256:e541ec6f2ec456934fd279a3120f856cd0aedd209fc3852eca563f81738f6861",
-                "sha256:e546e768d08ad55b20b11dbb78a745151acbd938f8f00d0cfbabe8b0199b9880",
-                "sha256:ea7d4a99f3b38c37eac212dbd6ec42b7a5ec51e2c74b5d3223e43c811609e65f",
-                "sha256:ed4eb745efbff0a8e9587d22a84be94a5eb7d2d99c02dacf7bd0911713ed14dd",
-                "sha256:f8a2f084546cc59ea99fda8e070be2fd140c3092dc11524a71aa8f0f3d5a55ca",
-                "sha256:fcb25daa9219b4cf3a0ab24b0eb9a5cc8949ed4dc72acb8fa16b7e1681aa3c58",
-                "sha256:fdea4952db2793c4ad0bdccd27c1d8fdd1423a92f04598bc39425bcc2b8ee46e"
+                "sha256:06db23d43f26478303e954c34c75182356ca9aa7797d22c5345b16871ab9c45c",
+                "sha256:0e13e6952ef264c40587d510ad676a988df19adea20444c2b295e536457bc585",
+                "sha256:11ef6ce74616342888b69878d45e9f779b95d4bd48b382a229fe624a409b72c5",
+                "sha256:1259c7b3705ac0a0bd38197565a5d603218591d3f6cee6e614e380b6ba61c6f6",
+                "sha256:18d7585c463087bddcfa74c2ba267339f14f2515158ac4db30b1f9cbdb62c8ef",
+                "sha256:1e0f80b739e5a8f54837be5d5c924483996b603d5502bfff79bf33da06164ee2",
+                "sha256:1e5f3cd7397c8f86c8cc72d5a791071431c108edd79872cdd96e00abd8497d29",
+                "sha256:220002c1b846db9afd83371d08d239fdc865e8f8c5795bbaec20916a76db3318",
+                "sha256:22e6c9976e38f4d8c4a63bd8a8edac5307dffd3ee7e6026d97f3cc3a2dc02a0b",
+                "sha256:238a2d5b1cad28cdc6ed15faf93a998336eb041c4e440dd7f902528b8891b399",
+                "sha256:2580b0c34583b85efec8c5c5ec9edf2dfe817330cc882ee972ae650e7b5ef739",
+                "sha256:28527c685f237c05445efec62426d285e47a58fb05ba0090a4340b73ecda6dee",
+                "sha256:2cf126d33a91ee6eedc7f3197b53e87a2acdac63602c0f03a02dd69e4b138174",
+                "sha256:338ca4539aad4ce70a656e5187a3a31c5204f261aef9f6ab50e50bcdffaf050a",
+                "sha256:39ed0d010457a78f54090fafb5d108501b5aa5604cc22408fc1c0c77eac14344",
+                "sha256:3ad0fda1635f8439cde85c700f964b23ed5fc2d28016b32b9ee5fe30da5c84e2",
+                "sha256:3d2b1ad682a3dfda2a4e8ad8572f3100f95fad98cb99faf37ff0ddfe9cbf9d03",
+                "sha256:3d61339e9f84a3f0767b1995adfb171a0d00a1185192718a17af6e124728e0f5",
+                "sha256:3fde368e9140312b6e8b6c09fb9f8c8c2f00999d1823403ae90cc00480221b22",
+                "sha256:40ce74fc86ee4645d0a225498d091d8bc61f39b709ebef8204cb8b5a464d3c0e",
+                "sha256:49a8063ea4296b3a7e81a5dfb8f7b2d73f0b1c20c2af401fb0cdf22e14711a96",
+                "sha256:4a1f1d51eccb7e6c32ae89243cb352389228ea62f89cd80823ea7dd1b98e0b91",
+                "sha256:4b16aa0107ecb512b568244ef461f27697164d9a68d8b35090e9b0c1c8b27752",
+                "sha256:4f1ed4749a08379555cebf4650453f14452eaa9c43d0a95c49db50c18b7da075",
+                "sha256:4fe84294c7019456e56d93e8ababdad5a329cd25975be749c3f5f558abb48253",
+                "sha256:50eccbf054e62a7b2209b28dc7a22d6254860209d6753e6b78cfaeb0075d7bee",
+                "sha256:514b3293b64187172bc77c8fb0cdae26981618021053b30d8371c3a902d4d5ad",
+                "sha256:54b43a2b07db18314669092bb2de584524d1ef414588780261e31e85846c26a5",
+                "sha256:55fea87029cded5df854ca7e192ec7bdb7ecd1d9a3f63d5c4eb09148acf4a7ce",
+                "sha256:569b3ea770c2717b730b61998b6c54996adee3cef69fc28d444f3e7920313cf7",
+                "sha256:56e27147a5a4c2c21633ff8475d185734c0e4befd1c989b5b95a5d0db699b21b",
+                "sha256:57eb94a8c16ab08fef6404301c38318e2c5a32216bf5de453e2714c964c125c8",
+                "sha256:5a35df9f5548fd79cb2f52d27182108c3e6641a4feb0f39067911bf2adaa3e57",
+                "sha256:5a8c94dad2e45324fc74dce25e1645d4d14df9a4e54a30fa0ae8bad9a63928e3",
+                "sha256:5b4f105deeffa28bbcdff6c49b34e74903139afa690e35d2d9e3c2c2fba18cec",
+                "sha256:5c1dc0f53856b9cc9a0ccca0a7cc61d3d20a7088201c0937f3f4048c1718a209",
+                "sha256:614fdafe9f5f19c63ea02817fa4861c606a59a604a77c8cdef5aa01d28b97921",
+                "sha256:617c7357272c67696fd052811e352ac54ed1d9b49ab370261a80d3b6ce385045",
+                "sha256:65794e4048ee837494aea3c21a28ad5fc080994dfba5b036cf84de37f7ad5074",
+                "sha256:6632f2d04f15d1bd6fe0eedd3b86d9061b836ddca4c03d5cf5c7e9e6b7c14580",
+                "sha256:6c8ef2ebf76df43f5750b46851ed1cdf8f109d7787ca40035fe19fbdc1acc5a7",
+                "sha256:758406267907b3781beee0f0edfe4a179fbd97c0be2e9b1154d7f0a1279cf8e5",
+                "sha256:7e60cb630f674a31f0368ed32b2a6b4331b8350d67de53c0359992444b116dd3",
+                "sha256:89c19a494bf3ad08c1da49445cc5d13d8fefc265f48ee7e7556839acdacf69d0",
+                "sha256:8a86a9b96070674fc88b6f9f71a97d2c1d3e5165574615d1f9168ecba4cecb24",
+                "sha256:8bc7690f7caee50b04a79bf017a8d020c1f48c2a1077ffe172abec59870f1139",
+                "sha256:8d7919548df3f25374a1f5d01fbcd38dacab338ef5f33e044744b5c36729c8db",
+                "sha256:9426133526f69fcaba6e42146b4e12d6bc6c839b8b555097020e2b78ce908dcc",
+                "sha256:9824fb430c9cf9af743cf7aaf6707bf14323fb51ee74425c380f4c846ea70789",
+                "sha256:9bb4a0d90fdb03437c109a17eade42dfbf6190408f29b2744114d11586611d6f",
+                "sha256:9bc2d153989e3216b0559251b0c260cfd168ec78b1fac33dd485750a228db5a2",
+                "sha256:9d35cef91e59ebbeaa45214861874bc6f19eb35de96db73e467a8358d701a96c",
+                "sha256:a1862d2d7ce1674cffa6d186d53ca95c6e17ed2b06b3f4c476173565c862d232",
+                "sha256:a84ab91cbe7aab97f7446652d0ed37d35b68a465aeef8fc41932a9d7eee2c1a6",
+                "sha256:aa7f429242aae2947246587d2964fad750b79e8c233a2367f71b554e9447949c",
+                "sha256:aa9a0521aeca7d4941499a73ad7d4f8ffa3d1affc50b9ea11d992cd7eff18a29",
+                "sha256:ac2f4f7a98934c2ed6505aead07b979e6f999389f16b714448fb39bbaa86a489",
+                "sha256:ae94bd0b2f02c28e199e9bc51485d0c5601f58780636185660f86bf80c89af94",
+                "sha256:af0fc424a5842a11e28956e69395fbbeab2c97c42253169d87e90aac2886d751",
+                "sha256:b2a5db5397d82fa847e4c624b0c98fe59d2d9b7cf0ce6de09e4d2e80f8f5b3f2",
+                "sha256:b4c29cbbba378759ac5786730d1c3cb4ec6f8ababf5c42a9ce303dc4b3d08cda",
+                "sha256:b74b25f024b421d5859d156750ea9a65651793d51b76a2e9238c05c9d5f203a9",
+                "sha256:b7f19250ceef892adf27f0399b9e5afad019288e9be756d6919cb58892129f51",
+                "sha256:b80d4a7900cf6b66bb9cee5c352b2d708e29e5a37fe9bf784fa97fc11504bf6c",
+                "sha256:b8c00a3b1e70c1d3891f0db1b05292747f0dbcfb49c43f9244d04c70fbc40eb8",
+                "sha256:bb273176be34a746bdac0b0d7e4e2c467323d13640b736c4c477881a3220a989",
+                "sha256:c3c20f0ddeb6e29126d45f89206b8291352b8c5b44384e78a6499d68b52ae511",
+                "sha256:c3e130fd0ec56cb76eb49ef52faead8ff09d13f4527e9b0c400307ff72b408e1",
+                "sha256:c52d3f2f82b763a24ef52f5d24358553e8403ce05f893b5347098014f2d9eff2",
+                "sha256:c6377e647bbfd0a0b159fe557f2c6c602c159fc752fa316572f012fc0bf67150",
+                "sha256:c638144ce971df84650d3ed0096e2ae7af8e62ecbbb7b201c8935c370df00a2c",
+                "sha256:ce9845054c13696f7af7f2b353e6b4f676dab1b4b215d7fe5e05c6f8bb06f965",
+                "sha256:cf258ede5bc22a45c8e726b29835b9303c285ab46fc7c3a4cc770736b5304c9f",
+                "sha256:d0a26ffe9d4dd35e4dfdd1e71f46401cff0181c75ac174711ccff0459135fa58",
+                "sha256:d0b67d87bb45ed1cd020e8fbf2307d449b68abc45402fe1a4ac9e46c3c8b192b",
+                "sha256:d20277fd62e1b992a50c43f13fbe13277a31f8c9f70d59759c88f644d66c619f",
+                "sha256:d454b8749b4bd70dd0a79f428731ee263fa6995f83ccb8bada706e8d1d3ff89d",
+                "sha256:d4c7d1a051eeb39f5c9547e82ea27cbcc28338482242e3e0b7768033cb083821",
+                "sha256:d72278a30111e5b5525c1dd96120d9e958464316f55adb030433ea905866f4de",
+                "sha256:d72a210824facfdaf8768cf2d7ca25a042c30320b3020de2fa04640920d4e121",
+                "sha256:d807dc2051abe041b6649681dce568f8e10668e3c1c6543ebae58f2d7e617855",
+                "sha256:dbe982f38565bb50cb7fb061ebf762c2f254ca3d8c20d4006878766e84266272",
+                "sha256:dcedf0b42bcb4cfff4101d7771a10532415a6106062f005ab97d1d0ab5681c60",
+                "sha256:deb62214c42a261cb3eb04d474f7155279c1a8a8c30ac89b7dcb1721d92c3c02",
+                "sha256:def7400461c3a3f26e49078302e1c1b38f6752342c77e3cf72ce91ca69fb1bc1",
+                "sha256:df3de6b7726b52966edf29663e57306b23ef775faf0ac01a3e9f4012a24a4140",
+                "sha256:e1940dae14e715e2e02dfd5b0f64a52e8374a517a1e531ad9412319dc3ac7879",
+                "sha256:e4df1e3b3bec320790f699890d41c59d250f6beda159ea3c44c3f5bac1976940",
+                "sha256:e6900ecdd50ce0facf703f7a00df12374b74bbc8ad9fe0f6559947fb20f82364",
+                "sha256:ea438162a9fcbee3ecf36c23e6c68237479f89f962f82dae83dc15feeceb37e4",
+                "sha256:eb851b7df9dda52dc1415ebee12362047ce771fc36914586b2e9fcbd7d293b3e",
+                "sha256:ec31a99ca63bf3cd7f1a5ac9fe95c5e2d060d3c768a09bc1d16e235840861420",
+                "sha256:f0475242f447cc6cb8a9dd486d68b2ef7fbee84427124c232bff5f63b1fe11e5",
+                "sha256:f2fbf7db2012d4876fb0d66b5b9ba6591197b0f165db8d99371d976546472a24",
+                "sha256:f60012a73aa396be721558caa3a6fd49b3dd0033d1675c6d59c4502e870fcf0c",
+                "sha256:f8e604fe73ba048c06085beaf51147eaec7df856824bfe7b98657cf436623daf",
+                "sha256:f90a4cd061914a60bd51c68bcb4357086991bd0bb93d8aa66a6da7701370708f",
+                "sha256:f918a1a130a6dfe1d7fe0f105064141342e7dd1611f2e6a21cd2f5c8cb1cfb3e",
+                "sha256:fa518bcd7600c584bf42e6617ee8132869e877db2f76bcdc281ec6a4113a53ab",
+                "sha256:faefcc78f53a88f3076b7f8be0a8f8d35133a3ecf7f3770895c25f8813460f08",
+                "sha256:fcaeb7b57f1a1e071ebd748984359fef83ecb026325b9d4ca847c95bc7311c92",
+                "sha256:fd2d84f40633bc475ef2d5490b9c19543fbf18596dcb1b291e3a12ea5d722f7a",
+                "sha256:fdfc3a892927458d98f3d55428ae46b921d1f7543b89382fdb483f5640daaec8"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.18.0"
+            "version": "==0.20.0"
         },
         "scikit-learn": {
             "hashes": [
@@ -2119,7 +2645,7 @@
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.16.0"
         },
         "slicer": {
@@ -2132,19 +2658,19 @@
         },
         "threadpoolctl": {
             "hashes": [
-                "sha256:8f4c689a65b23e5ed825c8436a92b818aac005e0f3715f6a1664d7c7ee29d262",
-                "sha256:f11b491a03661d6dd7ef692dd422ab34185d982466c49c8f98c8f716b5c93196"
+                "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107",
+                "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.4.0"
+            "version": "==3.5.0"
         },
         "tqdm": {
             "hashes": [
-                "sha256:1ee4f8a893eb9bef51c6e35730cebf234d5d0b6bd112b0271e10ed7c24a02bd9",
-                "sha256:6cd52cdf0fef0e0f543299cfc96fec90d7b8a7e88745f411ec33eb44d5ed3531"
+                "sha256:90279a3770753eafc9194a0364852159802111925aa30eb3f9d85b0e805ac7cd",
+                "sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.66.2"
+            "version": "==4.66.5"
         },
         "tzdata": {
             "hashes": [
@@ -2156,20 +2682,20 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d",
-                "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"
+                "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472",
+                "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.2.1"
+            "version": "==2.2.2"
         },
         "xarray": {
             "hashes": [
-                "sha256:5c1db19efdde61db7faedad8fc944f4e29698fb6fbd578d352668b63598bd1d8",
-                "sha256:ca2bc4da2bf2e7879e15862a7a7c3fc76ad19f6a08931d030220cef39a29118d"
+                "sha256:1b0fd51ec408474aa1f4a355d75c00cc1c02bd425d97b2c2e551fd21810e7f64",
+                "sha256:4cae512d121a8522d41e66d942fb06c526bc1fd32c2c181d5fe62fe65b671638"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.9'",
-            "version": "==2024.3.0"
+            "version": "==2024.7.0"
         }
     }
 }

From 2953400dd2966c5c2d4e6246359d3b35f2a83801 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Sun, 8 Sep 2024 17:18:52 +0200
Subject: [PATCH 60/65] add missing option in doc string

---
 .gitignore                        | 2 ++
 hiclass/HierarchicalClassifier.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index dc49827b..ef80c474 100644
--- a/.gitignore
+++ b/.gitignore
@@ -248,6 +248,8 @@ instance/
 # Sphinx documentation
 docs/_build/
 doc/_build/
+docs/examples/trained_model.sav
+docs/source/auto_examples/
 
 # PyBuilder
 target/
diff --git a/hiclass/HierarchicalClassifier.py b/hiclass/HierarchicalClassifier.py
index 0112d768..75ec94c6 100644
--- a/hiclass/HierarchicalClassifier.py
+++ b/hiclass/HierarchicalClassifier.py
@@ -105,7 +105,7 @@ def __init__(
             If True, skip scikit-learn's checks and sample_weight passing for BERT.
         classifier_abbreviation : str, default=""
             The abbreviation of the local hierarchical classifier to be displayed during logging.
-        calibration_method : {"ivap", "cvap", "platt", "isotonic"}, str, default=None
+        calibration_method : {"ivap", "cvap", "platt", "isotonic", "beta"}, str, default=None
             If set, use the desired method to calibrate probabilities returned by predict_proba().
         tmp_dir : str, default=None
             Temporary directory to persist local classifiers that are trained. If the job needs to be restarted,

From 3f2ae97d617112abc1e65d429ccae6a33533f5e6 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Mon, 9 Sep 2024 10:23:23 +0200
Subject: [PATCH 61/65] change try-catch to if statement

---
 .../probability_combiner/ProbabilityCombiner.py   | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/hiclass/probability_combiner/ProbabilityCombiner.py b/hiclass/probability_combiner/ProbabilityCombiner.py
index bd5016b0..3ff9ccaa 100644
--- a/hiclass/probability_combiner/ProbabilityCombiner.py
+++ b/hiclass/probability_combiner/ProbabilityCombiner.py
@@ -30,14 +30,11 @@ def _normalize(self, proba: List[np.ndarray]):
     def _find_predecessors(self, level: int):
         predecessors = defaultdict(list)
         for node in self.classifier.global_classes_[level]:
-            try:
+            if self.classifier.hierarchy_.has_node(node):
                 predecessor = list(self.classifier.hierarchy_.predecessors(node))[0]
-            except NetworkXError:
-                # skip empty levels
-                continue
-
-            predecessor_name = str(predecessor).split(self.classifier.separator_)[-1]
-            node_name = str(node).split(self.classifier.separator_)[-1]
-
-            predecessors[node_name].append(predecessor_name)
+                predecessor_name = str(predecessor).split(self.classifier.separator_)[
+                    -1
+                ]
+                node_name = str(node).split(self.classifier.separator_)[-1]
+                predecessors[node_name].append(predecessor_name)
         return predecessors

From 0f347f9871b0a3bfb6b9b0b6c777072a05e81f5a Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Mon, 9 Sep 2024 12:47:37 +0200
Subject: [PATCH 62/65] make multiply the default aggregation method; add
 'platt' alias to 'sigmoid' as calibration method

---
 hiclass/LocalClassifierPerLevel.py      | 4 ++--
 hiclass/LocalClassifierPerNode.py       | 4 ++--
 hiclass/LocalClassifierPerParentNode.py | 4 ++--
 hiclass/_calibration/Calibrator.py      | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index f62c842f..cbdb7785 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -59,7 +59,7 @@ def __init__(
         bert: bool = False,
         calibration_method: str = None,
         return_all_probabilities: bool = False,
-        probability_combiner: str = "geometric",
+        probability_combiner: str = "multiply",
         tmp_dir: str = None,
     ):
         """
@@ -88,7 +88,7 @@ def __init__(
             If set, use the desired method to calibrate probabilities returned by predict_proba().
         return_all_probabilities : bool, default=False
             If True, return probabilities for all levels. Otherwise, return only probabilities for the last level.
-        probability_combiner: {"geometric", "arithmetic", "multiply"}, str, default="geometric"
+        probability_combiner: {"geometric", "arithmetic", "multiply"}, str, default="multiply"
             Specify the rule for combining probabilities over multiple levels:
 
             - `geometric`: Each levels probabilities are calculated by taking the geometric mean of itself and its predecessors;
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index 68c23400..e504b0d1 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -56,7 +56,7 @@ def __init__(
         bert: bool = False,
         calibration_method: str = None,
         return_all_probabilities: bool = False,
-        probability_combiner: str = "geometric",
+        probability_combiner: str = "multiply",
         tmp_dir: str = None,
     ):
         """
@@ -96,7 +96,7 @@ def __init__(
             If set, use the desired method to calibrate probabilities returned by predict_proba().
         return_all_probabilities : bool, default=False
             If True, return probabilities for all levels. Otherwise, return only probabilities for the last level.
-        probability_combiner: {"geometric", "arithmetic", "multiply"}, str, default="geometric"
+        probability_combiner: {"geometric", "arithmetic", "multiply"}, str, default="multiply"
             Specify the rule for combining probabilities over multiple levels:
 
             - `geometric`: Each levels probabilities are calculated by taking the geometric mean of itself and its predecessors;
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index 7c90fa4b..ca1530ec 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -54,7 +54,7 @@ def __init__(
         bert: bool = False,
         calibration_method: str = None,
         return_all_probabilities: bool = False,
-        probability_combiner: str = "geometric",
+        probability_combiner: str = "multiply",
         tmp_dir: str = None,
     ):
         """
@@ -83,7 +83,7 @@ def __init__(
             If set, use the desired method to calibrate probabilities returned by predict_proba().
         return_all_probabilities : bool, default=False
             If True, return probabilities for all levels. Otherwise, return only probabilities for the last level.
-        probability_combiner: {"geometric", "arithmetic", "multiply"}, str, default="geometric"
+        probability_combiner: {"geometric", "arithmetic", "multiply"}, str, default="multiply"
             Specify the rule for combining probabilities over multiple levels:
 
             - `geometric`: Each levels probabilities are calculated by taking the geometric mean of itself and its predecessors;
diff --git a/hiclass/_calibration/Calibrator.py b/hiclass/_calibration/Calibrator.py
index e39bdab2..5bb3ab73 100644
--- a/hiclass/_calibration/Calibrator.py
+++ b/hiclass/_calibration/Calibrator.py
@@ -117,7 +117,7 @@ def _create_calibrator(self, name: str, params):
             return _InductiveVennAbersCalibrator(**params)
         elif name == "cvap":
             return _CrossVennAbersCalibrator(self.estimator, **params)
-        elif name == "sigmoid":
+        elif name == "sigmoid" or name == "platt":
             return _PlattScaling()
         elif name == "isotonic":
             return _IsotonicRegression(params)

From f299ccdea0fdaf8314126b905593374d3a6249a1 Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Mon, 9 Sep 2024 12:56:20 +0200
Subject: [PATCH 63/65] update docstrings; add calibration example to
 documentation

---
 docs/examples/plot_calibration.py       | 183 ++++++++++++++++++++++++
 hiclass/LocalClassifierPerLevel.py      |   3 +-
 hiclass/LocalClassifierPerNode.py       |   3 +-
 hiclass/LocalClassifierPerParentNode.py |   3 +-
 4 files changed, 189 insertions(+), 3 deletions(-)
 create mode 100644 docs/examples/plot_calibration.py

diff --git a/docs/examples/plot_calibration.py b/docs/examples/plot_calibration.py
new file mode 100644
index 00000000..e6e47e77
--- /dev/null
+++ b/docs/examples/plot_calibration.py
@@ -0,0 +1,183 @@
+# -*- coding: utf-8 -*-
+"""
+=====================
+Calibrating a Classifier
+=====================
+
+A minimalist example showing how to calibrate a Hiclass LCN model. The calibration method can be selected with the :literal:`calibration_method` parameter, for example:
+
+.. tabs::
+
+    .. code-tab:: python
+        :caption: Isotonic Regression
+
+        rf = RandomForestClassifier()
+        classifier = LocalClassifierPerNode(
+            local_classifier=rf,
+            calibration_method='isotonic'
+        )
+
+    .. code-tab:: python
+        :caption: Platt scaling
+
+        rf = RandomForestClassifier()
+        classifier = LocalClassifierPerNode(
+            local_classifier=rf,
+            calibration_method='platt'
+        )
+
+    .. code-tab:: python
+        :caption: Beta scaling
+
+        rf = RandomForestClassifier()
+        classifier = LocalClassifierPerNode(
+            local_classifier=rf,
+            calibration_method='beta'
+        )
+
+    .. code-tab:: python
+        :caption: IVAP
+
+        rf = RandomForestClassifier()
+        classifier = LocalClassifierPerNode(
+            local_classifier=rf,
+            calibration_method='ivap'
+        )
+
+    .. code-tab:: python
+        :caption: CVAP
+
+        rf = RandomForestClassifier()
+        classifier = LocalClassifierPerNode(
+            local_classifier=rf,
+            calibration_method='cvap'
+        )
+
+Furthermore, probabilites of multiple levels can be aggregated by defining a probability combiner:
+
+.. tabs::
+
+    .. code-tab:: python
+        :caption: Multiply (Default)
+
+        rf = RandomForestClassifier()
+        classifier = LocalClassifierPerNode(
+            local_classifier=rf,
+            calibration_method='isotonic',
+            probability_combiner='multiply'
+        )
+
+    .. code-tab:: python
+        :caption: Geometric Mean
+
+        rf = RandomForestClassifier()
+        classifier = LocalClassifierPerNode(
+            local_classifier=rf,
+            calibration_method='isotonic',
+            probability_combiner='geometric'
+        )
+
+    .. code-tab:: python
+        :caption: Arithmetic Mean
+
+        rf = RandomForestClassifier()
+        classifier = LocalClassifierPerNode(
+            local_classifier=rf,
+            calibration_method='isotonic',
+            probability_combiner='arithmetic'
+        )
+
+    .. code-tab:: python
+        :caption: No Aggregation
+
+        rf = RandomForestClassifier()
+        classifier = LocalClassifierPerNode(
+            local_classifier=rf,
+            calibration_method='isotonic',
+            probability_combiner=None
+        )
+
+
+A hierarchical classifier can be calibrated by calling calibrate on the model or by using a Pipeline:
+
+.. tabs::
+
+    .. code-tab:: python
+        :caption: Default
+
+        rf = RandomForestClassifier()
+        classifier = LocalClassifierPerNode(
+            local_classifier=rf,
+            calibration_method='isotonic'
+        )
+        
+        classifier.fit(X_train, Y_train)
+        classifier.calibrate(X_cal, Y_cal)
+        classifier.predict_proba(X_test)
+
+    .. code-tab:: python
+        :caption: Pipeline
+
+        from hiclass import Pipeline
+        
+        rf = RandomForestClassifier()
+        classifier = LocalClassifierPerNode(
+            local_classifier=rf,
+            calibration_method='isotonic'
+        )
+
+        pipeline = Pipeline([
+            ('classifier', classifier),
+        ])
+
+        pipeline.fit(X_train, Y_train)
+        pipeline.calibrate(X_cal, Y_cal)
+        pipeline.predict_proba(X_test)
+
+In the code below, isotonic regression is used to calibrate the model.
+
+"""
+from sklearn.ensemble import RandomForestClassifier
+
+from hiclass import LocalClassifierPerNode
+
+# Define data
+X_train = [[1], [2], [3], [4]]
+X_test = [[4], [3], [2], [1]]
+X_cal = [[5], [6], [7], [8]]
+Y_train = [
+    ["Animal", "Mammal", "Sheep"],
+    ["Animal", "Mammal", "Cow"],
+    ["Animal", "Reptile", "Snake"],
+    ["Animal", "Reptile", "Lizard"],
+]
+
+Y_cal = [
+    ["Animal", "Mammal", "Cow"],
+    ["Animal", "Mammal", "Sheep"],
+    ["Animal", "Reptile", "Lizard"],
+    ["Animal", "Reptile", "Snake"],
+]
+
+# Use random forest classifiers for every node
+rf = RandomForestClassifier()
+
+# Use local classifier per node with isotonic regression as calibration method
+classifier = LocalClassifierPerNode(
+    local_classifier=rf,
+    calibration_method='isotonic',
+    probability_combiner='multiply'
+)
+
+# Train local classifier per node
+classifier.fit(X_train, Y_train)
+
+# Calibrate local classifier per node
+classifier.calibrate(X_cal, Y_cal)
+
+# Predict probabilities
+probabilities = classifier.predict_proba(X_test)
+
+# Print probabilities and labels for the last level
+print(classifier.classes_[2])
+print(probabilities)
diff --git a/hiclass/LocalClassifierPerLevel.py b/hiclass/LocalClassifierPerLevel.py
index cbdb7785..94680ab4 100644
--- a/hiclass/LocalClassifierPerLevel.py
+++ b/hiclass/LocalClassifierPerLevel.py
@@ -88,12 +88,13 @@ def __init__(
             If set, use the desired method to calibrate probabilities returned by predict_proba().
         return_all_probabilities : bool, default=False
             If True, return probabilities for all levels. Otherwise, return only probabilities for the last level.
-        probability_combiner: {"geometric", "arithmetic", "multiply"}, str, default="multiply"
+        probability_combiner: {"geometric", "arithmetic", "multiply", None}, str, default="multiply"
             Specify the rule for combining probabilities over multiple levels:
 
             - `geometric`: Each levels probabilities are calculated by taking the geometric mean of itself and its predecessors;
             - `arithmetic`: Each levels probabilities are calculated by taking the arithmetic mean of itself and its predecessors;
             - `multiply`: Each levels probabilities are calculated by multiplying itself with its predecessors.
+            - `None`: No aggregation.
         tmp_dir : str, default=None
             Temporary directory to persist local classifiers that are trained. If the job needs to be restarted,
             it will skip the pre-trained local classifier found in the temporary directory.
diff --git a/hiclass/LocalClassifierPerNode.py b/hiclass/LocalClassifierPerNode.py
index e504b0d1..c32c0781 100644
--- a/hiclass/LocalClassifierPerNode.py
+++ b/hiclass/LocalClassifierPerNode.py
@@ -96,12 +96,13 @@ def __init__(
             If set, use the desired method to calibrate probabilities returned by predict_proba().
         return_all_probabilities : bool, default=False
             If True, return probabilities for all levels. Otherwise, return only probabilities for the last level.
-        probability_combiner: {"geometric", "arithmetic", "multiply"}, str, default="multiply"
+        probability_combiner: {"geometric", "arithmetic", "multiply", None}, str, default="multiply"
             Specify the rule for combining probabilities over multiple levels:
 
             - `geometric`: Each levels probabilities are calculated by taking the geometric mean of itself and its predecessors;
             - `arithmetic`: Each levels probabilities are calculated by taking the arithmetic mean of itself and its predecessors;
             - `multiply`: Each levels probabilities are calculated by multiplying itself with its predecessors.
+            - `None`: No aggregation.
         tmp_dir : str, default=None
             Temporary directory to persist local classifiers that are trained. If the job needs to be restarted,
             it will skip the pre-trained local classifier found in the temporary directory.
diff --git a/hiclass/LocalClassifierPerParentNode.py b/hiclass/LocalClassifierPerParentNode.py
index ca1530ec..c5373922 100644
--- a/hiclass/LocalClassifierPerParentNode.py
+++ b/hiclass/LocalClassifierPerParentNode.py
@@ -83,12 +83,13 @@ def __init__(
             If set, use the desired method to calibrate probabilities returned by predict_proba().
         return_all_probabilities : bool, default=False
             If True, return probabilities for all levels. Otherwise, return only probabilities for the last level.
-        probability_combiner: {"geometric", "arithmetic", "multiply"}, str, default="multiply"
+        probability_combiner: {"geometric", "arithmetic", "multiply", None}, str, default="multiply"
             Specify the rule for combining probabilities over multiple levels:
 
             - `geometric`: Each levels probabilities are calculated by taking the geometric mean of itself and its predecessors;
             - `arithmetic`: Each levels probabilities are calculated by taking the arithmetic mean of itself and its predecessors;
             - `multiply`: Each levels probabilities are calculated by multiplying itself with its predecessors.
+            - `None`: No aggregation.
         tmp_dir : str, default=None
             Temporary directory to persist local classifiers that are trained. If the job needs to be restarted,
             it will skip the pre-trained local classifier found in the temporary directory.

From 5b1c79517a3c789020bdc8e14cdc2d953828b25e Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Mon, 9 Sep 2024 13:02:15 +0200
Subject: [PATCH 64/65] lint

---
 docs/examples/plot_calibration.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/docs/examples/plot_calibration.py b/docs/examples/plot_calibration.py
index e6e47e77..c867c7b6 100644
--- a/docs/examples/plot_calibration.py
+++ b/docs/examples/plot_calibration.py
@@ -110,7 +110,7 @@
             local_classifier=rf,
             calibration_method='isotonic'
         )
-        
+
         classifier.fit(X_train, Y_train)
         classifier.calibrate(X_cal, Y_cal)
         classifier.predict_proba(X_test)
@@ -119,7 +119,7 @@
         :caption: Pipeline
 
         from hiclass import Pipeline
-        
+
         rf = RandomForestClassifier()
         classifier = LocalClassifierPerNode(
             local_classifier=rf,
@@ -164,9 +164,7 @@
 
 # Use local classifier per node with isotonic regression as calibration method
 classifier = LocalClassifierPerNode(
-    local_classifier=rf,
-    calibration_method='isotonic',
-    probability_combiner='multiply'
+    local_classifier=rf, calibration_method="isotonic", probability_combiner="multiply"
 )
 
 # Train local classifier per node

From b318e5989c6d323cff05aedbdc37d7ec963639ff Mon Sep 17 00:00:00 2001
From: Lukas Drews <drews.lukas@yahoo.com>
Date: Wed, 11 Sep 2024 21:34:41 +0200
Subject: [PATCH 65/65] add more documentation

---
 docs/examples/plot_calibration.py      |   2 +-
 docs/source/algorithms/calibration.rst | 110 +++++++++++++++++++++++++
 docs/source/algorithms/index.rst       |   1 +
 docs/source/algorithms/metrics.rst     |   7 ++
 docs/source/conf.py                    |   2 +-
 5 files changed, 120 insertions(+), 2 deletions(-)
 create mode 100644 docs/source/algorithms/calibration.rst

diff --git a/docs/examples/plot_calibration.py b/docs/examples/plot_calibration.py
index c867c7b6..fef30968 100644
--- a/docs/examples/plot_calibration.py
+++ b/docs/examples/plot_calibration.py
@@ -4,7 +4,7 @@
 Calibrating a Classifier
 =====================
 
-A minimalist example showing how to calibrate a Hiclass LCN model. The calibration method can be selected with the :literal:`calibration_method` parameter, for example:
+A minimalist example showing how to calibrate a HiClass LCN model. The calibration method can be selected with the :literal:`calibration_method` parameter, for example:
 
 .. tabs::
 
diff --git a/docs/source/algorithms/calibration.rst b/docs/source/algorithms/calibration.rst
new file mode 100644
index 00000000..c02341ee
--- /dev/null
+++ b/docs/source/algorithms/calibration.rst
@@ -0,0 +1,110 @@
+.. _calibration-overview:
+
+===========================
+Classifier Calibration
+===========================
+HiClass provides support for probability calibration using various post-hoc calibration methods. 
+
+++++++++++++++++++++++++++
+Motivation
+++++++++++++++++++++++++++
+While many machine learning models can output uncertainty scores, these scores are known to be often poorly calibrated [1]_ [2]_. Model calibration aims to improve the quality of probabilistic forecasts by learning a transformation of the scores, using a separate dataset.
+
+++++++++++++++++++++++++++
+Methods
+++++++++++++++++++++++++++
+
+HiClass supports the following calibration methods:
+
+* Isotonic Regression [3]_
+
+* Platt Scaling [4]_
+
+* Beta Calibration [5]_
+
+* Inductive Venn-Abers Calibration [6]_
+
+* Cross Venn-Abers Calibration [6]_
+
+++++++++++++++++++++++++++
+Probability Aggregation
+++++++++++++++++++++++++++
+
+Combining probabilities over multiple levels is another method to improve probabilistic forecasts. The following methods are supported:
+
+Conditional Probability Aggregation (Multiply Aggregation)
+--------------
+Given a node hierarchy with :math:`n` levels, the probability of a node :math:`A_i`, where :math:`i` denotes the level, is calculated as:
+
+:math:`\displaystyle{\mathbb{P}(A_1 \cap A_2 \cap \ldots \cap A_i) = \mathbb{P}(A_1) \cdot \mathbb{P}(A_2 \mid A_1) \cdot \mathbb{P}(A_3 \mid A_1 \cap A_2) \cdot \ldots}`
+:math:`\displaystyle{\cdot \mathbb{P}(A_i \mid A_1 \cap A_2 \cap \ldots \cap A_{i-1})}`
+
+Arithmetic Mean Aggregation
+--------------
+:math:`\displaystyle{\mathbb{P}(A_i) = \frac{1}{i} \sum_{j=1}^{i} \mathbb{P}(A_{j})}`
+
+Geometric Mean Aggregation
+--------------
+:math:`\displaystyle{\mathbb{P}(A_i) = \exp{\left(\frac{1}{i} \sum_{j=1}^{i} \ln \mathbb{P}(A_{j})\right)}}`
+
+++++++++++++++++++++++++++
+Code sample
+++++++++++++++++++++++++++
+
+.. code-block:: python
+
+    from sklearn.ensemble import RandomForestClassifier
+
+    from hiclass import LocalClassifierPerNode
+
+    # Define data
+    X_train = [[1], [2], [3], [4]]
+    X_test = [[4], [3], [2], [1]]
+    X_cal = [[5], [6], [7], [8]]
+    Y_train = [
+        ["Animal", "Mammal", "Sheep"],
+        ["Animal", "Mammal", "Cow"],
+        ["Animal", "Reptile", "Snake"],
+        ["Animal", "Reptile", "Lizard"],
+    ]
+
+    Y_cal = [
+        ["Animal", "Mammal", "Cow"],
+        ["Animal", "Mammal", "Sheep"],
+        ["Animal", "Reptile", "Lizard"],
+        ["Animal", "Reptile", "Snake"],
+    ]
+
+    # Use random forest classifiers for every node
+    rf = RandomForestClassifier()
+
+    # Use local classifier per node with isotonic regression as calibration method
+    classifier = LocalClassifierPerNode(
+        local_classifier=rf, calibration_method="isotonic", probability_combiner="multiply"
+    )
+
+    # Train local classifier per node
+    classifier.fit(X_train, Y_train)
+
+    # Calibrate local classifier per node
+    classifier.calibrate(X_cal, Y_cal)
+
+    # Predict probabilities
+    probabilities = classifier.predict_proba(X_test)
+
+    # Print probabilities and labels for the last level
+    print(classifier.classes_[2])
+    print(probabilities)
+
+.. [1] Niculescu-Mizil, Alexandru; Caruana, Rich (2005): Predicting good probabilities with supervised learning. In: Saso Dzeroski (Hg.): Proceedings of the 22nd international conference on Machine learning - ICML '05. the 22nd international conference. Bonn, Germany, 07.08.2005 - 11.08.2005. New York, New York, USA: ACM Press, S. 625-632.
+
+.. [2] Chuan Guo; Geoff Pleiss; Yu Sun; Kilian Q. Weinberger (2017): On Calibration of Modern Neural Networks. In: Doina Precup und Yee Whye Teh (Hg.): Proceedings of the 34th International Conference on Machine Learning, Bd. 70: PMLR (Proceedings of Machine Learning Research), S. 1321-1330.
+
+.. [3] Zadrozny, Bianca; Elkan, Charles (2002): Transforming classifier scores into accurate multiclass probability estimates. In: Proceedings of the Eighth ACM SIGKDD International Conference on Knowledge Discovery and Data Mining. New York, NY, USA: Association for Computing Machinery (KDD ’02), S. 694-699.
+
+.. [4] Platt, John (2000): Probabilistic Outputs for Support Vector Machines and Comparisons to Regularized Likelihood Methods. In: Adv. Large Margin Classif. 10.
+
+.. [5] Kull, Meelis; Filho, Telmo Silva; Flach, Peter (2017): Beta calibration: a well-founded and easily implemented improvement on logistic calibration for binary classifiers. In: Aarti Singh und Jerry Zhu (Hg.): Proceedings of the 20th International Conference on Artificial Intelligence and Statistics, Bd. 54: PMLR (Proceedings of Machine Learning Research), S. 623-631.
+
+.. [6] Vovk, Vladimir; Petej, Ivan; Fedorova, Valentina (2015): Large-scale probabilistic predictors with and without guarantees of validity. In: C. Cortes, N. Lawrence, D. Lee, M. Sugiyama und R. Garnett (Hg.): Advances in Neural Information Processing Systems, Bd. 28: Curran Associates, Inc. 
+
diff --git a/docs/source/algorithms/index.rst b/docs/source/algorithms/index.rst
index 092a6079..0087781d 100644
--- a/docs/source/algorithms/index.rst
+++ b/docs/source/algorithms/index.rst
@@ -17,3 +17,4 @@ HiClass provides implementations for the most popular machine learning models fo
     multi_label
     metrics
     explainer
+    calibration
diff --git a/docs/source/algorithms/metrics.rst b/docs/source/algorithms/metrics.rst
index bdac5909..6d4f542d 100644
--- a/docs/source/algorithms/metrics.rst
+++ b/docs/source/algorithms/metrics.rst
@@ -3,12 +3,19 @@
 Metrics
 ====================
 
+Classification Metrics
+--------------
+
 According to [1]_, the use of flat classification metrics might not be adequate to give enough insight of which algorithm is better at classifying hierarchical data. Hence, in HiClass we implemented the metrics of hierarchical precision (hP), hierarchical recall (hR) and hierarchical F-score (hF), which are extensions of the renowned metrics of precision, recall and F-score, but tailored to the hierarchical classification scenario. These hierarchical counterparts were initially proposed by [2]_, and are defined as follows:
 
 :math:`\displaystyle{hP = \frac{\sum_i|\alpha_i\cap\beta_i|}{\sum_i|\alpha_i|}}`, :math:`\displaystyle{hR = \frac{\sum_i|\alpha_i\cap\beta_i|}{\sum_i|\beta_i|}}`, :math:`\displaystyle{hF = \frac{2 \times hP \times hR}{hP + hR}}`
 
 where :math:`\alpha_i` is the set consisting of the most specific classes predicted for test example :math:`i` and all their ancestor classes, while :math:`\beta_i` is the set containing the true most specific classes of test example :math:`i` and all their ancestors, with summations computed over all test examples.
 
+Calibration Metrics
+--------------
+
+
 .. [1] Silla, C. N., & Freitas, A. A. (2011). A survey of hierarchical classification across different application domains. Data Mining and Knowledge Discovery, 22(1), 31-72.
 
 .. [2] Kiritchenko, S., Matwin, S., Nock, R., & Famili, A. F. (2006, June). Learning and evaluation in the presence of class hierarchies: Application to text categorization. In Conference of the Canadian Society for Computational Studies of Intelligence (pp. 395-406). Springer, Berlin, Heidelberg.
diff --git a/docs/source/conf.py b/docs/source/conf.py
index aecd85fd..6bd3c591 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -23,7 +23,7 @@
 # -- Project information -----------------------------------------------------
 
 project = "hiclass"
-copyright = "2022, Fabio Malcher Miranda, Niklas Köhnecke"
+copyright = "2024, Fabio Malcher Miranda, Niklas Köhnecke"
 author = "Fabio Malcher Miranda, Niklas Köhnecke"