diff --git a/ipper/common/wiki.py b/ipper/common/wiki.py
index daf0100..dd1015e 100644
--- a/ipper/common/wiki.py
+++ b/ipper/common/wiki.py
@@ -63,7 +63,7 @@ def child_page_generator(wiki_page_info, chunk: int, timeout: int) -> Generator[
first_child_request: requests.Response = requests.get(
APACHE_CONFLUENCE_BASE_URL + wiki_page_child_info_request.json()["_expandable"]["page"],
params={
- "limit": chunk,
+ "limit": str(chunk),
"expand": "history.lastUpdated,body.view",
},
timeout=timeout,
diff --git a/ipper/kafka/mailing_list.py b/ipper/kafka/mailing_list.py
index 7aa3635..0293945 100644
--- a/ipper/kafka/mailing_list.py
+++ b/ipper/kafka/mailing_list.py
@@ -179,9 +179,12 @@ def extract_message_payload(msg: Message) -> List[str]:
for message in msg.walk():
- temp_payload: Union[List[Message], str] = message.get_payload()
+ temp_payload: Union[List[Union[Message, str]], Message, str] = message.get_payload()
if isinstance(temp_payload, list):
- payload: str = cast(str, temp_payload[0].get_payload())
+ if isinstance(temp_payload[0], Message):
+ payload: str = cast(str, temp_payload[0].get_payload())
+ elif isinstance(temp_payload[0], str):
+ payload = cast(str, message.get_payload())
elif isinstance(temp_payload, str):
payload = cast(str, message.get_payload())
else:
diff --git a/ipper/kafka/output.py b/ipper/kafka/output.py
index c729004..ae66d9e 100644
--- a/ipper/kafka/output.py
+++ b/ipper/kafka/output.py
@@ -5,7 +5,7 @@
from enum import Enum
from pathlib import Path
-from pandas import DataFrame, Timestamp, Timedelta, to_datetime
+from pandas import DataFrame, Series, Timestamp, Timedelta, to_datetime
from jinja2 import Template, Environment, FileSystemLoader
from ipper.common.utils import calculate_age
@@ -69,7 +69,6 @@ def create_vote_dict(kip_mentions: DataFrame) -> Dict[int, Dict[str, List[str]]]
from vote type to list of those who voted that way"""
vote_dict: Dict[int, Dict[str, List[str]]] = {}
- kip_id: int
kip_votes: DataFrame
for kip_id, kip_votes in kip_mentions[~kip_mentions["vote"].isna()][
["kip", "from", "vote"]
@@ -82,7 +81,7 @@ def create_vote_dict(kip_mentions: DataFrame) -> Dict[int, Dict[str, List[str]]]
for name in kip_votes[kip_votes["vote"] == vote]["from"]
)
)
- vote_dict[kip_id] = kip_dict
+ vote_dict[cast(int, kip_id)] = kip_dict
return vote_dict
@@ -95,7 +94,7 @@ def create_status_dict(
recent_mentions: DataFrame = get_most_recent_mention_by_type(kip_mentions)
- subject_mentions: DataFrame = recent_mentions["subject"].dropna()
+ subject_mentions: Series = recent_mentions["subject"].dropna()
vote_dict: Dict[int, Dict[str, List[str]]] = create_vote_dict(kip_mentions)
diff --git a/ipper/kafka/wiki.py b/ipper/kafka/wiki.py
index f3f45fb..a66e64c 100644
--- a/ipper/kafka/wiki.py
+++ b/ipper/kafka/wiki.py
@@ -86,7 +86,7 @@ def get_current_state(html: str) -> Optional[str]:
return None
-def enrich_kip_info(body_html: str, kip_dict: dict[str, Union[str, int]]) -> None:
+def enrich_kip_info(body_html: str, kip_dict: dict[str, Union[list[str], str, int]]) -> None:
"""Parses the body of the KIP wiki page pointed to by the 'content_url'
key in the supplied dictionary. It will add the derived data to the
supplied dict."""
@@ -111,7 +111,7 @@ def enrich_kip_info(body_html: str, kip_dict: dict[str, Union[str, int]]) -> Non
elif not jira_processed and "jira" in para.text.lower():
link: Tag = para.find("a")
if link:
- href: Optional[str] = link.get("href")
+ href: Optional[Union[list, str]] = link.get("href")
else:
href = None
@@ -134,7 +134,7 @@ def process_child_kip(kip_id: int, child: dict):
"""Process and enrich the KIP child page dictionary"""
print(f"Processing KIP {kip_id} wiki page")
- child_dict: dict[str, Union[int, str]] = {}
+ child_dict: dict[str, Union[list[str], str, int]] = {}
child_dict["kip_id"] = kip_id
child_dict["title"] = child["title"]
child_dict["web_url"] = APACHE_CONFLUENCE_BASE_URL + child["_links"]["webui"]
@@ -234,6 +234,9 @@ def process_discussion_table(
kip_dict: dict[str, str] = {}
columns: list[Tag] = row.find_all("td")
+ if not columns[0].a:
+ continue
+
kip_text: str = columns[0].a.text
kip_match: Optional[re.Match] = re.search(KIP_PATTERN, kip_text)
@@ -244,7 +247,14 @@ def process_discussion_table(
try:
kip_dict["url"] = cast(str, kip_child_urls[kip_id]["web_url"])
except KeyError:
- kip_dict["url"] = columns[0].a.get("href")
+ href: Optional[Union[list[str], str]] = columns[0].a.get("href")
+ if href:
+ if isinstance(href, list):
+ kip_dict["url"] = href[0]
+ else:
+ kip_dict["url"] = href
+ else:
+ kip_dict["url"] = "Unknown"
output[kip_id] = kip_dict
else:
continue
diff --git a/poetry.lock b/poetry.lock
index aac8b44..23569e7 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -698,6 +698,21 @@ sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-d
test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"]
xml = ["lxml (>=4.9.2)"]
+[[package]]
+name = "pandas-stubs"
+version = "2.2.3.241126"
+description = "Type annotations for pandas"
+optional = false
+python-versions = ">=3.10"
+files = [
+ {file = "pandas_stubs-2.2.3.241126-py3-none-any.whl", hash = "sha256:74aa79c167af374fe97068acc90776c0ebec5266a6e5c69fe11e9c2cf51f2267"},
+ {file = "pandas_stubs-2.2.3.241126.tar.gz", hash = "sha256:cf819383c6d9ae7d4dabf34cd47e1e45525bb2f312e6ad2939c2c204cb708acd"},
+]
+
+[package.dependencies]
+numpy = ">=1.23.5"
+types-pytz = ">=2022.1.1"
+
[[package]]
name = "parso"
version = "0.8.4"
@@ -944,6 +959,42 @@ files = [
docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"]
test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0,<8.2)", "pytest-mock", "pytest-mypy-testing"]
+[[package]]
+name = "types-beautifulsoup4"
+version = "4.12.0.20241020"
+description = "Typing stubs for beautifulsoup4"
+optional = false
+python-versions = ">=3.8"
+files = [
+ {file = "types-beautifulsoup4-4.12.0.20241020.tar.gz", hash = "sha256:158370d08d0cd448bd11b132a50ff5279237a5d4b5837beba074de152a513059"},
+ {file = "types_beautifulsoup4-4.12.0.20241020-py3-none-any.whl", hash = "sha256:c95e66ce15a4f5f0835f7fbc5cd886321ae8294f977c495424eaf4225307fd30"},
+]
+
+[package.dependencies]
+types-html5lib = "*"
+
+[[package]]
+name = "types-html5lib"
+version = "1.1.11.20241018"
+description = "Typing stubs for html5lib"
+optional = false
+python-versions = ">=3.8"
+files = [
+ {file = "types-html5lib-1.1.11.20241018.tar.gz", hash = "sha256:98042555ff78d9e3a51c77c918b1041acbb7eb6c405408d8a9e150ff5beccafa"},
+ {file = "types_html5lib-1.1.11.20241018-py3-none-any.whl", hash = "sha256:3f1e064d9ed2c289001ae6392c84c93833abb0816165c6ff0abfc304a779f403"},
+]
+
+[[package]]
+name = "types-pytz"
+version = "2024.2.0.20241221"
+description = "Typing stubs for pytz"
+optional = false
+python-versions = ">=3.8"
+files = [
+ {file = "types_pytz-2024.2.0.20241221-py3-none-any.whl", hash = "sha256:8fc03195329c43637ed4f593663df721fef919b60a969066e22606edf0b53ad5"},
+ {file = "types_pytz-2024.2.0.20241221.tar.gz", hash = "sha256:06d7cde9613e9f7504766a0554a270c369434b50e00975b3a4a0f6eed0f2c1a9"},
+]
+
[[package]]
name = "types-requests"
version = "2.32.0.20241016"
@@ -1011,4 +1062,4 @@ files = [
[metadata]
lock-version = "2.0"
python-versions = "^3.12"
-content-hash = "fc563d16ccd0d380ecb7be48f84460cf217a7d64b06b4138bee160183d5eee45"
+content-hash = "cc375158fdf68fc3c62a4f31139fa913d5054c7de66d1336ecad1c6345a5b418"
diff --git a/pyproject.toml b/pyproject.toml
index fe9eb9e..631deee 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -11,13 +11,15 @@ beautifulsoup4 = "^4.10.0"
pandas = "^2.2.3"
Jinja2 = "^3.1.4"
-[tool.poetry.dev-dependencies]
+[tool.poetry.group.dev.dependencies]
ipython = "^8.29.0"
pylint = "^3.3.3"
black = "^24.10.0"
mypy = "^1.14.0"
types-requests = "^2.32.0.20241016"
ipdb = "^0.13.13"
+pandas-stubs = "^2.2.3.241126"
+types-beautifulsoup4 = "^4.12.0.20241020"
[build-system]
requires = ["poetry-core>=1.0.0"]
diff --git a/templates/flink-index.html b/templates/flink-index.html
index 86de145..6ebb777 100644
--- a/templates/flink-index.html
+++ b/templates/flink-index.html
@@ -11,6 +11,7 @@
Flink Improvement Proposals
+ Back to OSSIP
Coming Soon..
diff --git a/templates/index.html b/templates/index.html
index 7843a2b..da86e50 100644
--- a/templates/index.html
+++ b/templates/index.html
@@ -14,7 +14,7 @@ OSSIP
A hub for information on Open Source Software Improvement Proposals
Created by Tom Cooper
- GitHub Repository: tomncooper/kipper
+ GitHub Repository: tomncooper/ossip
diff --git a/templates/kafka-index.html.jinja b/templates/kafka-index.html.jinja
index 286af85..e49d67e 100644
--- a/templates/kafka-index.html.jinja
+++ b/templates/kafka-index.html.jinja
@@ -53,8 +53,8 @@
Kafka Improvement Proposals (KIPs)
- Back to IPper
- GitHub Repository: tomncooper/kipper
+ Back to OSSIP
+ GitHub Repository: tomncooper/ossip
Last Updated: {{ date }}
Status Key