feat: Python 3.12 only and CustomComp. Unit testing

.devcontainer/dev.Dockerfile

@@ -29,7 +29,7 @@ USER vscode
 # Define the version of Poetry to install (default is 1.4.2)
 # Define the directory of python virtual environment
 ARG PYTHON_VIRTUALENV_HOME=/home/vscode/ukbc-py-env \
-    POETRY_VERSION=1.8.3
+    POETRY_VERSION=1.8.4
 
 ENV POETRY_VIRTUALENVS_IN_PROJECT=false \
     POETRY_NO_INTERACTION=true

.devcontainer/devcontainer.json

@@ -47,7 +47,10 @@
                 "python.testing.autoTestDiscoverOnSaveEnabled": false,
                 "python.defaultInterpreterPath": "/home/vscode/ukbc-py-env",
                 "python.testing.pytestArgs": [
+                    "--disable-socket", 
+                    "--allow-unix-socket",
                     "${workspaceFolder}/uk_bin_collection",
+                    "${workspaceFolder}/custom_components/uk_bin_collection/tests",
                     "--headless=False",
                     "-o cache_dir=${workspaceFolder}/.pytest_cache"
                 ],

.github/ISSUE_TEMPLATE/HOME_ASSISTANT_CUSTOM_COMPONENT_ISSUE.yaml

@@ -5,7 +5,7 @@ labels: ["bug", "home assistant custom component"]
 body:
   - type: markdown
     attributes:
-      value: If you were trying to add a specific council, please check it is listed as working [here](https://robbrad.github.io/UKBinCollectionData/3.11/) and open a [Council Issue](https://github.com/robbrad/UKBinCollectionData/issues/new/choose) instead if it's failing
+      value: If you were trying to add a specific council, please check it is listed as working [here](https://robbrad.github.io/UKBinCollectionData/3.12/) and open a [Council Issue](https://github.com/robbrad/UKBinCollectionData/issues/new/choose) instead if it's failing
   - type: input
     id: ha_version
     attributes:
@@ -46,7 +46,7 @@ body:
       options:
         - label: I searched for similar issues at https://github.com/robbrad/UKBinCollectionData/issues?q=is:issue and found no duplicates
           required: true
-        - label: If trying to add a specific council, I've checked it is listed as working at https://robbrad.github.io/UKBinCollectionData/3.11/
+        - label: If trying to add a specific council, I've checked it is listed as working at https://robbrad.github.io/UKBinCollectionData/3.12/
           required: true
         - label: I have provided a detailed explanation of the issue as well as steps to replicate the issue
           required: true

.github/workflows/behave.yml

@@ -3,30 +3,28 @@ name: Test Councils
 on:
   workflow_dispatch:
   push:
-    # Trigger unless only the wiki directory changed
     paths-ignore:
       - "wiki/**"
       - "**/**.md"
       - "uk_bin_collection_api_server/**"
     branches: [ "master" ]
   pull_request:
-    # Trigger unless only the wiki directory changed
     paths-ignore:
       - "wiki/**"
       - "**/**.md"
       - "uk_bin_collection_api_server/**"
     branches: [ "master" ]
   schedule:
-    - cron: '0 0 * * *'
+    - cron: '0 0 * * *' # Nightly schedule for full test run
 
 jobs:
   build:
     if: "!startsWith(github.event.head_commit.message, 'bump:')"
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.11, 3.12]
-        poetry-version: [1.8.3]
+        python-version: [3.12]
+        poetry-version: [1.8.4]
     services:
       selenium:
         image: selenium/standalone-chrome:latest
@@ -42,7 +40,6 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          #cache: 'poetry'
 
       - name: Install
         run: make install
@@ -89,46 +86,69 @@ jobs:
         run: make unit-tests
         continue-on-error: true
 
-      #- name: Upload test coverage to Codecov
-      #  uses: codecov/codecov-action@v4
-      #  with:
-      #    gcov_ignore: uk_bin_collection/tests/**
-
       - name: Upload test results to Codecov
         uses: codecov/codecov-action@v4
         with:
           fail_ci_if_error: false
           token: ${{ secrets.CODECOV_TOKEN }}
           file: coverage.xml
 
+      # Separate steps for Full and Partial Reports
+      - name: Get Allure history - Full Report
+        if: github.event_name == 'schedule' || github.event_name == 'push'
+        uses: actions/checkout@v4
+        continue-on-error: true
+        with:
+          ref: gh-pages
+          path: gh-pages/allure-full-history
 
-      - name: Get Allure history
+      - name: Get Allure history - Partial Report
+        if: github.event_name != 'schedule' && github.event_name != 'push'
         uses: actions/checkout@v4
-        if: github.ref == 'refs/heads/master'
         continue-on-error: true
         with:
           ref: gh-pages
-          path: gh-pages
+          path: gh-pages/allure-partial-history
+
+      - name: Allure report action for Full Run
+        uses: simple-elf/allure-report-action@master
+        if: github.event_name == 'schedule' || github.event_name == 'push'
+        with:
+          allure_results: build/${{ matrix.python-version }}/allure-results
+          subfolder: ${{ matrix.python-version }}
+          allure_history: gh-pages/allure-full-history
+          keep_reports: 20
 
-      - name: Allure report action from marketplace
+      - name: Allure report action for Partial Run
         uses: simple-elf/allure-report-action@master
-        if: github.ref == 'refs/heads/master'
+        if: github.event_name != 'schedule' && github.event_name != 'push'
         with:
           allure_results: build/${{ matrix.python-version }}/allure-results
           subfolder: ${{ matrix.python-version }}
-          allure_history: allure-history
+          allure_history: gh-pages/allure-partial-history
           keep_reports: 20
 
-      - name: Tar report
-        if: github.ref == 'refs/heads/master'
-        run: tar -cvf allure_history_${{ matrix.python-version }}.tar allure-history/${{ matrix.python-version }}
+      - name: Tar full report
+        if: github.event_name == 'schedule' || github.event_name == 'push'
+        run: tar -cvf allure_full_history_${{ matrix.python-version }}.tar gh-pages/allure-full-history/${{ matrix.python-version }}
+
+      - name: Tar partial report
+        if: github.event_name != 'schedule' && github.event_name != 'push'
+        run: tar -cvf allure_partial_history_${{ matrix.python-version }}.tar gh-pages/allure-partial-history/${{ matrix.python-version }}
+
+      - name: Upload artifact for Full Report
+        uses: actions/upload-artifact@v4
+        if: github.event_name == 'schedule' || github.event_name == 'push'
+        with:
+          name: allure_full_history_${{ matrix.python-version }}
+          path: allure_full_history_${{ matrix.python-version }}.tar
 
-      - name: Upload artifact
+      - name: Upload artifact for Partial Report
         uses: actions/upload-artifact@v4
-        if: github.ref == 'refs/heads/master'
+        if: github.event_name != 'schedule' && github.event_name != 'push'
         with:
-          name: allure_history_${{ matrix.python-version }}
-          path: allure_history_${{ matrix.python-version }}.tar
+          name: allure_partial_history_${{ matrix.python-version }}
+          path: allure_partial_history_${{ matrix.python-version }}.tar
 
   deploy:
     runs-on: ubuntu-latest
@@ -137,29 +157,36 @@ jobs:
     steps:
 
       - uses: actions/download-artifact@v4
-        name: Download 3.11 artifacts
+        name: Download artifacts
         with:
-          name: allure_history_3.11
-          path: allure-history/tars
+          name: allure_full_history_3.12
+          path: allure-history/tars/full
 
       - uses: actions/download-artifact@v4
-        name: Download 3.12 artifacts
+        name: Download partial artifacts
         with:
-          name: allure_history_3.12
-          path: allure-history/tars
+          name: allure_partial_history_3.12
+          path: allure-history/tars/partial
 
-      - name: Untar reports
-        run: for i in allure-history/tars/*.tar; do tar -xvf "$i" allure-history ;done
+      - name: Untar full reports
+        run: for i in allure-history/tars/full/*.tar; do tar -xvf "$i" allure-history/full ;done
+
+      - name: Untar partial reports
+        run: for i in allure-history/tars/partial/*.tar; do tar -xvf "$i" allure-history/partial ;done
 
       - name: Remove tar reports
         run: rm -rf allure-history/tars
 
-      - name: Display structure of downloaded files
-        run: ls -R
+      - name: Deploy Full Report
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          PERSONAL_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PUBLISH_BRANCH: gh-pages
+          PUBLISH_DIR: allure-history/full
 
-      - name: Deploy
+      - name: Deploy Partial Report
         uses: peaceiris/actions-gh-pages@v4
         with:
           PERSONAL_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           PUBLISH_BRANCH: gh-pages
-          PUBLISH_DIR: allure-history
+          PUBLISH_DIR: allure-history/partial

.github/workflows/release.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Run image
         uses: abatilo/[email protected]
         with:
-          poetry-version: '1.8.3'
+          poetry-version: '1.8.4'
 
       - name: Install dependencies
         run: |

.github/workflows/wiki.yml

@@ -30,7 +30,7 @@ jobs:
     - name: Run image
       uses: abatilo/[email protected]
       with:
-        poetry-version: '1.8.3'
+        poetry-version: '1.8.4'
 
     - name: Install
       run: make install

.vscode/launch.json

@@ -24,7 +24,7 @@
                 "debug-test"
             ],
             "env": {
-                "PYTEST_ADDOPTS": "--headless=False --local_browser=False"
+                "PYTEST_ADDOPTS": "--headless=False --local_browser=False --disable-socket --allow-unix-socket"
             },
         }
     ]

CONTRIBUTING.md

@@ -41,7 +41,7 @@ This document contains guidelines on contributing to the UKBCD project including
 the environment, how we use our issue tracker, and how you can develop more scrapers.
 
 ## Getting Started
-You will need to install Python on the system you plan to run the script from. Python 3.11 and 3.12 are tested on this project .
+You will need to install Python on the system you plan to run the script from. Python 3.12 is tested on this project .
 
 The project uses [poetry](https://python-poetry.org/docs/) to manage dependencies and setup the build environment.
 

Makefile

@@ -24,17 +24,17 @@ pycodestyle:
 ## @Testing runs unit tests
 integration-tests: ## runs tests for the project
 	if [ -z "$(councils)" ]; then \
-		poetry run pytest uk_bin_collection/tests/step_defs/ -n logical --alluredir=build/$(matrix)/allure-results; \
+		poetry run pytest --disable-socket --allow-unix-socket uk_bin_collection/tests/step_defs/ -n logical --alluredir=build/$(matrix)/allure-results; \
 	else \
-		poetry run pytest uk_bin_collection/tests/step_defs/ -k "$(councils)" -n logical --alluredir=build/$(matrix)/allure-results; \
+		poetry run pytest --disable-socket --allow-unix-socket uk_bin_collection/tests/step_defs/ -k "$(councils)" -n logical --alluredir=build/$(matrix)/allure-results; \
 	fi
 
 parity-check:
 	poetry run python uk_bin_collection/tests/council_feature_input_parity.py $(repo) $(branch)
 
 unit-tests:
 	poetry run coverage erase
-	- poetry run coverage run --append --omit "*/tests/*" -m pytest uk_bin_collection/tests --ignore=uk_bin_collection/tests/step_defs/
+	- poetry run coverage run --append --omit "*/tests/*" -m pytest --disable-socket --allow-unix-socket uk_bin_collection/tests custom_components/uk_bin_collection/tests --ignore=uk_bin_collection/tests/step_defs/ 
 	poetry run coverage xml
 
 update-wiki:

README.md

@@ -267,7 +267,6 @@ docker pull selenium/standalone-chrome docker run -d -p 4444:4444 --name seleniu
 
 ## Reports
 
-- [3.11](https://robbrad.github.io/UKBinCollectionData/3.11/)
 - [3.12](https://robbrad.github.io/UKBinCollectionData/3.12/)
 
 ---

conftest.py

@@ -1,7 +1,16 @@
 import pytest
 from _pytest.config.argparsing import Parser
 from _pytest.fixtures import FixtureRequest
+import asyncio
 
+from pytest_socket import enable_socket, disable_socket, socket_allow_hosts
+
+@pytest.hookimpl(trylast=True)
+def pytest_runtest_setup():
+    enable_socket()
+    socket_allow_hosts(None)  # Allow all hosts
+
+## Integration Tests
 def pytest_addoption(parser: Parser) -> None:
     parser.addoption("--headless", action="store", default="True", type=str)
     parser.addoption("--local_browser", action="store", default="False", type=str)

custom_components/uk_bin_collection/config_flow.py

@@ -20,13 +20,20 @@ def __init__(self):
         self.councils_data = None
 
     async def get_councils_json(self) -> object:
-        """Returns an object of supported council's and their required fields."""
-        # Fetch the JSON data from the provided URL
+        """Returns an object of supported councils and their required fields."""
         url = "https://raw.githubusercontent.com/robbrad/UKBinCollectionData/0.104.0/uk_bin_collection/tests/input.json"
-        async with aiohttp.ClientSession() as session:
-            async with session.get(url) as response:
-                data_text = await response.text()
-                return json.loads(data_text)
+        try:
+            async with aiohttp.ClientSession() as session:
+                try:
+                    async with session.get(url) as response:
+                        data_text = await response.text()
+                        return json.loads(data_text)
+                except Exception as e:
+                    _LOGGER.error("Failed to fetch data from URL: %s", e)
+                    raise
+        except Exception as e:
+            _LOGGER.error("Failed to create aiohttp ClientSession: %s", e)
+            return {}
 
     async def get_council_schema(self, council=str) -> vol.Schema:
         """Returns a config flow form schema based on a specific council's fields."""
@@ -83,6 +90,10 @@ async def async_step_user(self, user_input=None):
         errors = {}
 
         self.councils_data = await self.get_councils_json()
+        if not self.councils_data:
+            _LOGGER.error("Council data is unavailable.")
+            return self.async_abort(reason="council_data_unavailable")
+
         self.council_names = list(self.councils_data.keys())
         self.council_options = [
             self.councils_data[name]["wiki_name"] for name in self.council_names
@@ -131,7 +142,8 @@ async def async_step_council(self, user_input=None):
         errors = {}
 
         if user_input is not None:
-            if "skip_get_url" in self.councils_data[self.data["council"]]:
+            # Check the value of 'skip_get_url' rather than just its presence
+            if self.councils_data[self.data["council"]].get("skip_get_url", False):
                 user_input["skip_get_url"] = True
                 user_input["url"] = self.councils_data[self.data["council"]]["url"]
 
@@ -152,7 +164,7 @@ async def async_step_council(self, user_input=None):
     async def async_step_init(self, user_input=None):
         """Handle a flow initiated by the user."""
         _LOGGER.info(LOG_PREFIX + "Initiating flow with user input: %s", user_input)
-        return await self.async_step_user(user_input)
+        return await self.async_step_user(user_input=user_input)
 
     async def async_step_reconfigure(self, user_input=None):
         """Handle reconfiguration of the integration."""