diff --git a/.devcontainer/dev.Dockerfile b/.devcontainer/dev.Dockerfile new file mode 100644 index 0000000000..613023bd98 --- /dev/null +++ b/.devcontainer/dev.Dockerfile @@ -0,0 +1,41 @@ +ARG VARIANT="3.11-bullseye" +FROM mcr.microsoft.com/devcontainers/python:${VARIANT} AS ukbc-dev-base + +USER vscode + +# Define the version of Poetry to install (default is 1.4.2) +# Define the directory of python virtual environment +ARG PYTHON_VIRTUALENV_HOME=/home/vscode/ukbc-py-env \ + POETRY_VERSION=1.5.1 + +ENV POETRY_VIRTUALENVS_IN_PROJECT=false \ + POETRY_NO_INTERACTION=true + +# Install Poetry outside of the v`irtual environment to avoid conflicts +RUN python3 -m pip install --user pipx && \ + python3 -m pipx ensurepath && \ + pipx install poetry==${POETRY_VERSION} + +# Create a Python virtual environment for the project +RUN python3 -m venv ${PYTHON_VIRTUALENV_HOME} && \ + $PYTHON_VIRTUALENV_HOME/bin/pip install --upgrade pip + +ENV PATH="$PYTHON_VIRTUALENV_HOME/bin:$PATH" \ + VIRTUAL_ENV=$PYTHON_VIRTUALENV_HOME + +# Setup for bash +RUN poetry completions bash >> /home/vscode/.bash_completion && \ + echo "export PATH=$PYTHON_VIRTUALENV_HOME/bin:$PATH" >> ~/.bashrc + +# Set the working directory for the app +WORKDIR /ukbc_build + +# Use a multi-stage build to install dependencies +FROM ukbc-dev-base AS ukbc-dev-dependencies + +ARG PYTHON_VIRTUALENV_HOME + +COPY . /ukbc_build/ + +RUN poetry install --no-interaction --no-ansi --with dev +#docker build -f .devcontainer/dev.Dockerfile -t ukbc_dev_container . \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..e5570366fa --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,71 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/python +{ + "dockerComposeFile": "docker-compose.yml", + "service": "devcontainer", + "workspaceFolder": "/workspaces/UKBinCollectionData", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "eamodio.gitlens", + "ms-python.black-formatter", + "ms-python.isort", + "ms-python.python", + "ryanluker.vscode-coverage-gutters", + "Gruntfuggly.todo-tree", + "oderwat.indent-rainbow", + "yzhang.markdown-all-in-one", + "alexkrechik.cucumberautocomplete" + ], + "settings": { + "makefile.makefilePath": "${workspaceFolder}/", + "files.exclude": { + "**/__pycache__": true, + "**/.pytest_cache": true + }, + "autoSave": true, + "git.autorefresh": true, + "extensions.ignoreRecommendations": true, + "isort.args": [ + "--profile", + "black" + ], + "python.analysis.diagnosticMode": "workspace", + "python.analysis.typeCheckingMode": "strict", + "python.analysis.logLevel": "Trace", + "python.analysis.disableSemanticOnNoPython": false, + "python.analysis.enableSyncServer": true, + "python.analysis.userFileIndexingLimit": -1, + "python.formatting.provider": "none", + "python.languageServer": "Pylance", + "python.linting.enabled": false, + "python.linting.lintOnSave": false, + "python.testing.autoTestDiscoverOnSaveEnabled": false, + "python.defaultInterpreterPath": "/home/vscode/ukbc-py-env", + "python.testing.pytestArgs": [ + "${workspaceFolder}/uk_bin_collection", + "--headless=False", + "-o cache_dir=${workspaceFolder}/.pytest_cache" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true, + "editor.formatOnPaste": false, + "editor.formatOnSaveMode": "file", + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + "workbench.colorCustomizations": { + "editorError.foreground": "#ff000088", + "editorWarning.foreground": "#ffe60033", + "editorInfo.foreground": "#00ff0088" + } + } + } + } +} \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 0000000000..6f9061fb95 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,63 @@ +services: + devcontainer: + image: ukbc_dev_container # This tags the built image + build: + context: ../ # Path to the directory containing the Dockerfile + dockerfile: .devcontainer/dev.Dockerfile + volumes: + - ../:/workspaces/UKBinCollectionData:rw + privileged: true + hostname: devcontainer + network_mode: host + depends_on: + - selenium-hub + command: sleep infinity + + chrome: + image: selenium/node-chrome:4.20.0-20240505 + shm_size: 2gb + networks: + - devnet + depends_on: + - selenium-hub + ports: + - "7900:7900" + - "5555:5555" + environment: + - SE_EVENT_BUS_HOST=selenium-hub + - SE_EVENT_BUS_PUBLISH_PORT=4442 + - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 + privileged: true + restart: always + + chrome_video: + image: selenium/video:ffmpeg-6.1.1-20240505 + networks: + - devnet + volumes: + - ../test_videos:/videos/ + depends_on: + - chrome + environment: + - DISPLAY_CONTAINER_NAME=chrome + - SE_VIDEO_FILE_NAME=auto + - SE_NODE_GRID_URL=http://selenium-hub:4444 + privileged: true + restart: always + + selenium-hub: + image: selenium/hub:4.20.0-20240505 + container_name: selenium-hub + hostname: selenium + ports: + - "4442:4442" + - "4443:4443" + - "4444:4444" + privileged: true + restart: always + networks: + - devnet + +networks: + devnet: + driver: bridge diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..47f47ff2e2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,38 @@ +# Ignore everything +* + +# But not these files... +!*.json +!*.py +!PipFile +!Pipfile.lock +!.gitignore +!.dockerignore +!*.toml +!*.md +!*.rst +!LICENSE +!*.schema +!Makefile +!dependabot.yaml +!poetry.lock +!behave.ini +!*.Dockerfile + +# Or these folders... +!.github +!*.png +!.github/ISSUE_TEMPLATE +!.github/ISSUE_TEMPLATE/*.yaml +!.github/workflows +!.github/workflows/*.yml +!uk_bin_collection +!uk_bin_collection/**/* +!uk_bin_collection_api_server +!uk_bin_collection_api_server/**/* +!wiki +!wiki/**/* +!custom_components +__pycache__ +!TO_BE_CONVERTED +!.devcontainer diff --git a/.github/workflows/behave.yml b/.github/workflows/behave.yml index 8a49bf1d47..16902c4592 100644 --- a/.github/workflows/behave.yml +++ b/.github/workflows/behave.yml @@ -33,7 +33,9 @@ jobs: services: selenium: image: selenium/standalone-chrome:latest - options: --shm-size=2gb + options: --shm-size=2gb --name selenium --hostname selenium + ports: + - 4444:4444 steps: - uses: actions/checkout@v4 diff --git a/.gitignore b/.gitignore index 86908d7504..1188989e2f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,9 @@ !PipFile !Pipfile.lock !.gitignore +!.dockerignore !*.toml -!.md +!*.md !*.rst !LICENSE !*.schema @@ -16,6 +17,9 @@ !dependabot.yaml !poetry.lock !behave.ini +!*.Dockerfile +!docker-compose.yml +!.vscode/launch.json # Or these folders... !.github @@ -33,4 +37,4 @@ !custom_components __pycache__ !TO_BE_CONVERTED - +!.devcontainer diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000000..45a39cb626 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,14 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: Current File", + "type": "debugpy", + "request": "launch", + "purpose": ["debug-test"], + "env": { + "PYTEST_ADDOPTS": "--headless=False --local_browser=False" + } + } + ] +} \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eaa4f83dc8..c197353775 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -89,6 +89,46 @@ There are a few different options for scraping, and you are free to choose which ## Developing To get started, first you will need to fork this repository and setup your own working environment before you can start developing. +### Developing using our Dev Container +You need to set up Docker, Visual Studio Code (VSCode), and a development container (devcontainer) after cloning the repository at https://github.com/robbrad/UKBinCollectionData. + +#### Prerequisites +Before you start, make sure you have the following installed on your computer: +- Docker: [Download Docker](https://www.docker.com/products/docker-desktop) +- Visual Studio Code (VSCode): [Download VSCode](https://code.visualstudio.com/download) +- Remote - Containers extension for VSCode: Install it from the VSCode Marketplace or directly from the Extensions view (`Ctrl+Shift+X` in VSCode and search for "Remote - Containers"). + +#### Step 1: Clone the Repository +First, clone the repository to your local machine. Open a terminal and run the following command: +```bash +git clone https://github.com/robbrad/UKBinCollectionData.git +``` +Navigate into the directory: +```bash +cd UKBinCollectionData +``` + +#### Step 2: Set Up Docker +Ensure Docker is running on your system. You can verify this by running: +```bash +docker -v +``` +This should return the version of Docker installed. If Docker is running, you’ll see no errors. + +#### Step 3: Open the Project in VSCode +Open VSCode, and then open the cloned repository by going to `File > Open Folder...` and selecting the `UKBinCollectionData` folder. + +#### Step 4: Reopen in Container +Once the folder is open in VSCode: +1. A prompt might appear asking you to reopen in a container. If it does, select "Reopen in Container". +2. If you don’t see the prompt, press `F1` to open the command palette, type "Remote-Containers: Reopen in Container", and select that option. + +VSCode will start building the Docker container as defined in the `.devcontainer/` folder in the repository. This process can take a few minutes as it involves downloading the base Docker and Selenium hub images and setting up the environment. + +#### Step 5: Verify the Development Environment +Once the container is set up, VSCode will connect to it automatically. You can start editing and running the code inside the container. This ensures that your development environment is consistent and controlled, replicating the same settings and tools as specified in the devcontainer configuration. + +### Developing Once your environment is ready, create a new branch from your master/main branch and then create a new .py file within the `uk_bin_collection\councils` directory then use the development mode to generate the input.json entry. The new .py file will be used in the CLI to call the parser, so be sure to pick a sensible name - e.g. CheshireEastCouncil.py is called with: ``` python collect_data.py CheshireEastCouncil @@ -225,20 +265,20 @@ Feature: Test each council output matches expected results Scenario Outline: Validate Council Output Given the council: - When we scrape the data from using and the is set + When we scrape the data from Then the result is valid json And the output should validate against the schema @AylesburyValeCouncil - Examples: AylesburyValeCouncil - | council | selenium_url | selenium_mode | - | AylesburyValeCouncil | None | None | + Examples: AylesburyValeCouncil + | council | + | AylesburyValeCouncil | None | None | @BarnetCouncil Examples: BarnetCouncil - | council | selenium_url | selenium_mode | - | BarnetCouncil | http://selenium:4444 | local | + | council | + | BarnetCouncil | ``` @@ -252,22 +292,38 @@ file. Based on the [input.json](https://github.com/robbrad/UKBinCollectionData/blob/master/uk_bin_collection/tests/input.json), this does an actual live run against the council's site and validates if the returned data is JSON and conforms to the common format [JSON Schema](https://github.com/robbrad/UKBinCollectionData/tree/master/uk_bin_collection/tests/output.schema). -By default if the council is a Selenium based council it will run in headless mode. If you pass `--headless=False` to pytest (possible in VS Code via the workspace settings.json useful for debugging code) It will run in headless. +By default if the council is a Selenium based council it will run in headless mode. If you pass `--headless=False` to pytest (possible in VS Code launch.json useful for debugging code) It will run in a visable browser. -``` +It also defaults the Selenium URL to be `http://localhost:4444` and the local_browser to False + +You can set pytest to test on your local web browser without Selenium Grid by setting `--local_browser=True` +If you want a different Selenium URL you can set it with `--selenium_url=http://selenium:4444` NOTE: you can't set `--local_browser=True` (defaults: False) as Selenium testing will be ignored + +In VSCode if you set a make a launch.json you can debug the test locally with the following setup +```json { - "python.testing.pytestArgs": [ - "uk_bin_collection", - "--headless=True" - ], - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: Current File", + "type": "debugpy", + "request": "launch", + "purpose": ["debug-test"], + "env": { + "PYTEST_ADDOPTS": "--headless=False --local_browser=True" + } + } + ] } ``` It is also possible to run -``` -poetry run pytest uk_bin_collection/tests/step_defs/ -k "Council_Name" --headless=False +```commandline +#Visable Selenium Run in Local Broswer +poetry run pytest uk_bin_collection/tests/step_defs/ -k "Council_Name" --headless=False --local_browser=True + +#Visable Selenium Run in on Selenium Grid +poetry run pytest uk_bin_collection/tests/step_defs/ -k "Council_Name" --headless=False --selenium_url=http://localhost:4444 ``` #### Running the Behave tests for all councils diff --git a/conftest.py b/conftest.py index c771779604..f3d7b29cbb 100644 --- a/conftest.py +++ b/conftest.py @@ -1,11 +1,23 @@ import pytest +from _pytest.config.argparsing import Parser +from _pytest.fixtures import FixtureRequest -def pytest_addoption(parser): - parser.addoption("--headless", action="store", default="True") +def pytest_addoption(parser: Parser) -> None: + parser.addoption("--headless", action="store", default="True", type=str) + parser.addoption("--local_browser", action="store", default="False", type=str) + parser.addoption("--selenium_url", action="store", default="http://localhost:4444", type=str) @pytest.fixture(scope='session') -def headless_mode(request): - headless_mode_value = request.config.option.headless - if headless_mode_value is None: - pytest.skip() - return headless_mode_value \ No newline at end of file +def headless_mode(request: FixtureRequest) -> str: + headless_mode_value = str(request.config.getoption("--headless")) + return headless_mode_value + +@pytest.fixture(scope='session') +def local_browser(request: FixtureRequest) -> str: + local_browser_value = str(request.config.getoption("--local_browser")) + return local_browser_value + +@pytest.fixture(scope='session') +def selenium_url(request: FixtureRequest) -> str: + selenium_url_value = str(request.config.getoption("--selenium_url")) + return selenium_url_value \ No newline at end of file diff --git a/uk_bin_collection/tests/features/validate_council_outputs.feature b/uk_bin_collection/tests/features/validate_council_outputs.feature index d1a52a2a2e..3b23918a3e 100644 --- a/uk_bin_collection/tests/features/validate_council_outputs.feature +++ b/uk_bin_collection/tests/features/validate_council_outputs.feature @@ -2,766 +2,766 @@ Feature: Test each council output matches expected results Scenario Outline: Validate Council Output Given the council: - When we scrape the data from using and the is set + When we scrape the data from Then the result is valid json And the output should validate against the schema @AdurAndWorthingCouncils Examples: AdurAndWorthingCouncils - | council | selenium_url | selenium_mode | - | AdurAndWorthingCouncils | None | None | + | council | + | AdurAndWorthingCouncils | @ArunCouncil Examples: ArunCouncil - | council | selenium_url | selenium_mode | - | ArunCouncil | http://selenium:4444 | local | + | council | + | ArunCouncil | @AylesburyValeCouncil Examples: AylesburyValeCouncil - | council | selenium_url | selenium_mode | - | AylesburyValeCouncil | None | None | + | council | + | AylesburyValeCouncil | @BarnetCouncil Examples: BarnetCouncil - | council | selenium_url | selenium_mode | - | BarnetCouncil | http://selenium:4444 | local | + | council | + | BarnetCouncil | @BarnsleyMBCouncil Examples: BarnsleyMBCouncil - | council | selenium_url | selenium_mode | - | BarnsleyMBCouncil | None | None | + | council | + | BarnsleyMBCouncil | @BasingstokeCouncil Examples: BasingstokeCouncil - | council | selenium_url | selenium_mode | - | BasingstokeCouncil | None | None | + | council | + | BasingstokeCouncil | @BathAndNorthEastSomersetCouncil Examples: BathAndNorthEastSomersetCouncil - | council | selenium_url | selenium_mode | - | BathAndNorthEastSomersetCouncil | None | None | + | council | + | BathAndNorthEastSomersetCouncil | @BCPCouncil Examples: BCPCouncil - | council | selenium_url | selenium_mode | - | BCPCouncil | None | None | + | council | + | BCPCouncil | @BedfordBoroughCouncil Examples: BedfordBoroughCouncil - | council | selenium_url | selenium_mode | - | BedfordBoroughCouncil | None | None | + | council | + | BedfordBoroughCouncil | @BedfordshireCouncil Examples: BedfordshireCouncil - | council | selenium_url | selenium_mode | - | BedfordshireCouncil | None | None | + | council | + | BedfordshireCouncil | @BexleyCouncil Examples: BexleyCouncil - | council | selenium_url | selenium_mode | - | BexleyCouncil | http://selenium:4444 | local | + | council | + | BexleyCouncil | @BirminghamCityCouncil Examples: BirminghamCityCouncil - | council | selenium_url | selenium_mode | - | BirminghamCityCouncil | None | None | + | council | + | BirminghamCityCouncil | @BlackburnCouncil Examples: BlackburnCouncil - | council | selenium_url | selenium_mode | - | BlackburnCouncil | http://selenium:4444 | local | + | council | + | BlackburnCouncil | @BoltonCouncil Examples: BoltonCouncil - | council | selenium_url | selenium_mode | - | BoltonCouncil | http://selenium:4444 | local | + | council | + | BoltonCouncil | @BradfordMDC Examples: BradfordMDC - | council | selenium_url | selenium_mode | - | BradfordMDC | None | None | - + | council | + | BradfordMDC | + @BrightonandHoveCityCouncil Examples: BrightonandHoveCityCouncil - | council | selenium_url | selenium_mode | - | BrightonandHoveCityCouncil | http://selenium:4444 | local | + | council | + | BrightonandHoveCityCouncil | @BristolCityCouncil Examples: BristolCityCouncil - | council | selenium_url | selenium_mode | - | BristolCityCouncil | None | None | + | council | + | BristolCityCouncil | @BromleyBoroughCouncil Examples: BromleyBoroughCouncil - | council | selenium_url | selenium_mode | - | BromleyBoroughCouncil | http://selenium:4444 | local | + | council | + | BromleyBoroughCouncil | @BroxtoweBoroughCouncil Examples: BroxtoweBoroughCouncil - | council | selenium_url | selenium_mode | - | BroxtoweBoroughCouncil | http://selenium:4444 | local | + | council | + | BroxtoweBoroughCouncil | @BuckinghamshireCouncil Examples: BuckinghamshireCouncil - | council | selenium_url | selenium_mode | - | BuckinghamshireCouncil | http://selenium:4444 | local | + | council | + | BuckinghamshireCouncil | @BuryCouncil Examples: BuryCouncil - | council | selenium_url | selenium_mode | - | BuryCouncil | None | None | + | council | + | BuryCouncil | @CalderdaleCouncil Examples: CalderdaleCouncil - | council | selenium_url | selenium_mode | - | CalderdaleCouncil | http://selenium:4444 | local | + | council | + | CalderdaleCouncil | @CannockChaseDistrictCouncil Examples: CannockChaseDistrictCouncil - | council | selenium_url | selenium_mode | - | CannockChaseDistrictCouncil | None | None | + | council | + | CannockChaseDistrictCouncil | @CardiffCouncil Examples: CardiffCouncil - | council | selenium_url | selenium_mode | - | CardiffCouncil | None | None | + | council | + | CardiffCouncil | @CastlepointDistrictCouncil Examples: CastlepointDistrictCouncil - | council | selenium_url | selenium_mode | - | CastlepointDistrictCouncil | None | None | + | council | + | CastlepointDistrictCouncil | @CharnwoodBoroughCouncil Examples: CharnwoodBoroughCouncil - | council | selenium_url | selenium_mode | - | CharnwoodBoroughCouncil | None | None | + | council | + | CharnwoodBoroughCouncil | @ChelmsfordCityCouncil Examples: ChelmsfordCityCouncil - | council | selenium_url | selenium_mode | - | ChelmsfordCityCouncil | http://selenium:4444 | local | + | council | + | ChelmsfordCityCouncil | @CheshireEastCouncil Examples: CheshireEastCouncil - | council | selenium_url | selenium_mode | - | CheshireEastCouncil | None | None | + | council | + | CheshireEastCouncil | @CheshireWestAndChesterCouncil Examples: CheshireWestAndChesterCouncil - | council | selenium_url | selenium_mode | - | CheshireWestAndChesterCouncil | http://selenium:4444 | local | + | council | + | CheshireWestAndChesterCouncil | @ChorleyCouncil Examples: ChorleyCouncil - | council | selenium_url | selenium_mode | - | ChorleyCouncil | http://selenium:4444 | local | + | council | + | ChorleyCouncil | @ConwyCountyBorough Examples: ConwyCountyBorough - | council | selenium_url | selenium_mode | - | ConwyCountyBorough | None | None | + | council | + | ConwyCountyBorough | @CrawleyBoroughCouncil Examples: CrawleyBoroughCouncil - | council | selenium_url | selenium_mode | - | CrawleyBoroughCouncil | None | None | + | council | + | CrawleyBoroughCouncil | @CroydonCouncil Examples: CroydonCouncil - | council | selenium_url | selenium_mode | - | CroydonCouncil | None | None | + | council | + | CroydonCouncil | @DerbyshireDalesDistrictCouncil Examples: DerbyshireDalesDistrictCouncil - | council | selenium_url | selenium_mode | - | DerbyshireDalesDistrictCouncil | http://selenium:4444 | local | + | council | + | DerbyshireDalesDistrictCouncil | @DoncasterCouncil Examples: DoncasterCouncil - | council | selenium_url | selenium_mode | - | DoncasterCouncil | None | None | + | council | + | DoncasterCouncil | @DorsetCouncil Examples: DorsetCouncil - | council | selenium_url | selenium_mode | - | DorsetCouncil | None | None | + | council | + | DorsetCouncil | @DoverDistrictCouncil Examples: DoverDistrictCouncil - | council | selenium_url | selenium_mode | - | DoverDistrictCouncil | None | None | + | council | + | DoverDistrictCouncil | @DurhamCouncil Examples: DurhamCouncil - | council | selenium_url | selenium_mode | - | DurhamCouncil | None | None | + | council | + | DurhamCouncil | @EastCambridgeshireCouncil Examples: EastCambridgeshireCouncil - | council | selenium_url | selenium_mode | - | EastCambridgeshireCouncil | None | None | + | council | + | EastCambridgeshireCouncil | @EastDevonDC Examples: EastDevonDC - | council | selenium_url | selenium_mode | - | EastDevonDC | None | None | + | council | + | EastDevonDC | @EastleighBoroughCouncil Examples: EastleighBoroughCouncil - | council | selenium_url | selenium_mode | - | EastleighBoroughCouncil | None | None | + | council | + | EastleighBoroughCouncil | @EastLindseyDistrictCouncil Examples: EastLindseyDistrictCouncil - | council | selenium_url | selenium_mode | - | EastLindseyDistrictCouncil | http://selenium:4444 | local | + | council | + | EastLindseyDistrictCouncil | @EastRidingCouncil Examples: EastRidingCouncil - | council | selenium_url | selenium_mode | - | EastRidingCouncil | http://selenium:4444 | local | + | council | + | EastRidingCouncil | @EastSuffolkCouncil Examples: EastSuffolkCouncil - | council | selenium_url | selenium_mode | - | EastSuffolkCouncil | http://selenium:4444 | local | + | council | + | EastSuffolkCouncil | @EnvironmentFirst Examples: EnvironmentFirst - | council | selenium_url | selenium_mode | - | EnvironmentFirst | None | None | + | council | + | EnvironmentFirst | @ErewashBoroughCouncil Examples: ErewashBoroughCouncil - | council | selenium_url | selenium_mode | - | ErewashBoroughCouncil | None | None | + | council | + | ErewashBoroughCouncil | @FenlandDistrictCouncil Examples: FenlandDistrictCouncil - | council | selenium_url | selenium_mode | - | FenlandDistrictCouncil | None | None | + | council | + | FenlandDistrictCouncil | @ForestOfDeanDistrictCouncil Examples: ForestOfDeanDistrictCouncil - | council | selenium_url | selenium_mode | - | ForestOfDeanDistrictCouncil | http://selenium:4444 | local | + | council | + | ForestOfDeanDistrictCouncil | @GatesheadCouncil Examples: GatesheadCouncil - | council | selenium_url | selenium_mode | - | GatesheadCouncil | http://selenium:4444 | local | + | council | + | GatesheadCouncil | @GedlingBoroughCouncil Examples: GedlingBoroughCouncil - | council | selenium_url | selenium_mode | - | GedlingBoroughCouncil | None | None | + | council | + | GedlingBoroughCouncil | @GlasgowCityCouncil Examples: GlasgowCityCouncil - | council | selenium_url | selenium_mode | - | GlasgowCityCouncil | None | None | + | council | + | GlasgowCityCouncil | @GuildfordCouncil Examples: GuildfordCouncil - | council | selenium_url | selenium_mode | - | GuildfordCouncil | http://selenium:4444 | local | + | council | + | GuildfordCouncil | @HaltonBoroughCouncil Examples: HaltonBoroughCouncil - | council | selenium_url | selenium_mode | - | HaltonBoroughCouncil | http://selenium:4444 | local | + | council | + | HaltonBoroughCouncil | @HaringeyCouncil Examples: HaringeyCouncil - | council | selenium_url | selenium_mode | - | HaringeyCouncil | None | None | + | council | + | HaringeyCouncil | @HarrogateBoroughCouncil Examples: HarrogateBoroughCouncil - | council | selenium_url | selenium_mode | - | HarrogateBoroughCouncil | None | None | + | council | + | HarrogateBoroughCouncil | @HighPeakCouncil Examples: HighPeakCouncil - | council | selenium_url | selenium_mode | - | HighPeakCouncil | http://selenium:4444 | local | + | council | + | HighPeakCouncil | @HullCityCouncil Examples: HullCityCouncil - | council | selenium_url | selenium_mode | - | HullCityCouncil | None | None | + | council | + | HullCityCouncil | @HuntingdonDistrictCouncil Examples: HuntingdonDistrictCouncil - | council | selenium_url | selenium_mode | - | HuntingdonDistrictCouncil | None | None | + | council | + | HuntingdonDistrictCouncil | @KingstonUponThamesCouncil Examples: KingstonUponThamesCouncil - | council | selenium_url | selenium_mode | - | KingstonUponThamesCouncil | None | None | + | council | + | KingstonUponThamesCouncil | @KirkleesCouncil Examples: KirkleesCouncil - | council | selenium_url | selenium_mode | - | KirkleesCouncil | http://selenium:4444 | local | + | council | + | KirkleesCouncil | @KnowsleyMBCouncil Examples: KnowsleyMBCouncil - | council | selenium_url | selenium_mode | - | KnowsleyMBCouncil | http://selenium:4444 | local | + | council | + | KnowsleyMBCouncil | @LancasterCityCouncil Examples: LancasterCityCouncil - | council | selenium_url | selenium_mode | - | LancasterCityCouncil | None | None | + | council | + | LancasterCityCouncil | @LeedsCityCouncil Examples: LeedsCityCouncil - | council | selenium_url | selenium_mode | - | LeedsCityCouncil | http://selenium:4444 | local | + | council | + | LeedsCityCouncil | @LisburnCastlereaghCityCouncil Examples: LisburnCastlereaghCityCouncil - | council | selenium_url | selenium_mode | - | LisburnCastlereaghCityCouncil | None | None | + | council | + | LisburnCastlereaghCityCouncil | @LiverpoolCityCouncil Examples: LiverpoolCityCouncil - | council | selenium_url | selenium_mode | - | LiverpoolCityCouncil | None | None | + | council | + | LiverpoolCityCouncil | @LondonBoroughHounslow Examples: LondonBoroughHounslow - | council | selenium_url | selenium_mode | - | LondonBoroughHounslow | None | None | + | council | + | LondonBoroughHounslow | @LondonBoroughRedbridge Examples: LondonBoroughRedbridge - | council | selenium_url | selenium_mode | - | LondonBoroughRedbridge | http://selenium:4444 | local | - + | council | + | LondonBoroughRedbridge | + @MaldonDistrictCouncil Examples: MaldonDistrictCouncil - | council | selenium_url | selenium_mode | - | MaldonDistrictCouncil | None | None | + | council | + | MaldonDistrictCouncil | @MalvernHillsDC Examples: MalvernHillsDC - | council | selenium_url | selenium_mode | - | MalvernHillsDC | None | None | + | council | + | MalvernHillsDC | @ManchesterCityCouncil Examples: ManchesterCityCouncil - | council | selenium_url | selenium_mode | - | ManchesterCityCouncil | None | None | + | council | + | ManchesterCityCouncil | @MansfieldDistrictCouncil Examples: MansfieldDistrictCouncil - | council | selenium_url | selenium_mode | - | MansfieldDistrictCouncil | None | None | + | council | + | MansfieldDistrictCouncil | @MertonCouncil Examples: MertonCouncil - | council | selenium_url | selenium_mode | - | MertonCouncil | None | None | + | council | + | MertonCouncil | @MidAndEastAntrimBoroughCouncil Examples: MidAndEastAntrimBoroughCouncil - | council | selenium_url | selenium_mode | - | MidAndEastAntrimBoroughCouncil | http://selenium:4444 | local | + | council | + | MidAndEastAntrimBoroughCouncil | @MidSussexDistrictCouncil Examples: MidSussexDistrictCouncil - | council | selenium_url | selenium_mode | - | MidSussexDistrictCouncil | None | None | + | council | + | MidSussexDistrictCouncil | @MiltonKeynesCityCouncil Examples: MiltonKeynesCityCouncil - | council | selenium_url | selenium_mode | - | MiltonKeynesCityCouncil | None | None | + | council | + | MiltonKeynesCityCouncil | @MoleValleyDistrictCouncil Examples: MoleValleyDistrictCouncil - | council | selenium_url | selenium_mode | - | MoleValleyDistrictCouncil | None | None | + | council | + | MoleValleyDistrictCouncil | @NeathPortTalbotCouncil Examples: NeathPortTalbotCouncil - | council | selenium_url | selenium_mode | - | NeathPortTalbotCouncil | http://selenium:4444 | local | + | council | + | NeathPortTalbotCouncil | @NewarkAndSherwoodDC Examples: NewarkAndSherwoodDC - | council | selenium_url | selenium_mode | - | NewarkAndSherwoodDC | None | None | + | council | + | NewarkAndSherwoodDC | @NewcastleCityCouncil Examples: NewcastleCityCouncil - | council | selenium_url | selenium_mode | - | NewcastleCityCouncil | None | None | + | council | + | NewcastleCityCouncil | @NewhamCouncil Examples: NewhamCouncil - | council | selenium_url | selenium_mode | - | NewhamCouncil | None | None | + | council | + | NewhamCouncil | @NewportCityCouncil Examples: NewportCityCouncil - | council | selenium_url | selenium_mode | - | NewportCityCouncil | None | None | + | council | + | NewportCityCouncil | @NorthEastDerbyshireDistrictCouncil Examples: NorthEastDerbyshireDistrictCouncil - | council | selenium_url | selenium_mode | - | NorthEastDerbyshireDistrictCouncil | http://selenium:4444 | local | + | council | + | NorthEastDerbyshireDistrictCouncil | @NorthEastLincs Examples: NorthEastLincs - | council | selenium_url | selenium_mode | - | NorthEastLincs | None | None | + | council | + | NorthEastLincs | @NorthKestevenDistrictCouncil Examples: NorthKestevenDistrictCouncil - | council | selenium_url | selenium_mode | - | NorthKestevenDistrictCouncil | None | None | + | council | + | NorthKestevenDistrictCouncil | @NorthLanarkshireCouncil Examples: NorthLanarkshireCouncil - | council | selenium_url | selenium_mode | - | NorthLanarkshireCouncil | None | None | + | council | + | NorthLanarkshireCouncil | @NorthLincolnshireCouncil Examples: NorthLincolnshireCouncil - | council | selenium_url | selenium_mode | - | NorthLincolnshireCouncil | None | None | + | council | + | NorthLincolnshireCouncil | @NorthNorfolkDistrictCouncil Examples: NorthNorfolkDistrictCouncil - | council | selenium_url | selenium_mode | - | NorthNorfolkDistrictCouncil | http://selenium:4444 | local | + | council | + | NorthNorfolkDistrictCouncil | @NorthNorthamptonshireCouncil Examples: NorthNorthamptonshireCouncil - | council | selenium_url | selenium_mode | - | NorthNorthamptonshireCouncil | None | None | + | council | + | NorthNorthamptonshireCouncil | @NorthSomersetCouncil Examples: NorthSomersetCouncil - | council | selenium_url | selenium_mode | - | NorthSomersetCouncil | None | None | + | council | + | NorthSomersetCouncil | @NorthTynesideCouncil Examples: NorthTynesideCouncil - | council | selenium_url | selenium_mode | - | NorthTynesideCouncil | None | None | + | council | + | NorthTynesideCouncil | @NorthumberlandCouncil Examples: NorthumberlandCouncil - | council | selenium_url | selenium_mode | - | NorthumberlandCouncil | http://selenium:4444 | local | + | council | + | NorthumberlandCouncil | @NorthWestLeicestershire Examples: NorthWestLeicestershire - | council | selenium_url | selenium_mode | - | NorthWestLeicestershire | http://selenium:4444 | local | + | council | + | NorthWestLeicestershire | @NorthYorkshire Examples: NorthYorkshire - | council | selenium_url | selenium_mode | - | NorthYorkshire | None | None | + | council | + | NorthYorkshire | @NottinghamCityCouncil Examples: NottinghamCityCouncil - | council | selenium_url | selenium_mode | - | NottinghamCityCouncil | None | None | + | council | + | NottinghamCityCouncil | @OldhamCouncil Examples: OldhamCouncil - | council | selenium_url | selenium_mode | - | OldhamCouncil | None | None | + | council | + | OldhamCouncil | @PortsmouthCityCouncil Examples: PortsmouthCityCouncil - | council | selenium_url | selenium_mode | - | PortsmouthCityCouncil | http://selenium:4444 | local | + | council | + | PortsmouthCityCouncil | @PrestonCityCouncil Examples: PrestonCityCouncil - | council | selenium_url | selenium_mode | - | PrestonCityCouncil | http://selenium:4444 | local | + | council | + | PrestonCityCouncil | @ReadingBoroughCouncil Examples: ReadingBoroughCouncil - | council | selenium_url | selenium_mode | - | ReadingBoroughCouncil | None | None | + | council | + | ReadingBoroughCouncil | @ReigateAndBansteadBoroughCouncil Examples: ReigateAndBansteadBoroughCouncil - | council | selenium_url | selenium_mode | - | ReigateAndBansteadBoroughCouncil | http://selenium:4444 | local | + | council | + | ReigateAndBansteadBoroughCouncil | @RenfrewshireCouncil Examples: RenfrewshireCouncil - | council | selenium_url | selenium_mode | - | RenfrewshireCouncil | http://selenium:4444 | local | + | council | + | RenfrewshireCouncil | @RhonddaCynonTaffCouncil Examples: RhonddaCynonTaffCouncil - | council | selenium_url | selenium_mode | - | RhonddaCynonTaffCouncil | None | None | + | council | + | RhonddaCynonTaffCouncil | @RochdaleCouncil Examples: RochdaleCouncil - | council | selenium_url | selenium_mode | - | RochdaleCouncil | None | None | + | council | + | RochdaleCouncil | @RochfordCouncil Examples: RochfordCouncil - | council | selenium_url | selenium_mode | - | RochfordCouncil | None | None | + | council | + | RochfordCouncil | @RugbyBoroughCouncil Examples: RugbyBoroughCouncil - | council | selenium_url | selenium_mode | - | RugbyBoroughCouncil | None | None | + | council | + | RugbyBoroughCouncil | @RushcliffeBoroughCouncil Examples: RushcliffeBoroughCouncil - | council | selenium_url | selenium_mode | - | RushcliffeBoroughCouncil | http://selenium:4444 | local | + | council | + | RushcliffeBoroughCouncil | @RushmoorCouncil Examples: RushmoorCouncil - | council | selenium_url | selenium_mode | - | RushmoorCouncil | None | None | + | council | + | RushmoorCouncil | @SalfordCityCouncil Examples: SalfordCityCouncil - | council | selenium_url | selenium_mode | - | SalfordCityCouncil | None | None | + | council | + | SalfordCityCouncil | @SevenoaksDistrictCouncil Examples: SevenoaksDistrictCouncil - | council | selenium_url | selenium_mode | - | SevenoaksDistrictCouncil | http://selenium:4444 | local | + | council | + | SevenoaksDistrictCouncil | @SheffieldCityCouncil Examples: SheffieldCityCouncil - | council | selenium_url | selenium_mode | - | SheffieldCityCouncil | None | None | + | council | + | SheffieldCityCouncil | @ShropshireCouncil Examples: ShropshireCouncil - | council | selenium_url | selenium_mode | - | ShropshireCouncil | None | None | + | council | + | ShropshireCouncil | @SolihullCouncil Examples: SolihullCouncil - | council | selenium_url | selenium_mode | - | SolihullCouncil | None | None | + | council | + | SolihullCouncil | @SomersetCouncil Examples: SomersetCouncil - | council | selenium_url | selenium_mode | - | SomersetCouncil | None | None | + | council | + | SomersetCouncil | @SouthAyrshireCouncil Examples: SouthAyrshireCouncil - | council | selenium_url | selenium_mode | - | SouthAyrshireCouncil | None | None | + | council | + | SouthAyrshireCouncil | @SouthCambridgeshireCouncil Examples: SouthCambridgeshireCouncil - | council | selenium_url | selenium_mode | - | SouthCambridgeshireCouncil | None | None | + | council | + | SouthCambridgeshireCouncil | @SouthGloucestershireCouncil Examples: SouthGloucestershireCouncil - | council | selenium_url | selenium_mode | - | SouthGloucestershireCouncil | None | None | + | council | + | SouthGloucestershireCouncil | @SouthLanarkshireCouncil Examples: SouthLanarkshireCouncil - | council | selenium_url | selenium_mode | - | SouthLanarkshireCouncil | None | None | + | council | + | SouthLanarkshireCouncil | @SouthNorfolkCouncil Examples: SouthNorfolkCouncil - | council | selenium_url | selenium_mode | - | SouthNorfolkCouncil | None | None | + | council | + | SouthNorfolkCouncil | @SouthOxfordshireCouncil Examples: SouthOxfordshireCouncil - | council | selenium_url | selenium_mode | - | SouthOxfordshireCouncil | None | None | + | council | + | SouthOxfordshireCouncil | @SouthTynesideCouncil Examples: SouthTynesideCouncil - | council | selenium_url | selenium_mode | - | SouthTynesideCouncil | None | None | + | council | + | SouthTynesideCouncil | @StAlbansCityAndDistrictCouncil Examples: StAlbansCityAndDistrictCouncil - | council | selenium_url | selenium_mode | - | StAlbansCityAndDistrictCouncil | None | None | + | council | + | StAlbansCityAndDistrictCouncil | @StaffordshireMoorlandsDistrictCouncil Examples: StaffordshireMoorlandsDistrictCouncil - | council | selenium_url | selenium_mode | - | StaffordshireMoorlandsDistrictCouncil | http://selenium:4444 | local | + | council | + | StaffordshireMoorlandsDistrictCouncil | @StHelensBC Examples: StHelensBC - | council | selenium_url | selenium_mode | - | StHelensBC | None | None | + | council | + | StHelensBC | @StockportBoroughCouncil Examples: StockportBoroughCouncil - | council | selenium_url | selenium_mode | - | StockportBoroughCouncil | None | None | + | council | + | StockportBoroughCouncil | @StokeOnTrentCityCouncil Examples: StokeOnTrentCityCouncil - | council | selenium_url | selenium_mode | - | StokeOnTrentCityCouncil | None | None | + | council | + | StokeOnTrentCityCouncil | @StratfordUponAvonCouncil Examples: StratfordUponAvonCouncil - | council | selenium_url | selenium_mode | - | StratfordUponAvonCouncil | None | None | + | council | + | StratfordUponAvonCouncil | @SunderlandCityCouncil Examples: SunderlandCityCouncil - | council | selenium_url | selenium_mode | - | SunderlandCityCouncil | http://selenium:4444 | local | + | council | + | SunderlandCityCouncil | @SwaleBoroughCouncil Examples: SwaleBoroughCouncil - | council | selenium_url | selenium_mode | - | SwaleBoroughCouncil | None | None | + | council | + | SwaleBoroughCouncil | @TamesideMBCouncil Examples: TamesideMBCouncil - | council | selenium_url | selenium_mode | - | TamesideMBCouncil | None | None | + | council | + | TamesideMBCouncil | @TandridgeDistrictCouncil Examples: TandridgeDistrictCouncil - | council | selenium_url | selenium_mode | - | TandridgeDistrictCouncil | None | None | + | council | + | TandridgeDistrictCouncil | @TelfordAndWrekinCouncil Examples: TelfordAndWrekinCouncil - | council | selenium_url | selenium_mode | - | TelfordAndWrekinCouncil | None | None | + | council | + | TelfordAndWrekinCouncil | @TestValleyBoroughCouncil Examples: TestValleyBoroughCouncil - | council | selenium_url | selenium_mode | - | TestValleyBoroughCouncil | None | None | + | council | + | TestValleyBoroughCouncil | @TonbridgeAndMallingBC Examples: TonbridgeAndMallingBC - | council | selenium_url | selenium_mode | - | TonbridgeAndMallingBC | None | None | + | council | + | TonbridgeAndMallingBC | @TorbayCouncil Examples: TorbayCouncil - | council | selenium_url | selenium_mode | - | TorbayCouncil | None | None | + | council | + | TorbayCouncil | @TorridgeDistrictCouncil Examples: TorridgeDistrictCouncil - | council | selenium_url | selenium_mode | - | TorridgeDistrictCouncil | None | None | + | council | + | TorridgeDistrictCouncil | @ValeofGlamorganCouncil Examples: ValeofGlamorganCouncil - | council | selenium_url | selenium_mode | - | ValeofGlamorganCouncil | None | None | + | council | + | ValeofGlamorganCouncil | @ValeofWhiteHorseCouncil Examples: ValeofWhiteHorseCouncil - | council | selenium_url | selenium_mode | - | ValeofWhiteHorseCouncil | None | None | + | council | + | ValeofWhiteHorseCouncil | @WakefieldCityCouncil Examples: WakefieldCityCouncil - | council | selenium_url | selenium_mode | - | WakefieldCityCouncil | http://selenium:4444 | local | + | council | + | WakefieldCityCouncil | @WarwickDistrictCouncil Examples: WarwickDistrictCouncil - | council | selenium_url | selenium_mode | - | WarwickDistrictCouncil | None | None | + | council | + | WarwickDistrictCouncil | @WaverleyBoroughCouncil Examples: WaverleyBoroughCouncil - | council | selenium_url | selenium_mode | - | WaverleyBoroughCouncil | None | None | + | council | + | WaverleyBoroughCouncil | @WealdenDistrictCouncil Examples: WealdenDistrictCouncil - | council | selenium_url | selenium_mode | - | WealdenDistrictCouncil | None | None | + | council | + | WealdenDistrictCouncil | @WelhatCouncil Examples: WelhatCouncil - | council | selenium_url | selenium_mode | - | WelhatCouncil | None | None | + | council | + | WelhatCouncil | @WestBerkshireCouncil Examples: WestBerkshireCouncil - | council | selenium_url | selenium_mode | - | WestBerkshireCouncil | http://selenium:4444 | local | + | council | + | WestBerkshireCouncil | @WestLindseyDistrictCouncil Examples: WestLindseyDistrictCouncil - | council | selenium_url | selenium_mode | - | WestLindseyDistrictCouncil | None | None | + | council | + | WestLindseyDistrictCouncil | @WestLothianCouncil Examples: WestLothianCouncil - | council | selenium_url | selenium_mode | - | WestLothianCouncil | http://selenium:4444 | local | + | council | + | WestLothianCouncil | @WestNorthamptonshireCouncil Examples: WestNorthamptonshireCouncil - | council | selenium_url | selenium_mode | - | WestNorthamptonshireCouncil | None | None | + | council | + | WestNorthamptonshireCouncil | @WestSuffolkCouncil Examples: WestSuffolkCouncil - | council | selenium_url | selenium_mode | - | WestSuffolkCouncil | http://selenium:4444 | local | + | council | + | WestSuffolkCouncil | @WiganBoroughCouncil Examples: WiganBoroughCouncil - | council | selenium_url | selenium_mode | - | WiganBoroughCouncil | None | None | + | council | + | WiganBoroughCouncil | @WiltshireCouncil Examples: WiltshireCouncil - | council | selenium_url | selenium_mode | - | WiltshireCouncil | None | None | + | council | + | WiltshireCouncil | @WindsorAndMaidenheadCouncil Examples: WindsorAndMaidenheadCouncil - | council | selenium_url | selenium_mode | - | WindsorAndMaidenheadCouncil | None | None | + | council | + | WindsorAndMaidenheadCouncil | @WokingBoroughCouncil Examples: WokingBoroughCouncil - | council | selenium_url | selenium_mode | - | WokingBoroughCouncil | None | None | + | council | + | WokingBoroughCouncil | @WyreCouncil Examples: WyreCouncil - | council | selenium_url | selenium_mode | - | WyreCouncil | None | None | + | council | + | WyreCouncil | @YorkCouncil Examples: YorkCouncil - | council | selenium_url | selenium_mode | - | YorkCouncil | None | None | + | council | + | YorkCouncil | diff --git a/uk_bin_collection/tests/step_defs/test_validate_council.py b/uk_bin_collection/tests/step_defs/test_validate_council.py index e954fd8eda..b444c708b4 100644 --- a/uk_bin_collection/tests/step_defs/test_validate_council.py +++ b/uk_bin_collection/tests/step_defs/test_validate_council.py @@ -2,9 +2,7 @@ import pytest import traceback from pytest_bdd import scenario, given, when, then, parsers -from hamcrest import assert_that, equal_to from functools import wraps -import os from step_helpers import file_handler from uk_bin_collection.uk_bin_collection import collect_data @@ -50,13 +48,11 @@ def get_council_step(context, council_name): @handle_test_errors @when( parsers.parse( - "we scrape the data from {council} using {selenium_mode} and the {selenium_url} is set" + "we scrape the data from {council}" ) ) -def scrape_step(context, council, selenium_mode, selenium_url, headless_mode): +def scrape_step(context, council, headless_mode, local_browser, selenium_url): context.council = council - context.selenium_mode = selenium_mode - context.selenium_url = selenium_url args = [council, context.metadata["url"]] @@ -81,9 +77,8 @@ def scrape_step(context, council, selenium_mode, selenium_url, headless_mode): # At the moment the feature file is set to local execution of the selenium so no url will be set # And it the behave test will execute locally - if selenium_mode != "None" and selenium_url != "None": - if selenium_mode != "local": - args.append(f"-w={selenium_url}") + if local_browser == "False": + args.append(f"-w={selenium_url}") if "skip_get_url" in context.metadata: args.append(f"-s") diff --git a/uk_bin_collection/uk_bin_collection/councils/ArunCouncil.py b/uk_bin_collection/uk_bin_collection/councils/ArunCouncil.py index 65c5e1b449..dcce127e62 100644 --- a/uk_bin_collection/uk_bin_collection/councils/ArunCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/ArunCouncil.py @@ -20,68 +20,78 @@ class CouncilClass(AbstractGetBinDataClass): """ def parse_data(self, page: str, **kwargs) -> dict: - # Make a BS4 object - data = {"bins": []} + try: + # Make a BS4 object + data = {"bins": []} - user_paon = kwargs.get("paon") - user_postcode = kwargs.get("postcode") - headless = kwargs.get("headless") - web_driver = kwargs.get("web_driver") - driver = create_webdriver(web_driver, headless) - page = "https://www1.arun.gov.uk/when-are-my-bins-collected/" - check_paon(user_paon) - check_postcode(user_postcode) - driver.get(page) + user_paon = kwargs.get("paon") + user_postcode = kwargs.get("postcode") + headless = kwargs.get("headless") + web_driver = kwargs.get("web_driver") + driver = create_webdriver(web_driver, headless) + page = "https://www1.arun.gov.uk/when-are-my-bins-collected/" + check_paon(user_paon) + check_postcode(user_postcode) + driver.get(page) - start_now_button = WebDriverWait(driver, timeout=15).until( - EC.presence_of_element_located((By.LINK_TEXT, "Start now")) - ) - start_now_button.click() + start_now_button = WebDriverWait(driver, timeout=15).until( + EC.presence_of_element_located((By.LINK_TEXT, "Start now")) + ) + start_now_button.click() - # Wait for the postcode field to appear then populate it - input_element_postcode = WebDriverWait(driver, 30).until( - EC.presence_of_element_located((By.ID, "postcode")) - ) - input_element_postcode.send_keys(user_postcode) + # Wait for the postcode field to appear then populate it + input_element_postcode = WebDriverWait(driver, 30).until( + EC.presence_of_element_located((By.ID, "postcode")) + ) + input_element_postcode.send_keys(user_postcode) - continue_button = WebDriverWait(driver, timeout=15).until( - EC.presence_of_element_located((By.CLASS_NAME, "govuk-button")) - ) - continue_button.click() + continue_button = WebDriverWait(driver, timeout=15).until( + EC.presence_of_element_located((By.CLASS_NAME, "govuk-button")) + ) + continue_button.click() - address_selection_menu = Select(driver.find_element(By.ID, "address")) - for idx, addr_option in enumerate(address_selection_menu.options): - option_name = addr_option.text[0 : len(user_paon)] - if option_name == user_paon: - selected_address = addr_option - break - address_selection_menu.select_by_visible_text(selected_address.text) + address_selection_menu = Select(driver.find_element(By.ID, "address")) + for idx, addr_option in enumerate(address_selection_menu.options): + option_name = addr_option.text[0 : len(user_paon)] + if option_name == user_paon: + selected_address = addr_option + break + address_selection_menu.select_by_visible_text(selected_address.text) - continue_button = WebDriverWait(driver, timeout=15).until( - EC.presence_of_element_located((By.CLASS_NAME, "govuk-button")) - ) - continue_button.click() - # Check for text saying "Next collection dates" - WebDriverWait(driver, 30).until( - EC.presence_of_element_located( - (By.XPATH, "//*[contains(text(), 'Next collection dates')]") + continue_button = WebDriverWait(driver, timeout=15).until( + EC.presence_of_element_located((By.CLASS_NAME, "govuk-button")) + ) + continue_button.click() + # Check for text saying "Next collection dates" + WebDriverWait(driver, 30).until( + EC.presence_of_element_located( + (By.XPATH, "//*[contains(text(), 'Next collection dates')]") + ) ) - ) - soup = BeautifulSoup(driver.page_source, "html.parser") - soup.prettify() - table = soup.find("table", class_="govuk-table") + soup = BeautifulSoup(driver.page_source, "html.parser") + soup.prettify() + table = soup.find("table", class_="govuk-table") - for row in table.find("tbody").find_all("tr"): - # Extract the type of collection and the date of next collection - collection_type = ( - row.find("th", class_="govuk-table__header").text.strip().split(" ") - )[0] - collection_date = row.find("td", class_="govuk-table__cell").text.strip() + for row in table.find("tbody").find_all("tr"): + # Extract the type of collection and the date of next collection + collection_type = ( + row.find("th", class_="govuk-table__header").text.strip().split(" ") + )[0] + collection_date = row.find("td", class_="govuk-table__cell").text.strip() - # Append the information to the data structure - data["bins"].append( - {"type": collection_type, "collectionDate": collection_date} - ) + # Append the information to the data structure + data["bins"].append( + {"type": collection_type, "collectionDate": collection_date} + ) + except Exception as e: + # Here you can log the exception if needed + print(f"An error occurred: {e}") + # Optionally, re-raise the exception if you want it to propagate + raise + finally: + # This block ensures that the driver is closed regardless of an exception. + if driver: + driver.quit() return data