From 2ded6f4b6bdf6f0b9c8ee005250ba7d4798f342d Mon Sep 17 00:00:00 2001 From: Marcelo Henrique Neppel Date: Thu, 8 Sep 2022 10:18:20 -0300 Subject: [PATCH] Parallelize tests (#26) * Change charm database user * Fix unit tests * Fix integration test call * Fix user name in library * Fix user * Add default postgres user creation * Change action name * Rework secrets management * Add password rotation logic * Add user to the action parameters * Add separate environments for different tests * Add all dependencies * Add pytest marks * Fix action description * Fix method docstring * Fix pytest mark * Fix pytest markers * Register pytest markers --- .github/workflows/ci.yaml | 60 +++++++++++- .github/workflows/release.yaml | 64 +++++++++++- pyproject.toml | 7 ++ .../new_relations/test_new_relations.py | 11 ++- tests/integration/test_charm.py | 6 ++ tests/integration/test_db.py | 2 + tests/integration/test_db_admin.py | 2 + tox.ini | 98 ++++++++++++++++++- 8 files changed, 240 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a759f2c515..ab8c30e558 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,6 +13,7 @@ jobs: run: python3 -m pip install tox - name: Run linters run: tox -e lint + unit-test: name: Unit tests runs-on: ubuntu-latest @@ -25,8 +26,61 @@ jobs: run: tox -e unit - name: Upload Coverage to Codecov uses: codecov/codecov-action@v2 - integration-test-lxd: - name: Integration tests (lxd) + + integration-test-lxd-charm: + name: Integration tests for charm deployment (lxd) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: lxd + - name: Run integration tests + run: tox -e charm-integration + + integration-test-lxd-database-relation: + name: Integration tests for database relation (lxd) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: lxd + - name: Run integration tests + run: tox -e database-relation-integration + + integration-test-lxd-db-relation: + name: Integration tests for db relation (lxd) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: lxd + - name: Run integration tests + run: tox -e db-relation-integration + + integration-test-lxd-db-admin-relation: + name: Integration tests for db-admin relation (lxd) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: lxd + - name: Run integration tests + run: tox -e db-admin-relation-integration + + integration-test-lxd-password-rotation: + name: Integration tests for password rotation (lxd) runs-on: ubuntu-latest steps: - name: Checkout @@ -36,4 +90,4 @@ jobs: with: provider: lxd - name: Run integration tests - run: tox -e integration + run: tox -e password-rotation-integration diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 80f700bba0..9c00b80780 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -42,8 +42,8 @@ jobs: - name: Run tests run: tox -e unit - integration-test-lxd: - name: Integration tests for (lxd) + integration-test-charm: + name: Integration tests for charm deployment runs-on: ubuntu-latest steps: - name: Checkout @@ -53,7 +53,59 @@ jobs: with: provider: lxd - name: Run integration tests - run: tox -e integration + run: tox -e charm-integration + + integration-test-database-relation: + name: Integration tests for database relation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: lxd + - name: Run integration tests + run: tox -e database-relation-integration + + integration-test-db-relation: + name: Integration tests for db relation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: lxd + - name: Run integration tests + run: tox -e db-relation-integration + + integration-test-db-admin-relation: + name: Integration tests for db-admin relation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: lxd + - name: Run integration tests + run: tox -e db-admin-relation-integration + + integration-test-password-rotation: + name: Integration tests for password rotation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: lxd + - name: Run integration tests + run: tox -e password-rotation-integration release-to-charmhub: name: Release to CharmHub @@ -61,7 +113,11 @@ jobs: - lib-check - lint - unit-test - - integration-test-lxd + - integration-test-charm + - integration-test-database-relation + - integration-test-db-relation + - integration-test-db-admin-relation + - integration-test-password-rotation runs-on: ubuntu-latest steps: - name: Checkout diff --git a/pyproject.toml b/pyproject.toml index f974d2ae79..d873e96cb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,13 @@ show_missing = true minversion = "6.0" log_cli_level = "INFO" asyncio_mode = "auto" +markers = [ + "charm_tests", + "database_relation_tests", + "db_relation_tests", + "db_admin_relation_tests", + "password_rotation_tests", +] # Formatting tools configuration [tool.black] diff --git a/tests/integration/new_relations/test_new_relations.py b/tests/integration/new_relations/test_new_relations.py index dc473f90b4..93a9b090ff 100644 --- a/tests/integration/new_relations/test_new_relations.py +++ b/tests/integration/new_relations/test_new_relations.py @@ -31,6 +31,7 @@ @pytest.mark.abort_on_fail +@pytest.mark.database_relation_tests async def test_deploy_charms(ops_test: OpsTest, application_charm, database_charm): """Deploy both charms (application and database) to use in the tests.""" # Deploy both charms (multiple units for each application to test that later they correctly @@ -65,6 +66,7 @@ async def test_deploy_charms(ops_test: OpsTest, application_charm, database_char await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active", timeout=3000) +@pytest.mark.database_relation_tests async def test_no_read_only_endpoint_in_standalone_cluster(ops_test: OpsTest): """Test that there is no read-only endpoint in a standalone cluster.""" async with ops_test.fast_forward(): @@ -91,6 +93,7 @@ async def test_no_read_only_endpoint_in_standalone_cluster(ops_test: OpsTest): ) +@pytest.mark.database_relation_tests async def test_read_only_endpoint_in_scaled_up_cluster(ops_test: OpsTest): """Test that there is read-only endpoint in a scaled up cluster.""" async with ops_test.fast_forward(): @@ -108,7 +111,7 @@ async def test_read_only_endpoint_in_scaled_up_cluster(ops_test: OpsTest): ) -@pytest.mark.abort_on_fail +@pytest.mark.database_relation_tests async def test_database_relation_with_charm_libraries(ops_test: OpsTest): """Test basic functionality of database relation interface.""" # Get the connection string to connect to the database using the read/write endpoint. @@ -156,6 +159,7 @@ async def test_database_relation_with_charm_libraries(ops_test: OpsTest): cursor.execute("DROP TABLE test;") +@pytest.mark.database_relation_tests async def test_user_with_extra_roles(ops_test: OpsTest): """Test superuser actions and the request for more permissions.""" # Get the connection string to connect to the database. @@ -176,6 +180,7 @@ async def test_user_with_extra_roles(ops_test: OpsTest): connection.close() +@pytest.mark.database_relation_tests async def test_two_applications_doesnt_share_the_same_relation_data( ops_test: OpsTest, application_charm ): @@ -210,6 +215,7 @@ async def test_two_applications_doesnt_share_the_same_relation_data( assert application_connection_string != another_application_connection_string +@pytest.mark.database_relation_tests async def test_an_application_can_connect_to_multiple_database_clusters( ops_test: OpsTest, database_charm ): @@ -242,6 +248,7 @@ async def test_an_application_can_connect_to_multiple_database_clusters( assert application_connection_string != another_application_connection_string +@pytest.mark.database_relation_tests async def test_an_application_can_connect_to_multiple_aliased_database_clusters( ops_test: OpsTest, database_charm ): @@ -277,6 +284,7 @@ async def test_an_application_can_connect_to_multiple_aliased_database_clusters( assert application_connection_string != another_application_connection_string +@pytest.mark.database_relation_tests async def test_an_application_can_request_multiple_databases(ops_test: OpsTest, application_charm): """Test that an application can request additional databases using the same interface.""" # Relate the charms using another relation and wait for them exchanging some connection data. @@ -297,6 +305,7 @@ async def test_an_application_can_request_multiple_databases(ops_test: OpsTest, assert first_database_connection_string != second_database_connection_string +@pytest.mark.database_relation_tests async def test_relation_data_is_updated_correctly_when_scaling(ops_test: OpsTest): """Test that relation data, like connection data, is updated correctly when scaling.""" # Retrieve the list of current database unit names. diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 09e0a16e3b..d8c6531bdc 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -33,7 +33,9 @@ @pytest.mark.abort_on_fail +@pytest.mark.charm_tests @pytest.mark.parametrize("series", SERIES) +@pytest.mark.skip_if_deployed async def test_deploy(ops_test: OpsTest, charm: str, series: str): """Deploy the charm-under-test. @@ -58,6 +60,7 @@ async def test_deploy(ops_test: OpsTest, charm: str, series: str): @pytest.mark.abort_on_fail +@pytest.mark.charm_tests @pytest.mark.parametrize("series", SERIES) @pytest.mark.parametrize("unit_id", UNIT_IDS) async def test_database_is_up(ops_test: OpsTest, series: str, unit_id: int): @@ -71,6 +74,7 @@ async def test_database_is_up(ops_test: OpsTest, series: str, unit_id: int): assert result.status_code == 200 +@pytest.mark.charm_tests @pytest.mark.parametrize("series", SERIES) @pytest.mark.parametrize("unit_id", UNIT_IDS) async def test_settings_are_correct(ops_test: OpsTest, series: str, unit_id: int): @@ -120,6 +124,7 @@ async def test_settings_are_correct(ops_test: OpsTest, series: str, unit_id: int assert settings["maximum_lag_on_failover"] == 1048576 +@pytest.mark.charm_tests @pytest.mark.parametrize("series", SERIES) async def test_scale_down_and_up(ops_test: OpsTest, series: str): """Test data is replicated to new units after a scale up.""" @@ -205,6 +210,7 @@ async def test_scale_down_and_up(ops_test: OpsTest, series: str): await scale_application(ops_test, application_name, initial_scale) +@pytest.mark.charm_tests @pytest.mark.parametrize("series", SERIES) async def test_persist_data_through_primary_deletion(ops_test: OpsTest, series: str): """Test data persists through a primary deletion.""" diff --git a/tests/integration/test_db.py b/tests/integration/test_db.py index 456f33b35e..7212d53086 100644 --- a/tests/integration/test_db.py +++ b/tests/integration/test_db.py @@ -24,6 +24,7 @@ RELATION_NAME = "db" +@pytest.mark.db_relation_tests async def test_mailman3_core_db(ops_test: OpsTest, charm: str) -> None: """Deploy Mailman3 Core to test the 'db' relation.""" async with ops_test.fast_forward(): @@ -92,6 +93,7 @@ async def test_mailman3_core_db(ops_test: OpsTest, charm: str) -> None: assert domain_name not in [domain.mail_host for domain in client.domains] +@pytest.mark.db_relation_tests async def test_relation_data_is_updated_correctly_when_scaling(ops_test: OpsTest): """Test that relation data, like connection data, is updated correctly when scaling.""" # Retrieve the list of current database unit names. diff --git a/tests/integration/test_db_admin.py b/tests/integration/test_db_admin.py index 51c5e8ac50..d8b29fb2ca 100644 --- a/tests/integration/test_db_admin.py +++ b/tests/integration/test_db_admin.py @@ -5,6 +5,7 @@ import json import logging +import pytest as pytest from landscape_api.base import run_query from pytest_operator.plugin import OpsTest @@ -24,6 +25,7 @@ DATABASE_UNITS = 3 +@pytest.mark.db_admin_relation_tests async def test_landscape_scalable_bundle_db(ops_test: OpsTest, charm: str) -> None: """Deploy Landscape Scalable Bundle to test the 'db-admin' relation.""" config = { diff --git a/tox.ini b/tox.ini index 5d07a7aa2e..c1d1bd1f97 100644 --- a/tox.ini +++ b/tox.ini @@ -65,8 +65,103 @@ commands = coverage report coverage xml +[testenv:charm-integration] +description = Run charm integration tests +deps = + pytest + juju==2.9.11 # juju 3.0.0 has issues with retrieving action results + landscape-api-py3 + mailmanclient + pytest-operator + psycopg2-binary + -r{toxinidir}/requirements.txt +commands = + # Download patroni resource to use in the charm deployment. + sh -c 'stat patroni.tar.gz > /dev/null 2>&1 || curl "https://github.com/zalando/patroni/archive/refs/tags/v2.1.3.tar.gz" -L -s > patroni.tar.gz' + pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} -m charm_tests + # Remove the downloaded resource. + sh -c 'rm -f patroni.tar.gz' +whitelist_externals = + sh + +[testenv:database-relation-integration] +description = Run database relation integration tests +deps = + pytest + juju==2.9.11 # juju 3.0.0 has issues with retrieving action results + landscape-api-py3 + mailmanclient + pytest-operator + psycopg2-binary + -r{toxinidir}/requirements.txt +commands = + # Download patroni resource to use in the charm deployment. + sh -c 'stat patroni.tar.gz > /dev/null 2>&1 || curl "https://github.com/zalando/patroni/archive/refs/tags/v2.1.3.tar.gz" -L -s > patroni.tar.gz' + pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} -m database_relation_tests + # Remove the downloaded resource. + sh -c 'rm -f patroni.tar.gz' +whitelist_externals = + sh + +[testenv:db-relation-integration] +description = Run db relation integration tests +deps = + pytest + juju==2.9.11 # juju 3.0.0 has issues with retrieving action results + landscape-api-py3 + mailmanclient + pytest-operator + psycopg2-binary + -r{toxinidir}/requirements.txt +commands = + # Download patroni resource to use in the charm deployment. + sh -c 'stat patroni.tar.gz > /dev/null 2>&1 || curl "https://github.com/zalando/patroni/archive/refs/tags/v2.1.3.tar.gz" -L -s > patroni.tar.gz' + pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} -m db_relation_tests + # Remove the downloaded resource. + sh -c 'rm -f patroni.tar.gz' +whitelist_externals = + sh + +[testenv:db-admin-relation-integration] +description = Run db-admin relation integration tests +deps = + pytest + juju==2.9.11 # juju 3.0.0 has issues with retrieving action results + landscape-api-py3 + mailmanclient + pytest-operator + psycopg2-binary + -r{toxinidir}/requirements.txt +commands = + # Download patroni resource to use in the charm deployment. + sh -c 'stat patroni.tar.gz > /dev/null 2>&1 || curl "https://github.com/zalando/patroni/archive/refs/tags/v2.1.3.tar.gz" -L -s > patroni.tar.gz' + pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} -m db_admin_relation_tests + # Remove the downloaded resource. + sh -c 'rm -f patroni.tar.gz' +whitelist_externals = + sh + +[testenv:password-rotation-integration] +description = Run password rotation integration tests +deps = + pytest + juju==2.9.11 # juju 3.0.0 has issues with retrieving action results + landscape-api-py3 + mailmanclient + pytest-operator + psycopg2-binary + -r{toxinidir}/requirements.txt +commands = + # Download patroni resource to use in the charm deployment. + sh -c 'stat patroni.tar.gz > /dev/null 2>&1 || curl "https://github.com/zalando/patroni/archive/refs/tags/v2.1.3.tar.gz" -L -s > patroni.tar.gz' + pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} -m password_rotation_tests + # Remove the downloaded resource. + sh -c 'rm -f patroni.tar.gz' +whitelist_externals = + sh + [testenv:integration] -description = Run integration tests +description = Run all integration tests deps = pytest juju==2.9.11 # juju 3.0.0 has issues with retrieving action results @@ -74,7 +169,6 @@ deps = mailmanclient pytest-operator psycopg2-binary - requests -r{toxinidir}/requirements.txt commands = # Download patroni resource to use in the charm deployment.