From ddca79c28ff0013d9ff0b3ec527ba603090ce948 Mon Sep 17 00:00:00 2001 From: harshilgajera-crest <69803385+harshilgajera-crest@users.noreply.github.com> Date: Thu, 11 Apr 2024 16:41:22 +0530 Subject: [PATCH] refactor: adding lovely-pytest-docker code (#816) This PR contains following changes - Added code for lovely-pytest-docker in PSA and migrated to v2 version of docker-compose as GitHub runners have stopped supporting docker compose v1. - Also fixed the CI runs for e2e tests as it always running on latest splunk instead of version provided by addonfactory-splunk-matrix. --- .github/workflows/build-test-release.yml | 10 +- .licenserc.yaml | 1 + Dockerfile.splunk | 2 - entrypoint.sh | 4 +- poetry.lock | 19 +-- pyproject.toml | 4 - pytest_splunk_addon/docker_class.py | 162 +++++++++++++++++++++++ pytest_splunk_addon/splunk.py | 59 +++++++++ tests/e2e/test_splunk_addon.py | 39 ++++-- 9 files changed, 256 insertions(+), 44 deletions(-) create mode 100644 pytest_splunk_addon/docker_class.py diff --git a/.github/workflows/build-test-release.yml b/.github/workflows/build-test-release.yml index 00a852bb0..ffe1ba370 100644 --- a/.github/workflows/build-test-release.yml +++ b/.github/workflows/build-test-release.yml @@ -111,7 +111,7 @@ jobs: - name: Install and run tests run: | curl -sSL https://install.python-poetry.org | python3 - --version 1.5.1 - poetry install --with docs -E docker + poetry install --with docs poetry run pytest -v -m doc tests/e2e test-splunk-external: @@ -143,12 +143,12 @@ jobs: export SPLUNK_APP_ID=TA_fiction export SPLUNK_VERSION=${{ matrix.splunk.version }} echo $SPLUNK_VERSION - docker-compose -f "docker-compose-ci.yml" build - SPLUNK_PASSWORD=Chang3d! docker-compose -f docker-compose-ci.yml up -d splunk + docker compose -f "docker-compose-ci.yml" build + SPLUNK_PASSWORD=Chang3d! docker compose -f docker-compose-ci.yml up -d splunk sleep 90 - name: Test run: | - SPLUNK_PASSWORD=Chang3d! docker-compose -f docker-compose-ci.yml up --abort-on-container-exit + SPLUNK_PASSWORD=Chang3d! docker compose -f docker-compose-ci.yml up --abort-on-container-exit docker volume ls - name: Collect Results run: | @@ -200,7 +200,7 @@ jobs: python-version: 3.7 - run: | curl -sSL https://install.python-poetry.org | python3 - --version 1.5.1 - poetry install -E docker + poetry install poetry run pytest -v --splunk-version=${{ matrix.splunk.version }} -m docker -m ${{ matrix.test-marker }} tests/e2e publish: diff --git a/.licenserc.yaml b/.licenserc.yaml index 57ef85616..9cf6ee243 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -36,5 +36,6 @@ header: - "entrypoint.sh" - "renovate.json" - "pytest_splunk_addon/.ignore_splunk_internal_errors" + - "pytest_splunk_addon/docker_class.py" comment: on-failure \ No newline at end of file diff --git a/Dockerfile.splunk b/Dockerfile.splunk index 78c13afcb..b9da72e98 100644 --- a/Dockerfile.splunk +++ b/Dockerfile.splunk @@ -15,10 +15,8 @@ # ARG SPLUNK_VERSION=latest FROM splunk/splunk:$SPLUNK_VERSION -ARG SPLUNK_VERSION=latest ARG SPLUNK_APP_ID=TA_UNKNOWN ARG SPLUNK_APP_PACKAGE=package -RUN echo ${SPLUNK_VERSION} $SPLUNK_APP_PACKAGE COPY ${SPLUNK_APP_PACKAGE} /opt/splunk/etc/apps/${SPLUNK_APP_ID} COPY deps/apps /opt/splunk/etc/apps/ COPY deps/build/addonfactory_test_matrix_splunk/packages/all/common /opt/splunk/etc/apps/ diff --git a/entrypoint.sh b/entrypoint.sh index 9834fa251..3664f2a46 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -6,9 +6,9 @@ export PATH="~/.pyenv/bin:$PATH" eval "$(pyenv init -)" pyenv install 3.7.8 pyenv local 3.7.8 -curl -sSL https://install.python-poetry.org | python +curl -sSL https://install.python-poetry.org | python - --version 1.5.1 export PATH="/root/.local/bin:$PATH" source ~/.poetry/env sleep 15 -poetry install -E docker +poetry install exec poetry run pytest -vv $@ diff --git a/poetry.lock b/poetry.lock index 8c6d9dc3b..0f69155a2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -495,20 +495,6 @@ files = [ [package.dependencies] future = "*" -[[package]] -name = "lovely-pytest-docker" -version = "0.3.1" -description = "Pytest testing utilities with docker containers." -optional = false -python-versions = "*" -files = [ - {file = "lovely-pytest-docker-0.3.1.tar.gz", hash = "sha256:4326a180bfd4dd4ad69c2ef3e3643c41075d965f40068488b40204602e6df85e"}, -] - -[package.dependencies] -pytest = "*" -six = "*" - [[package]] name = "markupsafe" version = "2.1.3" @@ -1144,10 +1130,7 @@ files = [ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] -[extras] -docker = ["lovely-pytest-docker"] - [metadata] lock-version = "2.0" python-versions = "^3.7" -content-hash = "384db44b6bda065e8afbb4574d233e623f54d86f7a12adba698830a655d6adf3" +content-hash = "02ea5ca0c0a2e37f94c6c618bfff794fb541a8acd568f77c1dafe89ea305498c" diff --git a/pyproject.toml b/pyproject.toml index 731e17664..58ef2b35f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,6 @@ jsonschema = ">=4,<5" pytest-xdist = ">=2.3.0" filelock = "^3.0" pytest-ordering = "~0.6" -lovely-pytest-docker = { version="^0", optional = true } junitparser = "^2.2.0" addonfactory-splunk-conf-parser-lib = "*" defusedxml = "^0.7.1" @@ -50,11 +49,8 @@ xmlschema = "^1.11.3" splunksplwrapper = "^1.1.1" urllib3 = "<2" -[tool.poetry.extras] -docker = ['lovely-pytest-docker'] [tool.poetry.group.dev.dependencies] -lovely-pytest-docker = "~0.3.0" pytest-cov = "^3.0.0" requests-mock = "^1.8.0" freezegun = "^1.2.1" diff --git a/pytest_splunk_addon/docker_class.py b/pytest_splunk_addon/docker_class.py new file mode 100644 index 000000000..7d13e9ef3 --- /dev/null +++ b/pytest_splunk_addon/docker_class.py @@ -0,0 +1,162 @@ +import functools +import os +from urllib.request import urlopen + +import pytest +import re +import subprocess +import time +import timeit + +from requests import HTTPError + + +def check_url(docker_ip, public_port, path="/"): + """Check if a service is reachable. + + Makes a simple GET request to path of the HTTP endpoint. Service is + available if returned status code is < 500. + """ + url = "http://{}:{}{}".format(docker_ip, public_port, path) + try: + r = urlopen(url) + return r.code < 500 + except HTTPError as e: + # If service returns e.g. a 404 it's ok + return e.code < 500 + except Exception: + # Possible service not yet started + return False + + +def execute(command, success_codes=(0,)): + """Run a shell command.""" + try: + output = subprocess.check_output( + command, + stderr=subprocess.STDOUT, + shell=False, + ) + status = 0 + except subprocess.CalledProcessError as error: + output = error.output or b"" + status = error.returncode + command = error.cmd + output = output.decode("utf-8") + if status not in success_codes: + raise Exception('Command %r returned %d: """%s""".' % (command, status, output)) + return output + + +class Services(object): + """A class which encapsulates services from docker compose definition. + + This code is partly taken from + https://github.com/AndreLouisCaron/pytest-docker + """ + + def __init__(self, compose_files, docker_ip, project_name="pytest"): + self._docker_compose = DockerComposeExecutor(compose_files, project_name) + self._services = {} + self.docker_ip = docker_ip + + def start(self, *services): + """Ensures that the given services are started via docker compose. + + :param services: the names of the services as defined in compose file + """ + self._docker_compose.execute("up", "--build", "-d", *services) + + def stop(self, *services): + """Ensures that the given services are stopped via docker compose. + + :param services: the names of the services as defined in compose file + """ + self._docker_compose.execute("stop", *services) + + def execute(self, service, *cmd): + """Execute a command inside a docker container. + + :param service: the name of the service as defined in compose file + :param cmd: list of command parts to execute + """ + return self._docker_compose.execute("exec", "-T", service, *cmd) + + def wait_for_service( + self, service, private_port, check_server=check_url, timeout=30.0, pause=0.1 + ): + """ + Waits for the given service to response to a http GET. + + :param service: the service name as defined in the docker compose file + :param private_port: the private port as defined in docker compose file + :param check_server: optional function to check if the server is ready + (default check method makes GET request to '/' + of HTTP endpoint) + :param timeout: maximum time to wait for the service in seconds + :param pause: time in seconds to wait between retries + + :return: the public port of the service exposed to host system if any + """ + public_port = self.port_for(service, private_port) + self.wait_until_responsive( + timeout=timeout, + pause=pause, + check=lambda: check_server(self.docker_ip, public_port), + ) + return public_port + + def shutdown(self): + self._docker_compose.execute("down", "-v") + + def port_for(self, service, port): + """Get the effective bind port for a service.""" + + # Lookup in the cache. + cache = self._services.get(service, {}).get(port, None) + if cache is not None: + return cache + + output = self._docker_compose.execute("port", service, str(port)) + endpoint = output.strip() + if not endpoint: + raise ValueError('Could not detect port for "%s:%d".' % (service, port)) + + # Usually, the IP address here is 0.0.0.0, so we don't use it. + match = int(endpoint.split(":", 1)[1]) + + # Store it in cache in case we request it multiple times. + self._services.setdefault(service, {})[port] = match + + return match + + @staticmethod + def wait_until_responsive(check, timeout, pause, clock=timeit.default_timer): + """Wait until a service is responsive.""" + + ref = clock() + now = ref + while (now - ref) < timeout: + if check(): + return + time.sleep(pause) + now = clock() + + raise Exception("Timeout reached while waiting on service!") + + +class DockerComposeExecutor(object): + def __init__(self, compose_files, project_name): + self._compose_files = compose_files + self._project_name = project_name + self.project_directory = os.path.dirname(os.path.realpath(compose_files[0])) + + def execute(self, *subcommand): + command = ["docker", "compose"] + for compose_file in self._compose_files: + command.append("-f") + command.append(compose_file) + command.append("-p") + command.append(self._project_name) + command += subcommand + return execute(command) diff --git a/pytest_splunk_addon/splunk.py b/pytest_splunk_addon/splunk.py index 3778f8dba..bfe764787 100644 --- a/pytest_splunk_addon/splunk.py +++ b/pytest_splunk_addon/splunk.py @@ -21,11 +21,13 @@ import json import pytest import requests +import re import splunklib.client as client from splunksplwrapper.manager.jobs import Jobs from splunksplwrapper.splunk.cloud import CloudSplunk from splunksplwrapper.SearchUtil import SearchUtil from .standard_lib.event_ingestors import IngestorHelper +from .docker_class import Services from .standard_lib.CIM_Models.datamodel_definition import datamodels import configparser from filelock import FileLock @@ -323,6 +325,13 @@ def pytest_addoption(parser): help="Should execute test or not (True|False)", default="True", ) + group.addoption( + "--keepalive", + "-K", + action="store_true", + default=False, + help="Keep docker containers alive", + ) @pytest.fixture(scope="session") @@ -830,6 +839,56 @@ def update_recommended_fields(model, datasets, cim_version): return update_recommended_fields +@pytest.fixture(scope="session") +def docker_ip(): + """Determine IP address for TCP connections to Docker containers.""" + + # When talking to the Docker daemon via a UNIX socket, route all TCP + # traffic to docker containers via the TCP loopback interface. + docker_host = os.environ.get("DOCKER_HOST", "").strip() + if not docker_host: + return "127.0.0.1" + + match = re.match("^tcp://(.+?):\d+$", docker_host) + if not match: + raise ValueError('Invalid value for DOCKER_HOST: "%s".' % (docker_host,)) + return match.group(1) + + +@pytest.fixture(scope="session") +def docker_compose_files(pytestconfig): + """Get the docker-compose.yml absolute path. + Override this fixture in your tests if you need a custom location. + """ + return [os.path.join(str(pytestconfig.rootdir), "tests", "docker-compose.yml")] + + +@pytest.fixture(scope="session") +def docker_services_project_name(pytestconfig): + """ + Create unique project name for docker compose based on the pytestconfig root directory. + Characters prohibited by Docker compose project names are replaced with hyphens. + """ + slug = re.sub(r"[^a-z0-9]+", "-", str(pytestconfig.rootdir).lower()) + project_name = "pytest{}".format(slug) + return project_name + + +@pytest.fixture(scope="session") +def docker_services( + request, docker_compose_files, docker_ip, docker_services_project_name +): + """Provide the docker services as a pytest fixture. + + The services will be stopped after all tests are run. + """ + keep_alive = request.config.getoption("--keepalive", False) + services = Services(docker_compose_files, docker_ip, docker_services_project_name) + yield services + if not keep_alive: + services.shutdown() + + def is_responsive_uf(uf): """ Verify if the management port of Universal Forwarder is responsive or not diff --git a/tests/e2e/test_splunk_addon.py b/tests/e2e/test_splunk_addon.py index 25dc68a42..4a98cb610 100644 --- a/tests/e2e/test_splunk_addon.py +++ b/tests/e2e/test_splunk_addon.py @@ -56,7 +56,7 @@ def setup_test_dir(testdir): @pytest.mark.external -def test_splunk_connection_external(testdir): +def test_splunk_connection_external(testdir, request): """Make sure that pytest accepts our fixture.""" # create a temporary pytest test module @@ -72,6 +72,7 @@ def test_splunk_connection_external(testdir): # run pytest with the following cmd args result = testdir.runpytest( + f"--splunk-version={request.config.getoption('splunk_version')}", "--splunk-app=addons/TA_fiction", "--splunk-type=external", "--splunk-host=splunk", @@ -89,7 +90,7 @@ def test_splunk_connection_external(testdir): @pytest.mark.docker @pytest.mark.splunk_connection_docker -def test_splunk_connection_docker(testdir): +def test_splunk_connection_docker(testdir, request): """Make sure that pytest accepts our fixture.""" # create a temporary pytest test module @@ -108,6 +109,7 @@ def test_splunk_connection_docker(testdir): # run pytest with the following cmd args result = testdir.runpytest( + f"--splunk-version={request.config.getoption('splunk_version')}", "--splunk-type=docker", "-v", ) @@ -121,7 +123,7 @@ def test_splunk_connection_docker(testdir): @pytest.mark.docker @pytest.mark.splunk_app_fiction -def test_splunk_app_fiction(testdir): +def test_splunk_app_fiction(testdir, request): """Make sure that pytest accepts our fixture.""" testdir.makepyfile( @@ -145,6 +147,7 @@ def empty_method(): # run pytest with the following cmd args result = testdir.runpytest( + f"--splunk-version={request.config.getoption('splunk_version')}", "--splunk-type=docker", "-v", "-m splunk_searchtime_fields", @@ -168,7 +171,7 @@ def empty_method(): @pytest.mark.docker @pytest.mark.splunk_fiction_indextime_wrong_hec_token -def test_splunk_fiction_indextime_wrong_hec_token(testdir): +def test_splunk_fiction_indextime_wrong_hec_token(testdir, request): """Make sure that pytest accepts our fixture.""" testdir.makepyfile( @@ -201,6 +204,7 @@ def empty_method(): # run pytest with the following cmd args result = testdir.runpytest( + f"--splunk-version={request.config.getoption('splunk_version')}", "--splunk-type=docker", "-v", "--search-interval=0", @@ -219,7 +223,7 @@ def empty_method(): @pytest.mark.docker @pytest.mark.splunk_app_broken -def test_splunk_app_broken(testdir): +def test_splunk_app_broken(testdir, request): """Make sure that pytest accepts our fixture.""" testdir.makepyfile( @@ -248,6 +252,7 @@ def empty_method(): # run pytest with the following cmd args result = testdir.runpytest( + f"--splunk-version={request.config.getoption('splunk_version')}", "--splunk-type=docker", "-v", "-m splunk_searchtime_fields", @@ -275,7 +280,7 @@ def empty_method(): @pytest.mark.docker @pytest.mark.splunk_app_cim_fiction -def test_splunk_app_cim_fiction(testdir): +def test_splunk_app_cim_fiction(testdir, request): """Make sure that pytest accepts our fixture.""" testdir.makepyfile( @@ -304,6 +309,7 @@ def empty_method(): # run pytest with the following cmd args result = testdir.runpytest( + f"--splunk-version={request.config.getoption('splunk_version')}", "--splunk-type=docker", "--splunk-dm-path=tests/data_models", "-v", @@ -328,7 +334,7 @@ def empty_method(): @pytest.mark.docker @pytest.mark.splunk_app_cim_broken -def test_splunk_app_cim_broken(testdir): +def test_splunk_app_cim_broken(testdir, request): """Make sure that pytest accepts our fixture.""" testdir.makepyfile( @@ -357,6 +363,7 @@ def empty_method(): # run pytest with the following cmd args result = testdir.runpytest( + f"--splunk-version={request.config.getoption('splunk_version')}", "--splunk-type=docker", "--splunk-dm-path=tests/data_models", "-v", @@ -384,7 +391,7 @@ def empty_method(): @pytest.mark.docker @pytest.mark.splunk_fiction_indextime -def test_splunk_fiction_indextime(testdir): +def test_splunk_fiction_indextime(testdir, request): """Make sure that pytest accepts our fixture.""" testdir.makepyfile( @@ -413,6 +420,7 @@ def empty_method(): # run pytest with the following cmd args result = testdir.runpytest( + f"--splunk-version={request.config.getoption('splunk_version')}", "--splunk-type=docker", "-v", "--search-interval=0", @@ -437,7 +445,7 @@ def empty_method(): @pytest.mark.docker @pytest.mark.splunk_fiction_indextime_broken -def test_splunk_fiction_indextime_broken(testdir): +def test_splunk_fiction_indextime_broken(testdir, request): """Make sure that pytest accepts our fixture.""" testdir.makepyfile( @@ -468,6 +476,7 @@ def empty_method(): # run pytest with the following cmd args result = testdir.runpytest( + f"--splunk-version={request.config.getoption('splunk_version')}", "--splunk-type=docker", "-v", "--search-interval=0", @@ -494,7 +503,7 @@ def empty_method(): @pytest.mark.docker @pytest.mark.splunk_setup_fixture -def test_splunk_setup_fixture(testdir): +def test_splunk_setup_fixture(testdir, request): testdir.makepyfile( """ from pytest_splunk_addon.standard_lib.addon_basic import Basic @@ -518,6 +527,7 @@ def empty_method(): ) result = testdir.runpytest( + f"--splunk-version={request.config.getoption('splunk_version')}", "--splunk-type=docker", "-v", "-k saved_search_lookup", @@ -568,7 +578,7 @@ def test_docstrings(testdir): @pytest.mark.docker @pytest.mark.splunk_app_req -def test_splunk_app_req(testdir): +def test_splunk_app_req(testdir, request): """Make sure that pytest accepts our fixture.""" testdir.makepyfile( @@ -596,6 +606,7 @@ def empty_method(): # run pytest with the following cmd args result = testdir.runpytest( + f"--splunk-version={request.config.getoption('splunk_version')}", "--splunk-type=docker", "-v", "--search-interval=4", @@ -622,7 +633,7 @@ def empty_method(): @pytest.mark.docker @pytest.mark.splunk_app_req_broken -def test_splunk_app_req_broken(testdir): +def test_splunk_app_req_broken(testdir, request): """Make sure that pytest accepts our fixture.""" testdir.makepyfile( @@ -650,6 +661,7 @@ def empty_method(): # run pytest with the following cmd args result = testdir.runpytest( + f"--splunk-version={request.config.getoption('splunk_version')}", "--splunk-type=docker", "-v", "--search-interval=4", @@ -676,7 +688,7 @@ def empty_method(): @pytest.mark.docker @pytest.mark.splunk_app_req -def test_splunk_app_req(testdir): +def test_splunk_app_req(testdir, request): """Make sure that pytest accepts our fixture.""" testdir.makepyfile( @@ -704,6 +716,7 @@ def empty_method(): # run pytest with the following cmd args result = testdir.runpytest( + f"--splunk-version={request.config.getoption('splunk_version')}", "--splunk-type=docker", "-v", "--search-interval=2",