From dffa15ec38665a1e4d0bb5f43a675fc2dc76ff4e Mon Sep 17 00:00:00 2001 From: Daniil Anfimov Date: Wed, 22 Nov 2023 14:43:39 +0100 Subject: [PATCH 1/4] Index git repositories instead of HTML pages --- .gitignore | 1 + Dockerfile | 1 + README.md | 24 +++--- alma_tests_cacher.py | 6 +- alma_tests_cacher/cacher.py | 147 ++++++++++++++++++++------------- alma_tests_cacher/constants.py | 3 + alma_tests_cacher/models.py | 34 ++++---- alma_tests_cacher/utils.py | 56 +++++++++++++ docker-compose.yml | 6 +- requirements.txt | 4 + 10 files changed, 192 insertions(+), 90 deletions(-) create mode 100644 alma_tests_cacher/utils.py diff --git a/.gitignore b/.gitignore index c8450df..b262089 100644 --- a/.gitignore +++ b/.gitignore @@ -122,6 +122,7 @@ celerybeat.pid # Environments .env .venv +vars.yaml *.env env/ venv/ diff --git a/Dockerfile b/Dockerfile index a85a5c6..6eaf7fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM almalinux:9 RUN mkdir -p /code && \ dnf update -y && \ + dnf install git -y && \ dnf clean all RUN curl https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh -o wait_for_it.sh && chmod +x wait_for_it.sh COPY ./requirements.txt /code/requirements.txt diff --git a/README.md b/README.md index a4f04dc..ad91bc6 100644 --- a/README.md +++ b/README.md @@ -4,28 +4,32 @@ Tool for caching third-party tests for ALTS from git repositories. ## Requirements +* git >= 2.41.0 * python >= 3.7 * aiohttp >= 3.8.6 * pydantic >= 2.5.0 * pydantic-settings >= 2.1.0 * sentry-sdk >= 1.35.0 * pytest >= 7.4.3 +* pytest-cov >= 4.1.0 +* anyio >= 4.0.0 ## Getting started 1. Create a Python Virtual Environment: `python -m venv env` 2. Activate the Virtual Environment: `source env/bin/activate` 3. Install dependencies: `pip install -r requirements.txt` -4. Create a `vars.env` file -``` -REQUESTS_LIMIT="5" -SLEEP_TIMEOUT="600" -BS_API_URL="http://web_server:8000" -LOGGING_LEVEL="DEBUG" -BS_JWT_TOKEN="" -CACHER_SENTRY_ENVIRONMENT="dev" -CACHER_SENTRY_DSN="" -CACHER_SENTRY_TRACES_SAMPLE_RATE="0.2" +4. Create a `vars.yaml` file +```yaml +--- +requests_limit: 5 +sleep_timeout: 600 +bs_api_url: http://web_server:8000 +logging_level: DEBUG +bs_jwt_token: +cacher_sentry_environment: dev +cacher_sentry_dsn: +cacher_sentry_traces_sample_rate: 0.2 ``` ## Running the AlmaLinux tests cacher diff --git a/alma_tests_cacher.py b/alma_tests_cacher.py index 9c3a228..42f6441 100644 --- a/alma_tests_cacher.py +++ b/alma_tests_cacher.py @@ -3,11 +3,11 @@ import sentry_sdk from alma_tests_cacher.cacher import AlmaTestsCacher -from alma_tests_cacher.models import Config +from alma_tests_cacher.utils import get_config async def main(): - config = Config() + config = get_config() if config.cacher_sentry_dsn: sentry_sdk.init( dsn=config.cacher_sentry_dsn, @@ -19,6 +19,8 @@ async def main(): sleep_timeout=config.sleep_timeout, bs_api_url=config.bs_api_url, bs_jwt_token=config.bs_jwt_token, + common_test_dir_name=config.common_test_dir_name, + gerrit_username=config.gerrit_username, ).run() diff --git a/alma_tests_cacher/cacher.py b/alma_tests_cacher/cacher.py index 7d58a38..6ecc3b7 100644 --- a/alma_tests_cacher/cacher.py +++ b/alma_tests_cacher/cacher.py @@ -2,6 +2,8 @@ import logging import re import urllib.parse +from pathlib import Path +from tempfile import TemporaryDirectory from typing import List, Optional, Union import aiohttp @@ -13,6 +15,11 @@ DEFAULT_SLEEP_TIMEOUT, ) from alma_tests_cacher.models import PackageTestRepository, TestRepository +from alma_tests_cacher.utils import ( + clone_git_repo, + git_pull, + prepare_gerrit_repo_url, +) class AlmaTestsCacher: @@ -23,6 +30,8 @@ def __init__( sleep_timeout: int = DEFAULT_SLEEP_TIMEOUT, bs_api_url: str = DEFAULT_BS_API_URL, logging_level: str = DEFAULT_LOGGING_LEVEL, + common_test_dir_name: str = '', + gerrit_username: str = '', ): self.requests_limit = asyncio.Semaphore(requests_limit) self.sleep_timeout = sleep_timeout @@ -33,6 +42,8 @@ def __init__( self.bs_jwt_token = bs_jwt_token self.session_mapping = {} self.logger = self.setup_logger(logging_level) + self.common_test_dir_name = common_test_dir_name + self.gerrit_username = gerrit_username def setup_logger(self, logging_level: str) -> logging.Logger: logger = logging.getLogger('tests-cacher') @@ -141,50 +152,67 @@ async def bulk_create_test_folders( f'/api/v1/test_repositories/{repository_id}/packages/bulk_create/', ), method='post', - json=[test_folder.dict() for test_folder in test_folders], + json=[ + test_folder.model_dump() for test_folder in test_folders + ], headers=self.bs_headers, ) except Exception: self.logger.exception('Cannot create new test folders:') - async def get_test_repo_content(self, url: str) -> str: - repo_content = '' - try: - repo_content = await self.make_request( - method='get', - endpoint=url, - return_text=True, - ) - except Exception: - self.logger.exception('Cannot get repo content:') - return repo_content - async def process_repo( self, - db_repo: TestRepository, + repo: TestRepository, + workdir: str, ): async with self.requests_limit: remote_test_folders = [] new_test_folders = [] - tests_prefix = db_repo.tests_prefix if db_repo.tests_prefix else '' - self.logger.info('Start processing "%s" repo', db_repo.name) - repo_content = await self.get_test_repo_content( - urllib.parse.urljoin(db_repo.url, db_repo.tests_dir), - ) - for line in repo_content.splitlines(): - result = re.search( - rf'href="({tests_prefix}.+)"', - line, + tests_prefix = repo.tests_prefix if repo.tests_prefix else '' + self.logger.info('Start processing "%s" repo', repo.name) + repo_dirname = Path(repo.url).name.replace('.git', '') + if 'gerrit' in repo.url: + repo.url = prepare_gerrit_repo_url( + repo.url, + self.gerrit_username, ) - if not result: + repo_dir = Path(workdir, repo_dirname) + if not repo_dir.exists(): + self.logger.info('Start cloning git repo: %s', repo.url) + exit_code, stdout, stderr = clone_git_repo(workdir, repo.url) + self.logger.debug( + 'Clone result:\nexit_code: %s\nstdout: %s\nstderr: %s', + exit_code, + stdout, + stderr, + ) + else: + self.logger.info( + 'Pulling the latest changes for git repo: %s', + repo.url, + ) + exit_code, stdout, stderr = git_pull(workdir) + self.logger.debug( + 'Pull result:\nexit_code: %s\nstdout: %s\nstderr: %s', + exit_code, + stdout, + stderr, + ) + regex_pattern = rf'^{tests_prefix}' + if self.common_test_dir_name: + regex_pattern = ( + rf'^({tests_prefix}|{self.common_test_dir_name})' + ) + for folder in repo_dir.glob(f'{repo.tests_dir}*'): + if not re.search(regex_pattern, folder.name): continue - remote_test_folders.append(result.groups()[0]) + remote_test_folders.append(folder.name) test_folders_mapping = { - test.folder_name: test for test in db_repo.packages + test.folder_name: test for test in repo.packages } for remote_test_folder in remote_test_folders: - db_test = test_folders_mapping.get(remote_test_folder) - if db_test: + existent_test = test_folders_mapping.get(remote_test_folder) + if existent_test: continue new_test = PackageTestRepository( folder_name=remote_test_folder, @@ -194,45 +222,50 @@ async def process_repo( remote_test_folder, ), url=urllib.parse.urljoin( - urllib.parse.urljoin(db_repo.url, db_repo.tests_dir), + urllib.parse.urljoin(repo.url, repo.tests_dir), remote_test_folder, ), ) new_test_folders.append(new_test) - db_repo.packages.append(new_test) - await self.bulk_create_test_folders(new_test_folders, db_repo.id) + repo.packages.append(new_test) + await self.bulk_create_test_folders(new_test_folders, repo.id) await self.bulk_remove_test_folders( [ - db_test.id - for db_test in db_repo.packages - if db_test.folder_name not in remote_test_folders - and db_test.id + existent_test.id + for existent_test in repo.packages + if existent_test.folder_name not in remote_test_folders + and existent_test.id ], - db_repo.id, + repo.id, ) - if not remote_test_folders and db_repo.packages: + if not remote_test_folders and repo.packages: await self.bulk_remove_test_folders( - [db_test.id for db_test in db_repo.packages if db_test.id], - db_repo.id, + [ + existent_test.id + for existent_test in repo.packages + if existent_test.id + ], + repo.id, ) - self.logger.info('Repo "%s" is processed', db_repo.name) + self.logger.info('Repo "%s" is processed', repo.name) async def run(self, dry_run: bool = False): - while True: - self.logger.info('Start processing test repositories') - try: - db_repos = await self.get_test_repositories() - await asyncio.gather( - *(self.process_repo(db_repo) for db_repo in db_repos) + with TemporaryDirectory(prefix='alma-cacher-') as workdir: + while True: + self.logger.info('Start processing test repositories') + try: + repos = await self.get_test_repositories() + await asyncio.gather( + *(self.process_repo(repo, workdir) for repo in repos) + ) + except Exception: + self.logger.exception('Cannot process test repositories:') + finally: + await self.close_sessions() + self.logger.info( + 'All repositories are processed, sleeping %d seconds', + self.sleep_timeout, ) - except Exception: - self.logger.exception('Cannot process test repositories:') - finally: - await self.close_sessions() - self.logger.info( - 'All repositories are processed, sleeping %d seconds', - self.sleep_timeout, - ) - await asyncio.sleep(self.sleep_timeout) - if dry_run: - break + await asyncio.sleep(self.sleep_timeout) + if dry_run: + break diff --git a/alma_tests_cacher/constants.py b/alma_tests_cacher/constants.py index 2db4eeb..35e8e62 100644 --- a/alma_tests_cacher/constants.py +++ b/alma_tests_cacher/constants.py @@ -1,4 +1,7 @@ +import os + DEFAULT_REQUESTS_LIMIT = 5 DEFAULT_SLEEP_TIMEOUT = 600 # 10 minutes DEFAULT_BS_API_URL = 'http://web_server:8000' DEFAULT_LOGGING_LEVEL = 'DEBUG' +DEFAULT_CONFIG_PATH = os.getenv('CONFIG_PATH', 'vars.yaml') diff --git a/alma_tests_cacher/models.py b/alma_tests_cacher/models.py index 09cf9e9..568df82 100644 --- a/alma_tests_cacher/models.py +++ b/alma_tests_cacher/models.py @@ -1,7 +1,7 @@ from typing import List, Optional -from pydantic import AnyHttpUrl, BaseModel -from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic import BaseModel +from pydantic_settings import BaseSettings from alma_tests_cacher.constants import ( DEFAULT_BS_API_URL, @@ -11,19 +11,6 @@ ) -class Config(BaseSettings): - model_config = SettingsConfigDict(env_file='vars.env') - - requests_limit: int = DEFAULT_REQUESTS_LIMIT - sleep_timeout: int = DEFAULT_SLEEP_TIMEOUT - bs_api_url: str = DEFAULT_BS_API_URL - logging_level: str = DEFAULT_LOGGING_LEVEL - bs_jwt_token: str = '' - cacher_sentry_environment: str = "dev" - cacher_sentry_dsn: str = "" - cacher_sentry_traces_sample_rate: float = 0.2 - - class PackageTestRepository(BaseModel): id: Optional[int] = None package_name: str @@ -34,7 +21,20 @@ class PackageTestRepository(BaseModel): class TestRepository(BaseModel): id: int name: str - url: AnyHttpUrl + url: str tests_dir: str - tests_prefix: Optional[str] + tests_prefix: Optional[str] = '' packages: List[PackageTestRepository] + + +class Config(BaseSettings): + requests_limit: int = DEFAULT_REQUESTS_LIMIT + sleep_timeout: int = DEFAULT_SLEEP_TIMEOUT + bs_api_url: str = DEFAULT_BS_API_URL + logging_level: str = DEFAULT_LOGGING_LEVEL + bs_jwt_token: str = '' + cacher_sentry_environment: str = "dev" + cacher_sentry_dsn: str = "" + cacher_sentry_traces_sample_rate: float = 0.2 + common_test_dir_name: str = "" + gerrit_username: str = "" diff --git a/alma_tests_cacher/utils.py b/alma_tests_cacher/utils.py new file mode 100644 index 0000000..61ac3f5 --- /dev/null +++ b/alma_tests_cacher/utils.py @@ -0,0 +1,56 @@ +from pathlib import Path +from typing import Optional, Tuple +from urllib.parse import urlparse, urlunparse + +import yaml +from plumbum import local + +from alma_tests_cacher.constants import DEFAULT_CONFIG_PATH +from alma_tests_cacher.models import Config + + +def get_config(config_path: Optional[Path] = None) -> Config: + with open(config_path or DEFAULT_CONFIG_PATH, 'rt') as file: + return Config.model_validate(yaml.safe_load(file)) + + +def prepare_gerrit_repo_url(url: str, username: str) -> str: + parsed = urlparse(url) + return urlunparse( + ( + parsed.scheme, + f'{username}@{parsed.netloc}', + parsed.path, + parsed.params, + parsed.query, + parsed.fragment, + ) + ) + + +def clone_git_repo( + workdir_path: str, + repo_url: str, +) -> Tuple[int, str, str]: + return ( + local['git'] + .with_cwd(workdir_path) + .run( + ['clone', repo_url], + retcode=None, + ) + ) + + +def git_pull( + workdir_path: str, + branch: str = 'master', +) -> Tuple[int, str, str]: + return ( + local['git'] + .with_cwd(workdir_path) + .run( + ['pull', 'origin', branch], + retcode=None, + ) + ) diff --git a/docker-compose.yml b/docker-compose.yml index 6fb1059..38d5f09 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,7 @@ version: "3.9" - services: - alma_tests_cacher: + cacher: image: alma-tests-cacher:latest - env_file: - - vars.env build: context: . dockerfile: Dockerfile @@ -13,6 +10,7 @@ services: - "./alma_tests_cacher:/code/alma_tests_cacher" - "./alma_tests_cacher.py:/code/alma_tests_cacher.py" - "./requirements.txt:/code/requirements.txt" + - "./vars.yaml:/code/vars.yaml" command: "bash -c '/wait_for_it.sh web_server:8000 && source env/bin/activate && pip3 install --upgrade pip && diff --git a/requirements.txt b/requirements.txt index fb96de2..f0eb2c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,8 @@ aiohttp==3.8.6 pydantic==2.5.0 pydantic-settings==2.1.0 sentry-sdk==1.35.0 +PyYAML==6.0.1 +plumbum==1.8.2 pytest==7.4.3 +pytest-cov==4.1.0 +anyio==4.0.0 From c1b9d5adc2411780c3ae79f22acb1c7dcf4b6784 Mon Sep 17 00:00:00 2001 From: Daniil Anfimov Date: Wed, 22 Nov 2023 14:43:54 +0100 Subject: [PATCH 2/4] Add unit-tests --- tests/test_cacher.py | 132 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 126 insertions(+), 6 deletions(-) diff --git a/tests/test_cacher.py b/tests/test_cacher.py index f2330ff..19f5d1e 100644 --- a/tests/test_cacher.py +++ b/tests/test_cacher.py @@ -1,19 +1,139 @@ +import json +from pathlib import Path +from typing import Any, Dict + import pytest from alma_tests_cacher.cacher import AlmaTestsCacher -from alma_tests_cacher.models import Config +from alma_tests_cacher.models import Config, TestRepository +from alma_tests_cacher.utils import get_config + + +@pytest.fixture +def default_vars() -> str: + return """ + sleep_timeout: 0 + common_test_dir_name: common + """ @pytest.fixture -def config(): - yield Config() +def config(tmp_path: Path, default_vars: str) -> Config: + conf_path = Path(tmp_path, 'vars.yaml') + conf_path.write_text(default_vars) + return get_config(config_path=conf_path) -def test_cacher_init(config: Config): - cacher = AlmaTestsCacher( +@pytest.fixture(scope='session') +def anyio_backend(): + return 'asyncio' + + +@pytest.fixture +def cacher(config: Config) -> AlmaTestsCacher: + return AlmaTestsCacher( requests_limit=config.requests_limit, - sleep_timeout=config.sleep_timeout, + sleep_timeout=0, bs_api_url=config.bs_api_url, bs_jwt_token=config.bs_jwt_token, + common_test_dir_name=config.common_test_dir_name, + gerrit_username=config.gerrit_username, + ) + + +@pytest.fixture +def repo_payload() -> TestRepository: + return TestRepository( + id=1, + name='test-repo', + url='git@github.com:anfimovdm/third-party-tests.git', + tests_dir='rpm_tests/', + tests_prefix='p_', + packages=[], ) + + +@pytest.fixture +def expected_payload() -> Dict[str, Any]: + return { + 'repo_id': 1, + 'test_repos': [ + { + 'id': None, + 'package_name': 'chan', + 'folder_name': 'p_chan', + 'url': 'git@github.com:anfimovdm/rpm_tests/p_chan', + }, + { + 'id': None, + 'package_name': 'common', + 'folder_name': 'common', + 'url': 'git@github.com:anfimovdm/rpm_tests/common', + }, + ], + } + + +def test_config(config: Config): + for assert_expr in ( + isinstance(config, Config), + config.sleep_timeout == 0, + config.common_test_dir_name == 'common', + ): + assert assert_expr + + +def test_cacher_init(cacher: AlmaTestsCacher): assert isinstance(cacher, AlmaTestsCacher) + + +@pytest.fixture +def mock_get_test_repos( + monkeypatch: pytest.MonkeyPatch, + repo_payload: TestRepository, +): + async def func(*args, **kwargs): + return [repo_payload] + + monkeypatch.setattr(AlmaTestsCacher, 'get_test_repositories', func) + + +@pytest.fixture +def mock_bulk_remove(monkeypatch: pytest.MonkeyPatch): + async def func(*args, **kwargs): + return + + monkeypatch.setattr(AlmaTestsCacher, 'bulk_remove_test_folders', func) + + +@pytest.fixture +def mock_bulk_create(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + async def func(*args, **kwargs): + _, test_repos, repo_id = args + test_repos = [repo.model_dump() for repo in test_repos] + Path(tmp_path, 'create_payload.json').write_text( + json.dumps({'repo_id': repo_id, 'test_repos': test_repos}), + ) + return + + monkeypatch.setattr(AlmaTestsCacher, 'bulk_create_test_folders', func) + + +@pytest.mark.anyio +@pytest.mark.usefixtures( + 'mock_get_test_repos', + 'mock_bulk_create', + 'mock_bulk_remove', +) +async def test_cacher_run( + cacher: AlmaTestsCacher, + tmp_path: Path, + expected_payload: Dict[str, Any], +): + await cacher.run(dry_run=True) + payload = json.loads(Path(tmp_path, 'create_payload.json').read_text()) + for assert_expr in ( + payload['repo_id'] == expected_payload['repo_id'], + payload['test_repos'] == payload['test_repos'], + ): + assert assert_expr From 037b346b443a2a308ad768010ec3ddebf730aa02 Mon Sep 17 00:00:00 2001 From: Daniil Anfimov Date: Wed, 22 Nov 2023 14:57:58 +0100 Subject: [PATCH 3/4] Add GH actions --- .github/workflows/commit-message.yml | 28 ++++++++++++++ .github/workflows/pytest.yml | 39 +++++++++++++++++++ .github/workflows/syntax.yml | 56 ++++++++++++++++++++++++++++ README.md | 5 +-- pyproject.toml | 11 ++++++ requirements.txt | 3 -- test-requirements.txt | 5 +++ 7 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/commit-message.yml create mode 100644 .github/workflows/pytest.yml create mode 100644 .github/workflows/syntax.yml create mode 100644 pyproject.toml create mode 100644 test-requirements.txt diff --git a/.github/workflows/commit-message.yml b/.github/workflows/commit-message.yml new file mode 100644 index 0000000..f2422e9 --- /dev/null +++ b/.github/workflows/commit-message.yml @@ -0,0 +1,28 @@ +name: 'Commit Message Check' +on: + pull_request: + types: + - opened + - edited + - reopened + - synchronize +jobs: + check-commit-message: + name: Check Commit Message + runs-on: ubuntu-latest + steps: + - name: Check Line Length + uses: gsactions/commit-message-checker@v2 + with: + pattern: '^.{4,72}' + flags: 'gms' + error: 'The maximum line length of 72 characters is exceeded.' + excludeDescription: 'true' + excludeTitle: 'true' + - name: Check for Resolves / Fixes + uses: gsactions/commit-message-checker@v2 + with: + pattern: '(Resolves|Fixes) AlmaLinux\/build-system\#[0-9]+$' + error: 'You need at least one "Resolves|Fixes: " line.' + excludeTitle: 'true' + excludeDescription: 'true' diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 0000000..154df94 --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,39 @@ +name: pytest +on: + pull_request: + branches: + - "**" +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v3 + name: Check out repository + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Prepare python env + run: | + python -m venv env + source env/bin/activate + pip install -U pip + pip install -r test-requirements.txt + - name: Run unit tests (pytest) + run: bash -o pipefail -c 'source + env/bin/activate && pip3 install -r requirements.txt && + pytest -vv --cov-report term-missing:skip-covered + --cov-report xml:/tmp/coverage.xml --junitxml=/tmp/pytest.xml --cov=alma_tests_cacher + tests/ | tee /tmp/pytest-coverage.txt' + - name: Pytest coverage comment + uses: MishaKav/pytest-coverage-comment@main + with: + pytest-coverage-path: /tmp/pytest-coverage.txt + pytest-xml-coverage-path: /tmp/coverage.xml + title: Coverage report for changed files + badge-title: Total coverage + hide-badge: false + hide-report: false + report-only-changed-files: true + hide-comment: false + remove-link-from-badge: false + junitxml-path: /tmp/pytest.xml diff --git a/.github/workflows/syntax.yml b/.github/workflows/syntax.yml new file mode 100644 index 0000000..0c9310a --- /dev/null +++ b/.github/workflows/syntax.yml @@ -0,0 +1,56 @@ +name: syntax +on: + pull_request: + branches: + - "**" +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v39 + with: + files: | + **/*.py + - name: Prepare python env + if: ${{ steps.changed-files.outputs.all_changed_files }} + run: | + python -m venv env + source env/bin/activate + pip install -U pip + pip install -r test-requirements.txt + pip install isort[colors] black pylint + - name: Run pylint + id: pylint + if: ${{ steps.changed-files.outputs.all_changed_files }} + run: | + delimiter='$(openssl rand -hex 8)' + echo 'report<<$(delimiter)' >> $GITHUB_OUTPUT + source env/bin/activate + pylint ${{ steps.changed-files.outputs.all_changed_files }} --exit-zero >> $GITHUB_OUTPUT + echo '$(delimiter)' >> $GITHUB_OUTPUT + - name: Post pylint output + uses: mshick/add-pr-comment@v2 + if: ${{ steps.changed-files.outputs.all_changed_files }} + with: + message: | +
+ pylint output + + ``` + ${{ steps.pylint.outputs.report }} + ``` +
+ message-id: pylint-report + - name: Run black + if: ${{ steps.changed-files.outputs.all_changed_files }} + run: env/bin/black ${{ steps.changed-files.outputs.all_changed_files }} --check --diff --color + - name: Run isort + if: ${{ steps.changed-files.outputs.all_changed_files }} + run: env/bin/isort ${{ steps.changed-files.outputs.all_changed_files }} --diff --color --check-only diff --git a/README.md b/README.md index ad91bc6..ccaf024 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,8 @@ Tool for caching third-party tests for ALTS from git repositories. * pydantic >= 2.5.0 * pydantic-settings >= 2.1.0 * sentry-sdk >= 1.35.0 -* pytest >= 7.4.3 -* pytest-cov >= 4.1.0 -* anyio >= 4.0.0 +* PyYAML >= 6.0.1 +* plumbum >= 1.8.2 ## Getting started diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3502609 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,11 @@ +[tool.isort] +profile = "black" +split_on_trailing_comma = true + +[tool.black] +line-length = 79 +skip-string-normalization = true +preview = true + +[tool.pylint] +max-line-length = 80 diff --git a/requirements.txt b/requirements.txt index f0eb2c9..0fc7897 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,3 @@ pydantic-settings==2.1.0 sentry-sdk==1.35.0 PyYAML==6.0.1 plumbum==1.8.2 -pytest==7.4.3 -pytest-cov==4.1.0 -anyio==4.0.0 diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..079bb5c --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,5 @@ +-r requirements.txt + +pytest==7.4.3 +pytest-cov==4.1.0 +anyio==4.0.0 From 2484041909ffd240af0a33844a01e873a8a9e9ac Mon Sep 17 00:00:00 2001 From: Daniil Anfimov Date: Thu, 30 Nov 2023 13:47:35 +0100 Subject: [PATCH 4/4] Bug fixes --- alma_tests_cacher.py | 1 - alma_tests_cacher/cacher.py | 6 ++---- alma_tests_cacher/models.py | 12 ++++++------ tests/test_cacher.py | 3 --- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/alma_tests_cacher.py b/alma_tests_cacher.py index 42f6441..bfb66c4 100644 --- a/alma_tests_cacher.py +++ b/alma_tests_cacher.py @@ -19,7 +19,6 @@ async def main(): sleep_timeout=config.sleep_timeout, bs_api_url=config.bs_api_url, bs_jwt_token=config.bs_jwt_token, - common_test_dir_name=config.common_test_dir_name, gerrit_username=config.gerrit_username, ).run() diff --git a/alma_tests_cacher/cacher.py b/alma_tests_cacher/cacher.py index 6ecc3b7..2a4afef 100644 --- a/alma_tests_cacher/cacher.py +++ b/alma_tests_cacher/cacher.py @@ -30,7 +30,6 @@ def __init__( sleep_timeout: int = DEFAULT_SLEEP_TIMEOUT, bs_api_url: str = DEFAULT_BS_API_URL, logging_level: str = DEFAULT_LOGGING_LEVEL, - common_test_dir_name: str = '', gerrit_username: str = '', ): self.requests_limit = asyncio.Semaphore(requests_limit) @@ -42,7 +41,6 @@ def __init__( self.bs_jwt_token = bs_jwt_token self.session_mapping = {} self.logger = self.setup_logger(logging_level) - self.common_test_dir_name = common_test_dir_name self.gerrit_username = gerrit_username def setup_logger(self, logging_level: str) -> logging.Logger: @@ -199,9 +197,9 @@ async def process_repo( stderr, ) regex_pattern = rf'^{tests_prefix}' - if self.common_test_dir_name: + if repo.common_test_dir_name: regex_pattern = ( - rf'^({tests_prefix}|{self.common_test_dir_name})' + rf'^({tests_prefix}|{repo.common_test_dir_name})' ) for folder in repo_dir.glob(f'{repo.tests_dir}*'): if not re.search(regex_pattern, folder.name): diff --git a/alma_tests_cacher/models.py b/alma_tests_cacher/models.py index 568df82..691ad2d 100644 --- a/alma_tests_cacher/models.py +++ b/alma_tests_cacher/models.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import List, Optional, Union from pydantic import BaseModel from pydantic_settings import BaseSettings @@ -19,12 +19,13 @@ class PackageTestRepository(BaseModel): class TestRepository(BaseModel): - id: int + id: Union[int, str] name: str url: str tests_dir: str tests_prefix: Optional[str] = '' packages: List[PackageTestRepository] + common_test_dir_name: str = '' class Config(BaseSettings): @@ -33,8 +34,7 @@ class Config(BaseSettings): bs_api_url: str = DEFAULT_BS_API_URL logging_level: str = DEFAULT_LOGGING_LEVEL bs_jwt_token: str = '' - cacher_sentry_environment: str = "dev" - cacher_sentry_dsn: str = "" + cacher_sentry_environment: str = 'dev' + cacher_sentry_dsn: str = '' cacher_sentry_traces_sample_rate: float = 0.2 - common_test_dir_name: str = "" - gerrit_username: str = "" + gerrit_username: str = '' diff --git a/tests/test_cacher.py b/tests/test_cacher.py index 19f5d1e..86ba006 100644 --- a/tests/test_cacher.py +++ b/tests/test_cacher.py @@ -13,7 +13,6 @@ def default_vars() -> str: return """ sleep_timeout: 0 - common_test_dir_name: common """ @@ -36,7 +35,6 @@ def cacher(config: Config) -> AlmaTestsCacher: sleep_timeout=0, bs_api_url=config.bs_api_url, bs_jwt_token=config.bs_jwt_token, - common_test_dir_name=config.common_test_dir_name, gerrit_username=config.gerrit_username, ) @@ -78,7 +76,6 @@ def test_config(config: Config): for assert_expr in ( isinstance(config, Config), config.sleep_timeout == 0, - config.common_test_dir_name == 'common', ): assert assert_expr