ALBS-1168: Index git repositories instead of HTML pages

.github/workflows/commit-message.yml

@@ -0,0 +1,28 @@
+name: 'Commit Message Check'
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+jobs:
+  check-commit-message:
+    name: Check Commit Message
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check Line Length
+        uses: gsactions/commit-message-checker@v2
+        with:
+          pattern: '^.{4,72}'
+          flags: 'gms'
+          error: 'The maximum line length of 72 characters is exceeded.'
+          excludeDescription: 'true'
+          excludeTitle: 'true'
+      - name: Check for Resolves / Fixes
+        uses: gsactions/commit-message-checker@v2
+        with:
+          pattern: '(Resolves|Fixes) AlmaLinux\/build-system\#[0-9]+$'
+          error: 'You need at least one "Resolves|Fixes: <issue link>" line.'
+          excludeTitle: 'true'
+          excludeDescription: 'true'

.github/workflows/pytest.yml

@@ -0,0 +1,39 @@
+name: pytest
+on:
+  pull_request:
+    branches:
+      - "**"
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v3
+        name: Check out repository
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Prepare python env
+        run: |
+          python -m venv env
+          source env/bin/activate
+          pip install -U pip
+          pip install -r test-requirements.txt
+      - name: Run unit tests (pytest)
+        run: bash -o pipefail -c 'source
+          env/bin/activate && pip3 install -r requirements.txt &&
+          pytest -vv --cov-report term-missing:skip-covered
+          --cov-report xml:/tmp/coverage.xml --junitxml=/tmp/pytest.xml --cov=alma_tests_cacher
+          tests/ | tee /tmp/pytest-coverage.txt'
+      - name: Pytest coverage comment
+        uses: MishaKav/pytest-coverage-comment@main
+        with:
+          pytest-coverage-path: /tmp/pytest-coverage.txt
+          pytest-xml-coverage-path: /tmp/coverage.xml
+          title: Coverage report for changed files
+          badge-title: Total coverage
+          hide-badge: false
+          hide-report: false
+          report-only-changed-files: true
+          hide-comment: false
+          remove-link-from-badge: false
+          junitxml-path: /tmp/pytest.xml

.github/workflows/syntax.yml

@@ -0,0 +1,56 @@
+name: syntax
+on:
+  pull_request:
+    branches:
+      - "**"
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v39
+        with:
+          files: |
+            **/*.py
+      - name: Prepare python env
+        if: ${{ steps.changed-files.outputs.all_changed_files }}
+        run: |
+          python -m venv env
+          source env/bin/activate
+          pip install -U pip
+          pip install -r test-requirements.txt
+          pip install isort[colors] black pylint
+      - name: Run pylint
+        id: pylint
+        if: ${{ steps.changed-files.outputs.all_changed_files }}
+        run: |
+          delimiter='$(openssl rand -hex 8)'
+          echo 'report<<$(delimiter)' >> $GITHUB_OUTPUT
+          source env/bin/activate
+          pylint ${{ steps.changed-files.outputs.all_changed_files }} --exit-zero >> $GITHUB_OUTPUT
+          echo '$(delimiter)' >> $GITHUB_OUTPUT
+      - name: Post pylint output
+        uses: mshick/add-pr-comment@v2
+        if: ${{ steps.changed-files.outputs.all_changed_files }}
+        with:
+          message: |
+            <details>
+              <summary>pylint output</summary>
+
+            ```
+            ${{ steps.pylint.outputs.report }}
+            ```
+            </details>
+          message-id: pylint-report
+      - name: Run black
+        if: ${{ steps.changed-files.outputs.all_changed_files }}
+        run: env/bin/black ${{ steps.changed-files.outputs.all_changed_files }} --check --diff --color
+      - name: Run isort
+        if: ${{ steps.changed-files.outputs.all_changed_files }}
+        run: env/bin/isort ${{ steps.changed-files.outputs.all_changed_files }} --diff --color --check-only

.gitignore

@@ -122,6 +122,7 @@ celerybeat.pid
 # Environments
 .env
 .venv
+vars.yaml
 *.env
 env/
 venv/

Dockerfile

@@ -2,6 +2,7 @@ FROM almalinux:9
 
 RUN mkdir -p /code && \
     dnf update -y && \
+    dnf install git -y && \
     dnf clean all
 RUN curl https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh -o wait_for_it.sh && chmod +x wait_for_it.sh
 COPY ./requirements.txt /code/requirements.txt

README.md

@@ -4,28 +4,31 @@ Tool for caching third-party tests for ALTS from git repositories.
 
 ## Requirements
 
+* git >= 2.41.0
 * python >= 3.7
 * aiohttp >= 3.8.6
 * pydantic >= 2.5.0
 * pydantic-settings >= 2.1.0
 * sentry-sdk >= 1.35.0
-* pytest >= 7.4.3
+* PyYAML >= 6.0.1
+* plumbum >= 1.8.2
 
 ## Getting started
 
 1. Create a Python Virtual Environment: `python -m venv env`
 2. Activate the Virtual Environment: `source env/bin/activate`
 3. Install dependencies: `pip install -r requirements.txt`
-4. Create a `vars.env` file
-```
-REQUESTS_LIMIT="5"
-SLEEP_TIMEOUT="600"
-BS_API_URL="http://web_server:8000"
-LOGGING_LEVEL="DEBUG"
-BS_JWT_TOKEN=""
-CACHER_SENTRY_ENVIRONMENT="dev"
-CACHER_SENTRY_DSN=""
-CACHER_SENTRY_TRACES_SAMPLE_RATE="0.2"
+4. Create a `vars.yaml` file
+```yaml
+---
+requests_limit: 5
+sleep_timeout: 600
+bs_api_url: http://web_server:8000
+logging_level: DEBUG
+bs_jwt_token:
+cacher_sentry_environment: dev
+cacher_sentry_dsn:
+cacher_sentry_traces_sample_rate: 0.2
 ```
 
 ## Running the AlmaLinux tests cacher

alma_tests_cacher.py

@@ -3,11 +3,11 @@
 import sentry_sdk
 
 from alma_tests_cacher.cacher import AlmaTestsCacher
-from alma_tests_cacher.models import Config
+from alma_tests_cacher.utils import get_config
 
 
 async def main():
-    config = Config()
+    config = get_config()
     if config.cacher_sentry_dsn:
         sentry_sdk.init(
             dsn=config.cacher_sentry_dsn,
@@ -19,6 +19,7 @@ async def main():
         sleep_timeout=config.sleep_timeout,
         bs_api_url=config.bs_api_url,
         bs_jwt_token=config.bs_jwt_token,
+        gerrit_username=config.gerrit_username,
     ).run()