Merge pull request #983 from bklvsky/test_package_exporter

Test integration between packages_exporter and albs-sign-file

.github/workflows/preflight.yml

@@ -42,6 +42,12 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v4
 
+      - name: Checkout albs-sign-file repository
+        uses: actions/checkout@v4
+        with:
+          repository: AlmaLinux/albs-sign-file
+          path: albs-sign-file
+
       - name: Set up Docker Buildx
         # https://github.com/marketplace/actions/docker-setup-buildx
         uses: docker/setup-buildx-action@v3
@@ -69,15 +75,19 @@ jobs:
               - added|modified: 'alws/**/*.py'
               - added|modified: 'scripts/**/*.py'
 
+      - name: Prepare GPG key
+        run: |
+          mv albs-sign-file ../
+          bash tests/prepare_gpg_key.sh
+
       - name: Prepare working directory
         run: |
           mkdir -p $REPORTS_DIR
-          mkdir -p ../{alts,albs-frontend,albs-node,albs-sign-file,albs-sign-node,alma-tests-cacher}
-          touch ../albs-sign-file/.env
+          mkdir -p ../{alts,albs-frontend,albs-node,albs-sign-node,alma-tests-cacher}
           ln -sf tests/test-vars.env vars.env
 
       - name: Start services
-        run: docker compose up -d test_db
+        run: docker compose up -d test_db sign_file
 
       - name: Run pylint
         if: ${{ steps.changed-files.outputs.src == 'true' }}

.github/workflows/run-tests.yml

@@ -19,6 +19,12 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v4
 
+      - name: Checkout albs-sign-file repository
+        uses: actions/checkout@v4
+        with:
+          repository: AlmaLinux/albs-sign-file
+          path: albs-sign-file
+
       - name: Set up Docker Buildx
         # https://github.com/marketplace/actions/docker-setup-buildx
         uses: docker/setup-buildx-action@v3
@@ -33,15 +39,19 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+      - name: Prepare GPG key
+        run: |
+          mv albs-sign-file ../
+          bash tests/prepare_gpg_key.sh
+
       - name: Prepare working directory
         run: |
           mkdir -p $REPORTS_DIR
-          mkdir -p ../{alts,albs-frontend,albs-node,albs-sign-file,albs-sign-node,alma-tests-cacher}
-          touch ../albs-sign-file/.env
+          mkdir -p ../{alts,albs-frontend,albs-node,albs-sign-node,alma-tests-cacher}
           ln -sf tests/test-vars.env vars.env
 
       - name: Start services
-        run: docker compose up -d test_db
+        run: docker compose up -d test_db sign_file
 
       - name: Run pytest
         id: pytest

alws/config.py

@@ -10,7 +10,9 @@ class Settings(BaseSettings):
     pulp_user: str = 'admin'
     pulp_password: str = 'admin'
     pulp_export_path: str = '/srv/exports'
-    pulp_database_url: str = 'postgresql+psycopg2://postgres:password@pulp:5432/pulp'
+    pulp_database_url: str = (
+        'postgresql+psycopg2://postgres:password@pulp:5432/pulp'
+    )
     pulp_async_database_url: str = (
         'postgresql+asyncpg://postgres:password@pulp:5432/pulp'
     )
@@ -29,7 +31,9 @@ class Settings(BaseSettings):
     test_database_url: str = (
         'postgresql+asyncpg://postgres:password@db/test-almalinux-bs'
     )
-    sync_database_url: str = 'postgresql+psycopg2://postgres:password@db/almalinux-bs'
+    sync_database_url: str = (
+        'postgresql+psycopg2://postgres:password@db/almalinux-bs'
+    )
 
     fastapi_sqla__async__sqlalchemy_url: str = (
         'postgresql+asyncpg://postgres:password@db/almalinux-bs'
@@ -79,12 +83,15 @@ class Settings(BaseSettings):
     sign_server_api_url: Optional[str] = 'http://sign_file:8000/'
     sign_server_username: Optional[str] = None
     sign_server_password: Optional[str] = None
+    test_sign_key_id: Optional[str] = None
 
     documentation_path: str = 'alws/documentation/'
 
     logging_level: Optional[str] = 'INFO'
 
-    frontend_baseurl: Annotated[str, Field(validate_default=True)] = 'http://localhost:8080'
+    frontend_baseurl: Annotated[str, Field(validate_default=True)] = (
+        'http://localhost:8080'
+    )
     github_callback_endpoint: str = 'api/v1/auth/github/callback'
     almalinux_callback_endpoint: str = 'api/v1/auth/almalinux/callback'
 

tests/README.md

@@ -18,11 +18,18 @@
    ```bash
    ln -sf tests/test-vars.env vars.env
    ```
-3. Start the `test_db` service
+2. Start the `test_db` service
     ```bash
     docker compose up -d test_db
     ```
-3. Run `pytest` within `web_server_tests` container
+
+3. To run packages_exporter tests start `sign-file` service and prepare GPG keys
+    ```bash
+    . tests/prepare_gpg_key.sh
+    docker compose up -d sign_file
+    ```
+
+4. Run `pytest` within `web_server_tests` container
     ```bash
     docker compose run --rm web_server_tests pytest -v
     ```

tests/fixtures/sign_keys.py

@@ -4,17 +4,23 @@
 from sqlalchemy import delete, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from alws.config import settings
 from alws.crud.sign_key import create_sign_key
 from alws.models import SignKey
 from alws.schemas.sign_schema import SignKeyCreate
 
 
 @pytest.fixture
 def basic_sign_key_payload() -> dict:
+    keyid = (
+        settings.test_sign_key_id
+        if settings.test_sign_key_id
+        else "1234567890ABCDEF"
+    )
     return {
         "name": "Test key",
         "description": "Test sign key",
-        "keyid": "1234567890ABCDEF",
+        "keyid": settings.test_sign_key_id,
         "fingerprint": "1234567890ABCDEF1234567890ABCDEF12345678",
         "public_url": "no_url",
     }

tests/prepare_gpg_key.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+gpg_key_blueprint=$( gpg --list-keys [email protected] | head -n 2 | tail -1 )
+
+# Generating GPG key if it doesn't exist
+if [ -z "$gpg_key_blueprint" ]; then
+    echo "No GPG key found, generating a new one..."
+
+    echo "Key-Type: 1
+    Key-Length: 2048
+    Subkey-Type: 1
+    Subkey-Length: 2048
+    Name-Real: Test GPG key for ALBS sign node
+    Name-Email: \"[email protected]\"
+    Expire-Date: 0
+    Passphrase: 1234567890" > gpg_key_scenario
+
+    gpg --batch --gen-key gpg_key_scenario
+    gpg_key_blueprint=$( gpg --list-keys [email protected] | head -n 2 | tail -1 )
+
+fi
+
+gpg_key_blueprint=${gpg_key_blueprint: -16}
+
+# Setting according env variables
+if [ ! -f "../albs-sign-file/.env" ]; then
+    echo "Generating .env for albs-sign-file"
+    echo "SF_PASS_DB_DEV_PASS=\"1234567890\"
+SF_PASS_DB_DEV_MODE=True
+SF_PGP_KEYS_ID=[\"$gpg_key_blueprint\"]
+SF_JWT_SECRET_KEY=\"access-secret\"
+SF_HOST_GNUPG=\"~/.gnupg\"
+SF_ROOT_URL=\"/sign-file\"" > ../albs-sign-file/.env
+fi
+
+sed -i 's/^TEST_SIGN_KEY_ID=".*"/TEST_SIGN_KEY_ID="'"$gpg_key_blueprint"'"/' tests/test-vars.env

tests/test-vars.env

@@ -19,4 +19,7 @@ PULP_DATABASE_URL="postgresql+psycopg2://postgres:password@test_db/test-almalinu
 SQLALCHEMY_POOL_PRE_PING=True
 SQLALCHEMY_POOL_RECYCLE=3600
 SQLALCHEMY_URL="postgresql+psycopg2://postgres:password@test_db/test-almalinux-bs"
-SYNC_DATABASE_URL="postgresql+psycopg2://postgres:password@test_db/test-almalinux-bs"
+SYNC_DATABASE_URL="postgresql+psycopg2://postgres:password@test_db/test-almalinux-bs"
+SIGN_SERVER_USERNAME="[email protected]"
+SIGN_SERVER_PASSWORD="test"
+TEST_SIGN_KEY_ID=""

tests/test_scripts/test_packages_exporter.py

@@ -1,24 +1,26 @@
 import os
 import tempfile
-
 import pytest
-from syncer import sync
 
+from alws.config import settings
+from alws.models import SignKey
 from scripts.packages_exporter import Exporter
+from tests.mock_classes import BaseAsyncTestCase
 
 IN_GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS") is not None
 
 
-@pytest.mark.skip(
-    reason="See https://github.com/AlmaLinux/build-system/issues/204"
-)
-def test_repomd_signer():
-    exporter = Exporter(
-        pulp_client=None, repodata_cache_dir='~/.cache/pulp_exporter'
+class TestPackagesExporter(BaseAsyncTestCase):
+    @pytest.mark.skipif(
+        not settings.test_sign_key_id, reason="Testing sign key is not provided"
     )
-    db_keys = sync(exporter.get_sign_keys())
-    key_id = db_keys[0]['keyid']
-    with tempfile.NamedTemporaryFile() as fp:
-        fp.write(b'Hello world!')
-        res = sync(exporter.sign_repomd_xml(fp.name, key_id))
+    async def test_repomd_signer(self, sign_key: SignKey, tmp_path):
+        exporter = Exporter(
+            pulp_client=None, repodata_cache_dir='~/.cache/pulp_exporter'
+        )
+        key_id = sign_key.keyid
+        token = await exporter.get_sign_server_token()
+        temp_file = tmp_path / "tempfile.txt"
+        temp_file.write_bytes(b'Hello world!')
+        res = await exporter.sign_repomd_xml(str(temp_file), key_id, token)
         assert res['error'] is None