Scale testing #260
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| # yamllint disable rule:truthy rule:truthy rule:line-length | |
| name: "Scale testing" | |
| on: | |
| schedule: | |
| - cron: "0 2 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| commit: | |
| type: string | |
| description: commit sha or branch name | |
| default: '' | |
| required: false | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| INFRAHUB_DB_USERNAME: neo4j | |
| INFRAHUB_DB_PASSWORD: admin | |
| INFRAHUB_DB_ADDRESS: localhost | |
| INFRAHUB_DB_PORT: 7687 | |
| INFRAHUB_DB_PROTOCOL: bolt | |
| INFRAHUB_BROKER_ADDRESS: message-queue | |
| INFRAHUB_LOG_LEVEL: CRITICAL | |
| INFRAHUB_IMAGE_NAME: "opsmill/infrahub" | |
| INFRAHUB_IMAGE_VER: "local" | |
| INFRAHUB_SERVER_PORT: 0 | |
| INFRAHUB_DB_BACKUP_PORT: 0 | |
| VMAGENT_PORT: 0 | |
| jobs: | |
| scale-tests: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| include: | |
| - name: 1000 nodes | |
| node-amount: 1000 | |
| attrs-amount: 0 | |
| rels-amount: 0 | |
| changes-amount: 0 | |
| stager: stage_infranode | |
| - name: 10000 nodes | |
| node-amount: 10000 | |
| attrs-amount: 0 | |
| rels-amount: 0 | |
| changes-amount: 0 | |
| stager: stage_infranode | |
| - name: 100000 nodes | |
| node-amount: 100000 | |
| attrs-amount: 0 | |
| rels-amount: 0 | |
| changes-amount: 0 | |
| stager: stage_infranode | |
| - name: 100 attrs | |
| node-amount: 10 | |
| attrs-amount: 100 | |
| rels-amount: 0 | |
| changes-amount: 0 | |
| stager: stage_infranode | |
| - name: 1000 attrs | |
| node-amount: 10 | |
| attrs-amount: 1000 | |
| rels-amount: 0 | |
| changes-amount: 0 | |
| stager: stage_infranode | |
| - name: 100 rels | |
| node-amount: 10 | |
| attrs-amount: 0 | |
| rels-amount: 100 | |
| changes-amount: 0 | |
| stager: stage_infranode | |
| - name: 1000 rels | |
| node-amount: 10 | |
| attrs-amount: 0 | |
| rels-amount: 1000 | |
| changes-amount: 0 | |
| stager: stage_infranode | |
| - name: 100 branches (with new nodes) | |
| node-amount: 100 | |
| attrs-amount: 0 | |
| rels-amount: 0 | |
| changes-amount: 0 | |
| stager: stage_branch | |
| - name: 100 branches (with single node) | |
| node-amount: 100 | |
| attrs-amount: 0 | |
| rels-amount: 0 | |
| changes-amount: 0 | |
| stager: stage_branch_update | |
| - name: 1000 branches (with new nodes) | |
| node-amount: 1000 | |
| attrs-amount: 0 | |
| rels-amount: 0 | |
| changes-amount: 0 | |
| stager: stage_branch | |
| - name: 1000 branches (with single node) | |
| node-amount: 1000 | |
| attrs-amount: 0 | |
| rels-amount: 0 | |
| changes-amount: 0 | |
| stager: stage_branch_update | |
| - name: 1000 updates of a single node | |
| node-amount: 1 | |
| attrs-amount: 0 | |
| rels-amount: 0 | |
| changes-amount: 1000 | |
| stager: stage_infranode | |
| - name: 100 updates of a single node | |
| node-amount: 1 | |
| attrs-amount: 0 | |
| rels-amount: 0 | |
| changes-amount: 100 | |
| stager: stage_infranode | |
| - name: 100 updates of 100 nodes | |
| node-amount: 100 | |
| attrs-amount: 0 | |
| rels-amount: 0 | |
| changes-amount: 100 | |
| stager: stage_infranode | |
| - name: 1000 changes in diff | |
| node-amount: 1000 | |
| attrs-amount: 0 | |
| rels-amount: 0 | |
| changes-amount: 0 | |
| stager: stage_branch_diff | |
| - name: 10000 changes in diff | |
| node-amount: 10000 | |
| attrs-amount: 0 | |
| rels-amount: 0 | |
| changes-amount: 0 | |
| stager: stage_branch_diff | |
| name: ${{ matrix.name }} | |
| runs-on: | |
| group: huge-runners | |
| timeout-minutes: 300 | |
| env: | |
| INFRAHUB_DB_TYPE: neo4j | |
| METRICS_ENDPOINT: ${{ secrets.METRICS_ENDPOINT }} | |
| steps: | |
| - name: "Check out repository code" | |
| uses: "actions/checkout@v4" | |
| with: | |
| ref: ${{ inputs.commit == '' && 'develop' || inputs.commit }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.12 | |
| - name: "Setup python environment" | |
| run: | | |
| pipx install poetry | |
| poetry config virtualenvs.prefer-active-python true | |
| pip install invoke toml | |
| - name: "Install Package" | |
| run: "poetry install --with test-scale" | |
| - name: "Set environment variables" | |
| run: echo INFRAHUB_BUILD_NAME=infrahub-${{ runner.name }} >> $GITHUB_ENV | |
| - name: "Set environment variables" | |
| run: echo INFRAHUB_IMAGE_VER=local-${{ runner.name }}-${{ github.sha }} >> $GITHUB_ENV | |
| - name: "Clear docker environment" | |
| run: docker compose -p $INFRAHUB_BUILD_NAME down -v --remove-orphans --rmi local | |
| - name: Set INFRAHUB_URL | |
| run: | | |
| PORT=$(docker compose -p $INFRAHUB_BUILD_NAME port infrahub-server 8000 | cut -d: -f2) | |
| echo "INFRAHUB_URL=http://localhost:${PORT}" >> $GITHUB_ENV | |
| - name: Set job name | |
| run: echo "JOB_NAME=${{ matrix.name }}" >> $GITHUB_ENV | |
| - name: "Start scale environment" | |
| run: "poetry run invoke backend.test-scale-env-start -g 1" | |
| - name: "Store start time" | |
| run: echo TEST_START_TIME=$(date +%s)000 >> $GITHUB_ENV | |
| - name: "Run test" | |
| run: "poetry run invoke backend.test-scale --amount ${{ matrix.node-amount }} --attrs ${{ matrix.attrs-amount }} --rels ${{ matrix.rels-amount }} --changes ${{ matrix.changes-amount }} -s backend/tests/scale/schema.yml --stager ${{ matrix.stager }}" | |
| - name: "Push metrics" | |
| if: always() | |
| run: 'curl -v --data-binary @metrics.csv "http://localhost:${VMAGENT_PORT}/api/v1/import/csv?format=1:metric:scale_test_response_time,2:time:unix_ms,3:label:operation_name,4:label:stage,5:label:node_amount,6:label:attrs_amount,7:label:rels_amount"' | |
| - name: "Wait for vmagent to push metrics" | |
| if: always() | |
| run: "sleep 10" | |
| - name: "Show graph URL" | |
| if: always() | |
| run: 'echo "https://grafana-prod.tailc018d.ts.net/d/ebf7ec72-db79-4fb7-9b46-4621ca9c407a/scale-tests?orgId=1&var-run_id=$GITHUB_RUN_ID&var-job=${JOB_NAME// /%20}&var-stage=test&var-node_amount=${{ matrix.node-amount }}&var-attrs_amount=${{ matrix.attrs-amount }}&var-rels_amount=${{ matrix.rels-amount }}&var-runner=$INFRAHUB_BUILD_NAME&from=$TEST_START_TIME&to=$(date +%s)000"' | |
| - name: Display server logs | |
| if: always() | |
| run: docker logs "${INFRAHUB_BUILD_NAME}-server-1" | |
| - name: "Destroy scale environment" | |
| if: always() | |
| run: "invoke backend.test-scale-env-destroy" | |
| # - name: "Print results" | |
| # run: "cat *.csv" |