From e4f59deb812462acf27b3961881c788da6b32dad Mon Sep 17 00:00:00 2001 From: Thomas Carmet Date: Thu, 9 Jun 2022 18:47:33 -0700 Subject: [PATCH] RELENG-6059 add the ability to add more relabel jobs --- .../gh-action-exporter/templates/config.yaml | 7 ++ .../templates/deployment.yaml | 11 +++ charts/gh-action-exporter/values.yaml | 4 + gh_actions_exporter/Webhook.py | 8 +- gh_actions_exporter/config.py | 55 ++++++++++++ gh_actions_exporter/main.py | 33 +++++-- gh_actions_exporter/metrics.py | 46 ++++++++-- gh_actions_exporter/types.py | 1 + poetry.lock | 88 +++++++++---------- pyproject.toml | 1 + tests/api/conftest.py | 64 ++++++++++++-- tests/api/test_job.py | 29 ++++++ 12 files changed, 280 insertions(+), 67 deletions(-) create mode 100644 charts/gh-action-exporter/templates/config.yaml create mode 100644 gh_actions_exporter/config.py diff --git a/charts/gh-action-exporter/templates/config.yaml b/charts/gh-action-exporter/templates/config.yaml new file mode 100644 index 0000000..b469794 --- /dev/null +++ b/charts/gh-action-exporter/templates/config.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-config +data: + config.yaml: |- +{{ .Values.config | toYaml | indent 4 }} diff --git a/charts/gh-action-exporter/templates/deployment.yaml b/charts/gh-action-exporter/templates/deployment.yaml index 9effaf4..c440aae 100644 --- a/charts/gh-action-exporter/templates/deployment.yaml +++ b/charts/gh-action-exporter/templates/deployment.yaml @@ -15,6 +15,7 @@ spec: metadata: {{- with .Values.podAnnotations }} annotations: + checksum/config: {{ include (print $.Template.BasePath "/config.yaml") . | sha256sum }} {{- toYaml . | nindent 8 }} {{- end }} labels: @@ -47,6 +48,16 @@ spec: port: http resources: {{- toYaml .Values.resources | nindent 12 }} + env: + - name: CONFIG_FILE + value: /app/config/config.yaml + volumeMounts: + - name: config + mountPath: /app/config + volumes: + - name: config + configMap: + name: {{ .Release.Name }}-config {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/gh-action-exporter/values.yaml b/charts/gh-action-exporter/values.yaml index 80b1cc5..f4dd9f7 100644 --- a/charts/gh-action-exporter/values.yaml +++ b/charts/gh-action-exporter/values.yaml @@ -14,6 +14,10 @@ imagePullSecrets: [] nameOverride: "" fullnameOverride: "" + +config: + job_relabelling: [] + serviceAccount: # Specifies whether a service account should be created create: true diff --git a/gh_actions_exporter/Webhook.py b/gh_actions_exporter/Webhook.py index a79b6ad..5bd57e6 100644 --- a/gh_actions_exporter/Webhook.py +++ b/gh_actions_exporter/Webhook.py @@ -1,4 +1,5 @@ import logging +from gh_actions_exporter.config import Settings from gh_actions_exporter.types import WebHook from gh_actions_exporter.metrics import Metrics @@ -9,10 +10,11 @@ class WebhookManager(object): event: str payload: WebHook - def __init__(self, payload: WebHook, event: str, metrics: Metrics): + def __init__(self, payload: WebHook, event: str, metrics: Metrics, settings: Settings): self.event = event self.payload = payload self.metrics = metrics + self.settings = settings def __call__(self, *args, **kwargs): # Check if we managed this event @@ -28,8 +30,8 @@ def workflow_run(self): self.metrics.handle_workflow_rebuild(self.payload) def workflow_job(self): - self.metrics.handle_job_status(self.payload) - self.metrics.handle_job_duration(self.payload) + self.metrics.handle_job_status(self.payload, self.settings) + self.metrics.handle_job_duration(self.payload, self.settings) def ping(self): logger.info('Ping from Github') diff --git a/gh_actions_exporter/config.py b/gh_actions_exporter/config.py new file mode 100644 index 0000000..895a5cf --- /dev/null +++ b/gh_actions_exporter/config.py @@ -0,0 +1,55 @@ +import yaml +from enum import Enum +from pathlib import Path +from typing import Dict, Any, List, Optional + +from pydantic import BaseModel, BaseSettings + + +def yaml_config_settings_source(settings: BaseSettings) -> Dict[str, Any]: + """ + A simple settings source that loads variables from a yaml file + + """ + config_file: Path = settings.__config__.config.config_file + if config_file: + return yaml.full_load(config_file.read_text()) + return {} + + +class RelabelType(str, Enum): + name = 'name' + label = 'label' + + +class Relabel(BaseModel): + label: str + type: RelabelType = RelabelType.label + description: Optional[str] + default: Optional[str] = None + values: List[str] + + +class ConfigFile(BaseSettings): + config_file: Optional[Path] + + +class Settings(BaseSettings): + job_relabelling: Optional[List[Relabel]] = [] + + class Config: + config: ConfigFile = ConfigFile() + + @classmethod + def customise_sources( + cls, + init_settings, + env_settings, + file_secret_settings, + ): + return ( + init_settings, + yaml_config_settings_source, + env_settings, + file_secret_settings, + ) diff --git a/gh_actions_exporter/main.py b/gh_actions_exporter/main.py index 2e773cc..a2d2a5d 100644 --- a/gh_actions_exporter/main.py +++ b/gh_actions_exporter/main.py @@ -1,13 +1,26 @@ import uvicorn -from fastapi import FastAPI, Request +from fastapi import Depends, FastAPI, Request +from functools import lru_cache from gh_actions_exporter.metrics import prometheus_metrics, Metrics from gh_actions_exporter.types import WebHook from gh_actions_exporter.Webhook import WebhookManager +from gh_actions_exporter.config import Settings + + +@lru_cache() +def get_settings() -> Settings: + return Settings() + + +@lru_cache() +def metrics() -> Metrics: + return Metrics(get_settings()) + app = FastAPI() -metrics = Metrics() + app.add_route('/metrics', prometheus_metrics) @@ -18,13 +31,23 @@ def index(): @app.post("/webhook", status_code=202) -async def webhook(webhook: WebHook, request: Request): - WebhookManager(payload=webhook, event=request.headers['X-Github-Event'], metrics=metrics)() +async def webhook( + webhook: WebHook, + request: Request, + settings: Settings = Depends(get_settings), + metrics: Metrics = Depends(metrics) +): + WebhookManager( + payload=webhook, + event=request.headers['X-Github-Event'], + metrics=metrics, + settings=settings + )() return "Accepted" @app.delete("/clear", status_code=200) -async def clear(): +async def clear(metrics: Metrics = Depends(metrics)): metrics.workflow_rebuild.clear() metrics.workflow_duration.clear() metrics.job_duration.clear() diff --git a/gh_actions_exporter/metrics.py b/gh_actions_exporter/metrics.py index 4dbd1ec..a029ab9 100644 --- a/gh_actions_exporter/metrics.py +++ b/gh_actions_exporter/metrics.py @@ -3,9 +3,11 @@ from prometheus_client import (CONTENT_TYPE_LATEST, REGISTRY, CollectorRegistry, generate_latest) from prometheus_client.multiprocess import MultiProcessCollector +from typing import Dict, List from fastapi.requests import Request from fastapi.responses import Response -from gh_actions_exporter.types import WebHook +from gh_actions_exporter.config import Relabel, RelabelType, Settings +from gh_actions_exporter.types import WebHook, WorkflowJob from prometheus_client import Counter, Histogram @@ -23,8 +25,8 @@ def prometheus_metrics(request: Request) -> Response: class Metrics(object): - - def __init__(self): + def __init__(self, settings: Settings): + self.settings = settings self.workflow_labelnames = [ 'repository', 'workflow_name', @@ -36,6 +38,9 @@ def __init__(self): 'repository_visibility', 'runner_type' ] + for relabel in self.settings.job_relabelling: + self.job_labelnames.append(relabel.label) + self.workflow_rebuild = Counter( 'github_actions_workflow_rebuild_count', 'The number of workflow rebuild', labelnames=self.workflow_labelnames @@ -123,13 +128,38 @@ def runner_type(self, webhook: WebHook) -> str: return 'self-hosted' return 'github-hosted' - def job_labels(self, webhook: WebHook) -> dict: + def relabel_job_labels(self, relabel: Relabel, labels: List[str]) -> Dict[str, str or None]: + result = { + relabel.label: relabel.default + } + for label in relabel.values: + if label in labels: + result[relabel.label] = label + return result + + def relabel_job_names(self, relabel: Relabel, job: WorkflowJob) -> dict: + if job.status == 'queued': + return dict() + result = { + relabel.label: relabel.default + } + for label in relabel.values: + if label in job.runner_name: + result[relabel.label] = label + return result + + def job_labels(self, webhook: WebHook, settings: Settings) -> dict: labels = dict( runner_type=self.runner_type(webhook), job_name=webhook.workflow_job.name, repository_visibility=webhook.repository.visibility, repository=webhook.repository.full_name, ) + for relabel in settings.job_relabelling: + if relabel.type == RelabelType.label: + labels.update(self.relabel_job_labels(relabel, webhook.workflow_job.labels)) + elif relabel.type == RelabelType.name: + labels.update(self.relabel_job_names(relabel, webhook.workflow_job)) return labels def handle_workflow_rebuild(self, webhook: WebHook): @@ -165,8 +195,8 @@ def handle_workflow_duration(self, webhook: WebHook): - webhook.workflow_run.run_started_at.timestamp()) self.workflow_duration.labels(**labels).observe(duration) - def handle_job_status(self, webhook: WebHook): - labels = self.job_labels(webhook) + def handle_job_status(self, webhook: WebHook, settings: Settings): + labels = self.job_labels(webhook, settings) if webhook.workflow_job.conclusion: if webhook.workflow_job.conclusion == 'success': self.job_status_success.labels(**labels).inc() @@ -180,8 +210,8 @@ def handle_job_status(self, webhook: WebHook): elif webhook.workflow_job.status == 'queued': self.job_status_queued.labels(**labels).inc() - def handle_job_duration(self, webhook: WebHook): - labels = self.job_labels(webhook) + def handle_job_duration(self, webhook: WebHook, settings: Settings): + labels = self.job_labels(webhook, settings) if webhook.workflow_job.conclusion: duration = (webhook.workflow_job.completed_at.timestamp() - webhook.workflow_job.started_at.timestamp()) diff --git a/gh_actions_exporter/types.py b/gh_actions_exporter/types.py index ee9d9e7..7b650f9 100644 --- a/gh_actions_exporter/types.py +++ b/gh_actions_exporter/types.py @@ -18,6 +18,7 @@ class WorkflowJob(BaseModel): run_id: int run_url: str run_attempt: int + runner_name: Optional[str] = None node_id: str head_sha: str url: str diff --git a/poetry.lock b/poetry.lock index 6454de2..c62e134 100644 --- a/poetry.lock +++ b/poetry.lock @@ -88,14 +88,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "coverage" -version = "6.3.2" +version = "6.4.1" description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -tomli = {version = "*", optional = true, markers = "extra == \"toml\""} +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} [package.extras] toml = ["tomli"] @@ -459,7 +459,7 @@ python-versions = ">=3.7" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "5181341f20745174066c5277ab8d75ef1509c5f427bc5f1785f6e8e3f4463b1d" +content-hash = "49a002376a7c7ddf129da7cd118c2390d33c24081b94ed388bd46fb20e45cbe8" [metadata.files] anyio = [ @@ -495,47 +495,47 @@ colorama = [ {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] coverage = [ - {file = "coverage-6.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9b27d894748475fa858f9597c0ee1d4829f44683f3813633aaf94b19cb5453cf"}, - {file = "coverage-6.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:37d1141ad6b2466a7b53a22e08fe76994c2d35a5b6b469590424a9953155afac"}, - {file = "coverage-6.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9987b0354b06d4df0f4d3e0ec1ae76d7ce7cbca9a2f98c25041eb79eec766f1"}, - {file = "coverage-6.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:26e2deacd414fc2f97dd9f7676ee3eaecd299ca751412d89f40bc01557a6b1b4"}, - {file = "coverage-6.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4dd8bafa458b5c7d061540f1ee9f18025a68e2d8471b3e858a9dad47c8d41903"}, - {file = "coverage-6.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:46191097ebc381fbf89bdce207a6c107ac4ec0890d8d20f3360345ff5976155c"}, - {file = "coverage-6.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6f89d05e028d274ce4fa1a86887b071ae1755082ef94a6740238cd7a8178804f"}, - {file = "coverage-6.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:58303469e9a272b4abdb9e302a780072c0633cdcc0165db7eec0f9e32f901e05"}, - {file = "coverage-6.3.2-cp310-cp310-win32.whl", hash = "sha256:2fea046bfb455510e05be95e879f0e768d45c10c11509e20e06d8fcaa31d9e39"}, - {file = "coverage-6.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:a2a8b8bcc399edb4347a5ca8b9b87e7524c0967b335fbb08a83c8421489ddee1"}, - {file = "coverage-6.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f1555ea6d6da108e1999b2463ea1003fe03f29213e459145e70edbaf3e004aaa"}, - {file = "coverage-6.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5f4e1edcf57ce94e5475fe09e5afa3e3145081318e5fd1a43a6b4539a97e518"}, - {file = "coverage-6.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a15dc0a14008f1da3d1ebd44bdda3e357dbabdf5a0b5034d38fcde0b5c234b7"}, - {file = "coverage-6.3.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21b7745788866028adeb1e0eca3bf1101109e2dc58456cb49d2d9b99a8c516e6"}, - {file = "coverage-6.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8ce257cac556cb03be4a248d92ed36904a59a4a5ff55a994e92214cde15c5bad"}, - {file = "coverage-6.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b0be84e5a6209858a1d3e8d1806c46214e867ce1b0fd32e4ea03f4bd8b2e3359"}, - {file = "coverage-6.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:acf53bc2cf7282ab9b8ba346746afe703474004d9e566ad164c91a7a59f188a4"}, - {file = "coverage-6.3.2-cp37-cp37m-win32.whl", hash = "sha256:8bdde1177f2311ee552f47ae6e5aa7750c0e3291ca6b75f71f7ffe1f1dab3dca"}, - {file = "coverage-6.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b31651d018b23ec463e95cf10070d0b2c548aa950a03d0b559eaa11c7e5a6fa3"}, - {file = "coverage-6.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:07e6db90cd9686c767dcc593dff16c8c09f9814f5e9c51034066cad3373b914d"}, - {file = "coverage-6.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c6dbb42f3ad25760010c45191e9757e7dce981cbfb90e42feef301d71540059"}, - {file = "coverage-6.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c76aeef1b95aff3905fb2ae2d96e319caca5b76fa41d3470b19d4e4a3a313512"}, - {file = "coverage-6.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cf5cfcb1521dc3255d845d9dca3ff204b3229401994ef8d1984b32746bb45ca"}, - {file = "coverage-6.3.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fbbdc8d55990eac1b0919ca69eb5a988a802b854488c34b8f37f3e2025fa90d"}, - {file = "coverage-6.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ec6bc7fe73a938933d4178c9b23c4e0568e43e220aef9472c4f6044bfc6dd0f0"}, - {file = "coverage-6.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9baff2a45ae1f17c8078452e9e5962e518eab705e50a0aa8083733ea7d45f3a6"}, - {file = "coverage-6.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd9e830e9d8d89b20ab1e5af09b32d33e1a08ef4c4e14411e559556fd788e6b2"}, - {file = "coverage-6.3.2-cp38-cp38-win32.whl", hash = "sha256:f7331dbf301b7289013175087636bbaf5b2405e57259dd2c42fdcc9fcc47325e"}, - {file = "coverage-6.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:68353fe7cdf91f109fc7d474461b46e7f1f14e533e911a2a2cbb8b0fc8613cf1"}, - {file = "coverage-6.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b78e5afb39941572209f71866aa0b206c12f0109835aa0d601e41552f9b3e620"}, - {file = "coverage-6.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4e21876082ed887baed0146fe222f861b5815455ada3b33b890f4105d806128d"}, - {file = "coverage-6.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34626a7eee2a3da12af0507780bb51eb52dca0e1751fd1471d0810539cefb536"}, - {file = "coverage-6.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1ebf730d2381158ecf3dfd4453fbca0613e16eaa547b4170e2450c9707665ce7"}, - {file = "coverage-6.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd6fe30bd519694b356cbfcaca9bd5c1737cddd20778c6a581ae20dc8c04def2"}, - {file = "coverage-6.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:96f8a1cb43ca1422f36492bebe63312d396491a9165ed3b9231e778d43a7fca4"}, - {file = "coverage-6.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:dd035edafefee4d573140a76fdc785dc38829fe5a455c4bb12bac8c20cfc3d69"}, - {file = "coverage-6.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5ca5aeb4344b30d0bec47481536b8ba1181d50dbe783b0e4ad03c95dc1296684"}, - {file = "coverage-6.3.2-cp39-cp39-win32.whl", hash = "sha256:f5fa5803f47e095d7ad8443d28b01d48c0359484fec1b9d8606d0e3282084bc4"}, - {file = "coverage-6.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:9548f10d8be799551eb3a9c74bbf2b4934ddb330e08a73320123c07f95cc2d92"}, - {file = "coverage-6.3.2-pp36.pp37.pp38-none-any.whl", hash = "sha256:18d520c6860515a771708937d2f78f63cc47ab3b80cb78e86573b0a760161faf"}, - {file = "coverage-6.3.2.tar.gz", hash = "sha256:03e2a7826086b91ef345ff18742ee9fc47a6839ccd517061ef8fa1976e652ce9"}, + {file = "coverage-6.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f1d5aa2703e1dab4ae6cf416eb0095304f49d004c39e9db1d86f57924f43006b"}, + {file = "coverage-6.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4ce1b258493cbf8aec43e9b50d89982346b98e9ffdfaae8ae5793bc112fb0068"}, + {file = "coverage-6.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83c4e737f60c6936460c5be330d296dd5b48b3963f48634c53b3f7deb0f34ec4"}, + {file = "coverage-6.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84e65ef149028516c6d64461b95a8dbcfce95cfd5b9eb634320596173332ea84"}, + {file = "coverage-6.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f69718750eaae75efe506406c490d6fc5a6161d047206cc63ce25527e8a3adad"}, + {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e57816f8ffe46b1df8f12e1b348f06d164fd5219beba7d9433ba79608ef011cc"}, + {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:01c5615d13f3dd3aa8543afc069e5319cfa0c7d712f6e04b920431e5c564a749"}, + {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:75ab269400706fab15981fd4bd5080c56bd5cc07c3bccb86aab5e1d5a88dc8f4"}, + {file = "coverage-6.4.1-cp310-cp310-win32.whl", hash = "sha256:a7f3049243783df2e6cc6deafc49ea123522b59f464831476d3d1448e30d72df"}, + {file = "coverage-6.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:ee2ddcac99b2d2aec413e36d7a429ae9ebcadf912946b13ffa88e7d4c9b712d6"}, + {file = "coverage-6.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fb73e0011b8793c053bfa85e53129ba5f0250fdc0392c1591fd35d915ec75c46"}, + {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:106c16dfe494de3193ec55cac9640dd039b66e196e4641fa8ac396181578b982"}, + {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87f4f3df85aa39da00fd3ec4b5abeb7407e82b68c7c5ad181308b0e2526da5d4"}, + {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:961e2fb0680b4f5ad63234e0bf55dfb90d302740ae9c7ed0120677a94a1590cb"}, + {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:cec3a0f75c8f1031825e19cd86ee787e87cf03e4fd2865c79c057092e69e3a3b"}, + {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:129cd05ba6f0d08a766d942a9ed4b29283aff7b2cccf5b7ce279d50796860bb3"}, + {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bf5601c33213d3cb19d17a796f8a14a9eaa5e87629a53979a5981e3e3ae166f6"}, + {file = "coverage-6.4.1-cp37-cp37m-win32.whl", hash = "sha256:269eaa2c20a13a5bf17558d4dc91a8d078c4fa1872f25303dddcbba3a813085e"}, + {file = "coverage-6.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f02cbbf8119db68455b9d763f2f8737bb7db7e43720afa07d8eb1604e5c5ae28"}, + {file = "coverage-6.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ffa9297c3a453fba4717d06df579af42ab9a28022444cae7fa605af4df612d54"}, + {file = "coverage-6.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:145f296d00441ca703a659e8f3eb48ae39fb083baba2d7ce4482fb2723e050d9"}, + {file = "coverage-6.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d44996140af8b84284e5e7d398e589574b376fb4de8ccd28d82ad8e3bea13"}, + {file = "coverage-6.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2bd9a6fc18aab8d2e18f89b7ff91c0f34ff4d5e0ba0b33e989b3cd4194c81fd9"}, + {file = "coverage-6.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3384f2a3652cef289e38100f2d037956194a837221edd520a7ee5b42d00cc605"}, + {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9b3e07152b4563722be523e8cd0b209e0d1a373022cfbde395ebb6575bf6790d"}, + {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1480ff858b4113db2718848d7b2d1b75bc79895a9c22e76a221b9d8d62496428"}, + {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:865d69ae811a392f4d06bde506d531f6a28a00af36f5c8649684a9e5e4a85c83"}, + {file = "coverage-6.4.1-cp38-cp38-win32.whl", hash = "sha256:664a47ce62fe4bef9e2d2c430306e1428ecea207ffd68649e3b942fa8ea83b0b"}, + {file = "coverage-6.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:26dff09fb0d82693ba9e6231248641d60ba606150d02ed45110f9ec26404ed1c"}, + {file = "coverage-6.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9c80df769f5ec05ad21ea34be7458d1dc51ff1fb4b2219e77fe24edf462d6df"}, + {file = "coverage-6.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:39ee53946bf009788108b4dd2894bf1349b4e0ca18c2016ffa7d26ce46b8f10d"}, + {file = "coverage-6.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5b66caa62922531059bc5ac04f836860412f7f88d38a476eda0a6f11d4724f4"}, + {file = "coverage-6.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd180ed867e289964404051a958f7cccabdeed423f91a899829264bb7974d3d3"}, + {file = "coverage-6.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84631e81dd053e8a0d4967cedab6db94345f1c36107c71698f746cb2636c63e3"}, + {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8c08da0bd238f2970230c2a0d28ff0e99961598cb2e810245d7fc5afcf1254e8"}, + {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d42c549a8f41dc103a8004b9f0c433e2086add8a719da00e246e17cbe4056f72"}, + {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:309ce4a522ed5fca432af4ebe0f32b21d6d7ccbb0f5fcc99290e71feba67c264"}, + {file = "coverage-6.4.1-cp39-cp39-win32.whl", hash = "sha256:fdb6f7bd51c2d1714cea40718f6149ad9be6a2ee7d93b19e9f00934c0f2a74d9"}, + {file = "coverage-6.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:342d4aefd1c3e7f620a13f4fe563154d808b69cccef415415aece4c786665397"}, + {file = "coverage-6.4.1-pp36.pp37.pp38-none-any.whl", hash = "sha256:4803e7ccf93230accb928f3a68f00ffa80a88213af98ed338a57ad021ef06815"}, + {file = "coverage-6.4.1.tar.gz", hash = "sha256:4321f075095a096e70aff1d002030ee612b65a205a0a0f5b815280d5dc58100c"}, ] fastapi = [ {file = "fastapi-0.70.1-py3-none-any.whl", hash = "sha256:5367226c7bcd7bfb2e17edaf225fd9a983095b1372281e9a3eb661336fb93748"}, diff --git a/pyproject.toml b/pyproject.toml index 8482284..04370c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ fastapi = "^0.70.0" prometheus-client = "^0.12.0" uvicorn = {extras = ["standard"], version = "^0.15.0"} python-multipart = "^0.0.5" +PyYAML = "^6.0" [tool.poetry.dev-dependencies] pytest = "^7.1.0" diff --git a/tests/api/conftest.py b/tests/api/conftest.py index 2bf3da7..42000cd 100644 --- a/tests/api/conftest.py +++ b/tests/api/conftest.py @@ -1,15 +1,64 @@ +from functools import lru_cache import pytest from fastapi.testclient import TestClient -from gh_actions_exporter.main import app +from gh_actions_exporter.main import app, get_settings, metrics +from gh_actions_exporter.metrics import Metrics +from gh_actions_exporter.config import Relabel, Settings +from prometheus_client import REGISTRY + + +@lru_cache() +def job_relabel_config(): + return Settings(job_relabelling=[ + Relabel( + label="cloud", + values=[ + "mycloud" + ], + type="name", + default="github-hosted" + ), + Relabel( + label="image", + values=[ + "ubuntu-latest" + ], + ) + ]) + + +@lru_cache() +def relabel_metrics(): + return Metrics(job_relabel_config()) + + +def unregister_metrics(): + print(f'Unregistering {REGISTRY._collector_to_names}') + for collector, names in tuple(REGISTRY._collector_to_names.items()): + if any(name.startswith('github_actions') for name in names): + REGISTRY.unregister(collector) + + +@pytest.fixture(scope='function') +def override_job_config(fastapp): + unregister_metrics() + fastapp.dependency_overrides[get_settings] = job_relabel_config + fastapp.dependency_overrides[metrics] = relabel_metrics + + +@pytest.fixture(scope='function', autouse=True) +def fastapp(): + fastapp = app + return fastapp @pytest.fixture(scope='function') -def client(): - client = TestClient(app) +def client(fastapp): + client = TestClient(fastapp) return client -@pytest.fixture(autouse=True) +@pytest.fixture(scope='function', autouse=True) def destroy_client(client): client.delete('/clear') @@ -58,17 +107,18 @@ def workflow_job(): "started_at": "2021-11-29T14:46:57Z", "completed_at": None, "name": "greet (tata)", + "runner_name": "GitHub Actions 3", "steps": [ { "name": "Set up job", - "status": "in_progress", - "conclusion": None, + "status": "completed", + "conclusion": "success", "number": 1, "started_at": "2021-11-29T14:50:57Z", "completed_at": None } ], - "labels": ["github-hosted"] + "labels": ["ubuntu-latest"] }, "repository": { "name": "test-runner-operator", diff --git a/tests/api/test_job.py b/tests/api/test_job.py index e10a0d8..17dd5fa 100644 --- a/tests/api/test_job.py +++ b/tests/api/test_job.py @@ -57,3 +57,32 @@ def test_multiple_job_runs(client, workflow_job, headers): assert '1.0' in line if 'job_start_duration_seconds_sum{' in line: assert '240.0' in line + + +def test_job_relabel(override_job_config, client, workflow_job, headers): + response = client.post('/webhook', json=workflow_job, headers=headers) + metrics = client.get('/metrics') + assert response.status_code == 202 + result = False + # test default + for line in metrics.text.split('\n'): + if "cloud=\"github-hosted\"" in line: + result = True + assert result is True + + result = False + # test name label relabeling + workflow_job['workflow_job']['runner_name'] = "runner-mycloud-1" + response = client.post('/webhook', json=workflow_job, headers=headers) + metrics = client.get('/metrics') + for line in metrics.text.split('\n'): + if "cloud=\"mycloud\"" in line: + result = True + assert result is True + + # test that multiple options are handled + result = False + for line in metrics.text.split('\n'): + if "image=\"ubuntu-latest\"" in line: + result = True + assert result is True