Merge pull request #24 from lilatomic/feature/more-flow

More flow improvements

.github/workflows/ci-testing.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-latest]
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v3

BUILD

@@ -4,7 +4,7 @@ python_distribution(
         "//src/grafanarmadillo:grafanarmadillo",
         "//:docs",
         "//:pyproject",
-        "//:package_info",
+        # "//:package_info",
     ],
     provides=python_artifact(
         name="grafanarmadillo",
@@ -24,4 +24,4 @@ python_requirements(
 )
 
 resource(name="pyproject", source="pyproject.toml")
-resources(name="package_info", sources=["LICENSE", "README.RST"])
+resources(name="package_info", sources=["LICENSE", "README.rst"])

docs/CHANGELOG.rst

@@ -2,6 +2,24 @@
 Changelog
 =========
 
+* task : upgrade pinned deps
+* task : (breaking) drop support for Python 3.8
+* fix : also remove folderUID when creating templates
+* feature : caching in finder
+* feature : `FlowResult.ensure_success` for inline raising flow exceptions
+* feature : `FileStore.resolve_object_to_filepath` has the type of object being referenced, allowing separate folders for alerts and dashboards
+* feature : `Finder.get_alerts_in_folders`
+* refactor : (breaking) set compatibility for API in Finder
+* refactor : use `folderUIDs` for newer search API
+* feature : FileStore accepts json encoders and decoders
+* feature : Flow.extend for adding multiple flows in 1 call
+* fix : searching for multiple folders doesn't fail and return all of them
+* feature : add DashboardTransformer to help fill Grafana templates
+* feature : flow: add URLStore to import dashboards and alerts from remote sources
+* feature : flow: allow all PathLike instead of just str
+* feature : flow: easier to change filename convention
+* fix : fix typo in grafanarmadillo.flow.Store : write_dasbaord -> write_dashboard
+
 v0.7.1 (2024-06-03)
 ------------------------------------------------------------
 

pants.toml

@@ -1,5 +1,5 @@
 [GLOBAL]
-pants_version = "2.22.0.dev3"
+pants_version = "2.22.0"
 
 # needed to make in-repo plugins work
 pythonpath = ["%(buildroot)s/devtools"]
@@ -22,7 +22,7 @@ root_patterns = [
 ]
 
 [python]
-interpreter_constraints = ["CPython>=3.8"]
+interpreter_constraints = ["CPython>=3.9"]
 pip_version = "24.0"
 enable_resolves = true
 
@@ -39,6 +39,7 @@ install_from_resolve = "flake8"
 [test]
 extra_env_vars = [
 	"DOCKER_HOST",
+	"TESTCONTAINERS_RYUK_DISABLED=true",
 ]
 use_coverage=true
 attempts_default = 3

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "grafanarmadillo"
-version = "0.7.1"
+version = "0.8.0"
 description = "Simplify interacting with Grafana, with a focus on templating dashboards and alerts"
 readme = "README.rst"
 

setup.cfg

@@ -9,9 +9,10 @@ ignore =
     W191
     E133
     E741
+    D401
 per-file-ignores =
     setup.py:E501
-    tests/*:D102,D103,D100
+    tests/*:D101,D102,D103,D100
 docstring-convention = numpy
 # normally I exclude init because it is very hard to configure init
 # without breaking many rules

src/grafanarmadillo/alerter.py

@@ -1,19 +1,21 @@
 """Push and pull Grafana alerts."""
-from typing import Optional, Tuple
+from typing import Optional, Tuple, Union
 
 from grafana_client import GrafanaApi
 from grafana_client.client import GrafanaClientError
 
 from grafanarmadillo.types import AlertContent, AlertSearchResult, FolderSearchResult
+from grafanarmadillo.util import Cache, CacheMode
 
 
 class Alerter:
 	"""Collection of methods for managing alert rules."""
 
-	def __init__(self, api: GrafanaApi, disable_provenance=True) -> None:
+	def __init__(self, api: GrafanaApi, disable_provenance=True, cache_mode: Union[CacheMode, Cache] = CacheMode.SESSION) -> None:
 		super().__init__()
 		self.api = api
 		self.disable_provenance = disable_provenance
+		self._cache = CacheMode.select(cache_mode)
 
 	def import_alert(
 		self, content: AlertContent, folder: FolderSearchResult
@@ -39,6 +41,7 @@ def import_alert(
 			self.api.alertingprovisioning.update_alertrule(content["uid"], content, disable_provenance=self.disable_provenance)
 		else:
 			self.api.alertingprovisioning.create_alertrule(content, disable_provenance=self.disable_provenance)
+		self._cache.unset("list_alerts")
 
 	def export_alert(
 		self, alert: AlertSearchResult

src/grafanarmadillo/cmd.py

@@ -13,13 +13,14 @@
 from grafanarmadillo.alerter import Alerter
 from grafanarmadillo.bulk import BulkExporter, BulkImporter
 from grafanarmadillo.dashboarder import Dashboarder
-from grafanarmadillo.find import Finder
+from grafanarmadillo.find import Finder, default_api_v
 from grafanarmadillo.templator import (
 	Templator,
 	alert_dashboarduid_templator,
 	make_mapping_templator,
 	remove_edit_metadata_transformer,
 )
+from grafanarmadillo.types import GrafanaVersion
 from grafanarmadillo.util import load_data
 
 
@@ -97,8 +98,9 @@ def with_template_options(f):
 
 @click.group()
 @click.option("--cfg", "-c", help=f"Config for connecting to Grafana. {load_file_help}")
+@click.option("--api-version", help="Major Grafana API version", default=default_api_v)
 @click.pass_context
-def grafanarmadillo(ctx, cfg):
+def grafanarmadillo(ctx, cfg, api_version):
 	"""Template Grafana things."""
 	ctx.ensure_object(dict)
 	if cfg:
@@ -108,6 +110,7 @@ def grafanarmadillo(ctx, cfg):
 	else:
 		config = {}
 	ctx.obj["cfg"] = config
+	ctx.obj["api_v"] = api_version
 
 
 @grafanarmadillo.group()
@@ -124,12 +127,12 @@ def _export_dashboard(ctx, src, dst, mapping, env_grafana, env_template, templat
 	"""Capture a dashboard from Grafana."""
 	gfn = make_grafana(ctx.obj["cfg"])
 	templator = make_templator(gfn, mapping, env_grafana, env_template, templator_extra_opts)
-	return export_dashboard(gfn, src, dst, templator)
+	return export_dashboard(gfn, src, dst, templator, ctx.obj["api_v"])
 
 
-def export_dashboard(gfn: GrafanaApi, src: str, dst: IO, templator: Templator):
+def export_dashboard(gfn: GrafanaApi, src: str, dst: IO, templator: Templator, api_v: GrafanaVersion = default_api_v):
 	"""Capture a dashboard from Grafana."""
-	finder, dashboarder = Finder(gfn), Dashboarder(gfn)
+	finder, dashboarder = Finder(gfn, api_v), Dashboarder(gfn)
 
 	dashboard_info = finder.get_from_path(src)
 	dashboard_content, _ = dashboarder.export_dashboard(dashboard_info)
@@ -147,12 +150,12 @@ def _import_dashboard(ctx, src, dst, mapping, env_grafana, env_template, templat
 	"""Deploy a template to Grafana."""
 	gfn = make_grafana(ctx.obj["cfg"])
 	templator = make_templator(gfn, mapping, env_grafana, env_template, templator_extra_opts)
-	return import_dashboard(gfn, src, dst, templator)
+	return import_dashboard(gfn, src, dst, templator, ctx.obj["api_v"])
 
 
-def import_dashboard(gfn: GrafanaApi, src: IO, dst: str, templator: Templator):
+def import_dashboard(gfn: GrafanaApi, src: IO, dst: str, templator: Templator, api_v: GrafanaVersion = default_api_v):
 	"""Deploy a template to Grafana."""
-	finder, dashboarder = Finder(gfn), Dashboarder(gfn)
+	finder, dashboarder = Finder(gfn, api_v), Dashboarder(gfn)
 
 	template = load_data(src.read())
 
@@ -175,12 +178,12 @@ def _export_alert(ctx, src, dst, mapping, env_grafana, env_template, templator_e
 	"""Capture an alert from Grafana."""
 	gfn = make_grafana(ctx.obj["cfg"])
 	templator = make_templator(gfn, mapping, env_grafana, env_template, templator_extra_opts)
-	return export_alert(gfn, src, dst, templator)
+	return export_alert(gfn, src, dst, templator, ctx.obj["api_v"])
 
 
-def export_alert(gfn: GrafanaApi, src: str, dst: IO, templator: Templator):
+def export_alert(gfn: GrafanaApi, src: str, dst: IO, templator: Templator, api_v: GrafanaVersion = default_api_v):
 	"""Capture an alert from Grafana."""
-	finder, alerter = Finder(gfn), Alerter(gfn)
+	finder, alerter = Finder(gfn, api_v), Alerter(gfn)
 
 	alert_info = finder.get_alert_from_path(src)
 	alert, _ = alerter.export_alert(alert_info)
@@ -198,12 +201,12 @@ def _import_alert(ctx, src, dst, mapping, env_grafana, env_template, templator_e
 	"""Deploy an alert from a template."""
 	gfn = make_grafana(ctx.obj["cfg"])
 	templator = make_templator(gfn, mapping, env_grafana, env_template, templator_extra_opts)
-	return import_alert(gfn, src, dst, templator)
+	return import_alert(gfn, src, dst, templator, ctx.obj["api_v"])
 
 
-def import_alert(gfn: GrafanaApi, src: IO, dst: str, templator: Templator):
+def import_alert(gfn: GrafanaApi, src: IO, dst: str, templator: Templator, api_v: GrafanaVersion = default_api_v):
 	"""Deploy an alert from a template."""
-	finder, alerter = Finder(gfn), Alerter(gfn)
+	finder, alerter = Finder(gfn, api_v), Alerter(gfn)
 
 	template = load_data(src.read())
 

src/grafanarmadillo/find.py

@@ -1,4 +1,5 @@
 """Find Grafana dashboards and folders."""
+from __future__ import annotations
 
 from typing import List, Optional, Tuple, Union
 
@@ -10,30 +11,41 @@
 	DashboardSearchResult,
 	FolderSearchResult,
 	GrafanaPath,
+	GrafanaVersion,
 	PathLike,
 )
-from grafanarmadillo.util import exactly_one
+from grafanarmadillo.util import Cache, CacheMode, exactly_one
 
 
 def _query_message(query_type: str, query: str) -> str:
 	"""Format a message detailing the query."""
 	return f"type={query_type}, query={query}"
 
 
+default_api_v = GrafanaVersion(11)
+
+
 class Finder:
-	"""Collection of methods for finding Grafana dashboards and folders."""
+	"""
+	Collection of methods for finding Grafana dashboards and folders.
 
-	def __init__(self, api: GrafanaApi) -> None:
+	If not using the latest Grafana version, set the `api_v` parameter to the major version.
+	Some APIs have changed.
+	"""
+
+	def __init__(self, api: GrafanaApi, api_v: GrafanaVersion = default_api_v, cache_mode: Union[CacheMode, Cache] = CacheMode.SESSION) -> None:
 		super().__init__()
 		self.api = api
+		self.api_v = api_v
+		self._cache = CacheMode.select(cache_mode)
 
 	def list_dashboards(self) -> List[DashboardSearchResult]:
 		"""List all dashboards."""
-		return self.api.search.search_dashboards(type_="dash-db")
+		return self._cache.getor("list_dashboards", lambda: self.api.search.search_dashboards(type_="dash-db"))
 
 	def list_alerts(self) -> List[AlertSearchResult]:
 		"""List all alerts."""
-		return self.api.alertingprovisioning.get_alertrules_all()
+		return self._cache.getor("list_alerts", lambda: self.api.alertingprovisioning.get_alertrules_all())
 
 	def find_dashboards(self, name: str) -> List[DashboardSearchResult]:
 		"""Find all dashboards with a name. Returns exact matches only."""
@@ -44,34 +56,61 @@ def find_dashboards(self, name: str) -> List[DashboardSearchResult]:
 			)
 		)
 
-	def _enumerate_dashboards_in_folders(self, folder_ids: List[str]):
-		folder_param = ",".join(folder_ids)
-		return self.api.search.search_dashboards(
-			query=None, type_="dash-db", folder_ids=folder_param
-		)
+	@property
+	def _folder_lookup_param(self) -> str:
+		return "uid" if self.api_v >= 10 else "id"
+
+	def _enumerate_dashboards_in_folders(self, folder_uids: List[str]):
+		folder_uids = tuple(folder_uids)
+
+		def do_enumerate_dashboards():
+			if self.api_v >= 10:
+				folder_kwarg = {"folder_uids": folder_uids}
+			else:
+				folder_kwarg = {"folder_ids": folder_uids}
+			return self.api.search.search_dashboards(
+				query=None, type_="dash-db", **folder_kwarg
+			)
+		return self._cache.getor(("_enumerate_dashboards_in_folders", folder_uids), do_enumerate_dashboards)
 
 	def get_dashboards_in_folders(self, folder_names: List[str]) -> List[DashboardSearchResult]:
 		"""Get all dashboards in folders."""
 		folder_objects = list(
 			map(lambda folder_name: self.get_folder(name=folder_name), folder_names)
 		)
+
 		return self._enumerate_dashboards_in_folders(
-			list(map(lambda f: str(f["id"]), folder_objects))
+			list(map(lambda f: str(f[self._folder_lookup_param]), folder_objects))
 		)
 
+	def get_alerts_in_folders(self, folder_names: List[str]) -> List[AlertSearchResult]:
+		"""Get all alerts in folders."""
+		folder_objects = list(
+			map(lambda folder_name: self.get_folder(name=folder_name), folder_names)
+		)
+		folder_uids = {e["uid"] for e in folder_objects}
+		all_alerts = self.list_alerts()
+		return [e for e in all_alerts if e.get("folderUID") in folder_uids]
+
 	def get_folder(self, name) -> FolderSearchResult:
 		"""Get a folder by name. Folders don't nest, so this will return at most 1 folder."""
-		if name == "General":
-			return self.api.folder.get_folder_by_id(0)
-		else:
-			search_result = self.api.search.search_dashboards(query=name, type_="dash-folder")
-			return exactly_one(
-				list(filter(
-					lambda x: x["title"] == name,
-					map(lambda sr: self.api.folder.get_folder(sr["uid"]), search_result),
-				)),
-				_query_message("folder", name),
-			)
+		def _get_folder() -> FolderSearchResult:
+			if name == "General":
+				v = self.api.folder.get_folder_by_id(0)
+				if self.api_v >= 10:
+					# search API uses this for the folderUIDs parameter
+					v["uid"] = "general"
+				return v
+			else:
+				search_result = self.api.search.search_dashboards(query=name, type_="dash-folder")
+				return exactly_one(
+					list(filter(
+						lambda x: x["title"] == name,
+						map(lambda sr: self.api.folder.get_folder(sr["uid"]), search_result),
+					)),
+					_query_message("folder", name),
+				)
+		return self._cache.getor(("get_folder", name), _get_folder)
 
 	def create_or_get_folder(self, name: str) -> FolderSearchResult:
 		"""
@@ -92,7 +131,7 @@ def get_dashboard(self, folder_name: str, dashboard_name: str) -> DashboardSearc
 		Dashboards without a parent are children of the "General" folder.
 		"""
 		folder_object = self.get_folder(folder_name)
-		dashboards = self._enumerate_dashboards_in_folders([str(folder_object["id"])])
+		dashboards = self._enumerate_dashboards_in_folders([str(folder_object[self._folder_lookup_param])])
 		return exactly_one(
 			list(filter(lambda d: d["title"] == dashboard_name, dashboards)),
 			_query_message("dashboard", f"/{folder_name}/{dashboard_name}"),
@@ -112,7 +151,7 @@ def get_alert(self, folder_name, alert_name) -> AlertSearchResult:
 		return exactly_one(
 			list(filter(
 				lambda a: a["title"] == alert_name and a["folderUID"] == folder_uid,
-				self.api.alertingprovisioning.get_alertrules_all()
+				self.list_alerts()
 			)),
 			_query_message("alert", f"/{folder_name}/{alert_name}")
 		)
@@ -146,6 +185,12 @@ def create_or_get_dashboard(self, path: PathLike) -> Tuple[DashboardSearchResult
 					"folderUid": folder["uid"],
 				}
 			)
+			# we reset all enumerate search results rather than finding only those that apply
+			# since a search might have `(otherfolder, ourfolder)` as a key.
+			# sorting through that sounds difficult to get right
+			self._cache.unset_method("_enumerate_dashboards_in_folders")
+			self._cache.unset("list_dashboards")
+
 			dashboard = self.get_dashboard(address.folder, address.name)
 
 		return dashboard, folder
@@ -167,6 +212,8 @@ def create_or_get_alert(self, path: PathLike) -> Tuple[AlertSearchResult, Folder
 				self._mk_null_alert(folder["uid"], address.name),
 				disable_provenance=True
 			)
+			self._cache.unset("list_alerts")
+
 			alert = self.get_alert(address.folder, address.name)
 
 		return alert, folder