diff --git a/lib/charms/data_platform_libs/v0/data_secrets.py b/lib/charms/data_platform_libs/v0/data_secrets.py new file mode 100644 index 000000000..254b9af3d --- /dev/null +++ b/lib/charms/data_platform_libs/v0/data_secrets.py @@ -0,0 +1,143 @@ +"""Secrets related helper classes/functions.""" +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +from typing import Dict, Literal, Optional + +from ops import Secret, SecretInfo +from ops.charm import CharmBase +from ops.model import SecretNotFoundError + +# The unique Charmhub library identifier, never change it +LIBID = "d77fb3d01aba41ed88e837d0beab6be5" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 1 + + +APP_SCOPE = "app" +UNIT_SCOPE = "unit" +Scopes = Literal["app", "unit"] + + +class DataSecretsError(Exception): + """A secret that we want to create already exists.""" + + +class SecretAlreadyExistsError(DataSecretsError): + """A secret that we want to create already exists.""" + + +def generate_secret_label(charm: CharmBase, scope: Scopes) -> str: + """Generate unique group_mappings for secrets within a relation context. + + Defined as a standalone function, as the choice on secret labels definition belongs to the + Application Logic. To be kept separate from classes below, which are simply to provide a + (smart) abstraction layer above Juju Secrets. + """ + members = [charm.app.name, scope] + return f"{'.'.join(members)}" + + +# Secret cache + + +class CachedSecret: + """Abstraction layer above direct Juju access with caching. + + The data structure is precisely re-using/simulating Juju Secrets behavior, while + also making sure not to fetch a secret multiple times within the same event scope. + """ + + def __init__(self, charm: CharmBase, label: str, secret_uri: Optional[str] = None): + self._secret_meta = None + self._secret_content = {} + self._secret_uri = secret_uri + self.label = label + self.charm = charm + + def add_secret(self, content: Dict[str, str], scope: Scopes) -> Secret: + """Create a new secret.""" + if self._secret_uri: + raise SecretAlreadyExistsError( + "Secret is already defined with uri %s", self._secret_uri + ) + + if scope == APP_SCOPE: + secret = self.charm.app.add_secret(content, label=self.label) + else: + secret = self.charm.unit.add_secret(content, label=self.label) + self._secret_uri = secret.id + self._secret_meta = secret + return self._secret_meta + + @property + def meta(self) -> Optional[Secret]: + """Getting cached secret meta-information.""" + if self._secret_meta: + return self._secret_meta + + if not (self._secret_uri or self.label): + return + + try: + self._secret_meta = self.charm.model.get_secret(label=self.label) + except SecretNotFoundError: + if self._secret_uri: + self._secret_meta = self.charm.model.get_secret( + id=self._secret_uri, label=self.label + ) + return self._secret_meta + + def get_content(self) -> Dict[str, str]: + """Getting cached secret content.""" + if not self._secret_content: + if self.meta: + self._secret_content = self.meta.get_content() + return self._secret_content + + def set_content(self, content: Dict[str, str]) -> None: + """Setting cached secret content.""" + if self.meta: + self.meta.set_content(content) + self._secret_content = content + + def get_info(self) -> Optional[SecretInfo]: + """Wrapper function for get the corresponding call on the Secret object if any.""" + if self.meta: + return self.meta.get_info() + + +class SecretCache: + """A data structure storing CachedSecret objects.""" + + def __init__(self, charm): + self.charm = charm + self._secrets: Dict[str, CachedSecret] = {} + + def get(self, label: str, uri: Optional[str] = None) -> Optional[CachedSecret]: + """Getting a secret from Juju Secret store or cache.""" + if not self._secrets.get(label): + secret = CachedSecret(self.charm, label, uri) + + # Checking if the secret exists, otherwise we don't register it in the cache + if secret.meta: + self._secrets[label] = secret + return self._secrets.get(label) + + def add(self, label: str, content: Dict[str, str], scope: Scopes) -> CachedSecret: + """Adding a secret to Juju Secret.""" + if self._secrets.get(label): + raise SecretAlreadyExistsError(f"Secret {label} already exists") + + secret = CachedSecret(self.charm, label) + secret.add_secret(content, scope) + self._secrets[label] = secret + return self._secrets[label] + + +# END: Secret cache diff --git a/lib/charms/grafana_agent/v0/cos_agent.py b/lib/charms/grafana_agent/v0/cos_agent.py index d3130b2b5..259a90170 100644 --- a/lib/charms/grafana_agent/v0/cos_agent.py +++ b/lib/charms/grafana_agent/v0/cos_agent.py @@ -206,17 +206,15 @@ def __init__(self, *args): ``` """ -import base64 import json import logging -import lzma from collections import namedtuple from itertools import chain from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, List, Optional, Set, Union import pydantic -from cosl import JujuTopology +from cosl import GrafanaDashboard, JujuTopology from cosl.rules import AlertRules from ops.charm import RelationChangedEvent from ops.framework import EventBase, EventSource, Object, ObjectEvents @@ -236,7 +234,7 @@ class _MetricsEndpointDict(TypedDict): LIBID = "dc15fa84cef84ce58155fb84f6c6213a" LIBAPI = 0 -LIBPATCH = 6 +LIBPATCH = 7 PYDEPS = ["cosl", "pydantic < 2"] @@ -251,31 +249,6 @@ class _MetricsEndpointDict(TypedDict): SnapEndpoint = namedtuple("SnapEndpoint", "owner, name") -class GrafanaDashboard(str): - """Grafana Dashboard encoded json; lzma-compressed.""" - - # TODO Replace this with a custom type when pydantic v2 released (end of 2023 Q1?) - # https://github.com/pydantic/pydantic/issues/4887 - @staticmethod - def _serialize(raw_json: Union[str, bytes]) -> "GrafanaDashboard": - if not isinstance(raw_json, bytes): - raw_json = raw_json.encode("utf-8") - encoded = base64.b64encode(lzma.compress(raw_json)).decode("utf-8") - return GrafanaDashboard(encoded) - - def _deserialize(self) -> Dict: - try: - raw = lzma.decompress(base64.b64decode(self.encode("utf-8"))).decode() - return json.loads(raw) - except json.decoder.JSONDecodeError as e: - logger.error("Invalid Dashboard format: %s", e) - return {} - - def __repr__(self): - """Return string representation of self.""" - return "" - - class CosAgentProviderUnitData(pydantic.BaseModel): """Unit databag model for `cos-agent` relation.""" @@ -748,6 +721,10 @@ def metrics_jobs(self) -> List[Dict]: "job_name": job["job_name"], "metrics_path": job["path"], "static_configs": [{"targets": [f"localhost:{job['port']}"]}], + # We include insecure_skip_verify because we are always scraping localhost. + # Even if we have the certs for the scrape targets, we'd rather specify the scrape + # jobs with localhost rather than the SAN DNS the cert was issued for. + "tls_config": {"insecure_skip_verify": True}, } scrape_jobs.append(job) diff --git a/lib/charms/mysql/v0/mysql.py b/lib/charms/mysql/v0/mysql.py index 984c85c79..559397ac0 100644 --- a/lib/charms/mysql/v0/mysql.py +++ b/lib/charms/mysql/v0/mysql.py @@ -78,6 +78,13 @@ def wait_until_mysql_connection(self) -> None: from typing import Any, Dict, Iterable, List, Optional, Tuple, Union import ops +from charms.data_platform_libs.v0.data_secrets import ( + APP_SCOPE, + UNIT_SCOPE, + Scopes, + SecretCache, + generate_secret_label, +) from ops.charm import ActionEvent, CharmBase, RelationBrokenEvent from ops.model import Unit from tenacity import ( @@ -116,7 +123,7 @@ def wait_until_mysql_connection(self) -> None: # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 52 +LIBPATCH = 53 UNIT_TEARDOWN_LOCKNAME = "unit-teardown" UNIT_ADD_LOCKNAME = "unit-add" @@ -377,7 +384,7 @@ class MySQLCharmBase(CharmBase, ABC): def __init__(self, *args): super().__init__(*args) - self.app_secrets, self.unit_secrets = None, None + self.secrets = SecretCache(self) self.framework.observe(self.on.get_cluster_status_action, self._get_cluster_status) self.framework.observe(self.on.get_password_action, self._on_get_password) @@ -535,36 +542,44 @@ def has_cos_relation(self) -> bool: return len(active_cos_relations) > 0 - def _get_secret_from_juju(self, scope: str, key: str) -> Optional[str]: - """Retrieve and return the secret from the juju secret storage.""" - if scope == "unit": - secret_id = self.unit_peer_data.get(SECRET_ID_KEY) + def _scope_obj(self, scope: Scopes): + if scope == APP_SCOPE: + return self.app + if scope == UNIT_SCOPE: + return self.unit - if not self.unit_secrets and not secret_id: - logger.debug("Getting a secret when no secrets added in juju") - return None + def _peer_data(self, scope: Scopes) -> Dict: + """Return corresponding databag for app/unit.""" + if self.peers is None: + return {} + return self.peers.data[self._scope_obj(scope)] - if not self.unit_secrets: - secret = self.model.get_secret(id=secret_id) - content = secret.get_content() - self.unit_secrets = content - logger.debug(f"Retrieved secret {key} for unit from juju") + def _safe_get_secret(self, scope: Scopes, label: str) -> SecretCache: + """Safety measure, for upgrades between versions. - return self.unit_secrets.get(key) + Based on secret URI usage to others with labels usage. + If the secret can't be retrieved by label, we search for the uri -- and + if found, we "stick" the label on the secret for further usage. + """ + secret_uri = self._peer_data(scope).get(SECRET_ID_KEY, None) + secret = self.secrets.get(label, secret_uri) - secret_id = self.app_peer_data.get(SECRET_ID_KEY) + # Since now we switched to labels, the databag reference can be removed + if secret_uri and secret and scope == APP_SCOPE and self.unit.is_leader(): + self._peer_data(scope).pop(SECRET_ID_KEY, None) + return secret - if not self.app_secrets and not secret_id: - logger.debug("Getting a secret when no secrets added in juju") - return None + def _get_secret_from_juju(self, scope: Scopes, key: str) -> Optional[str]: + """Retrieve and return the secret from the juju secret storage.""" + label = generate_secret_label(self, scope) + secret = self._safe_get_secret(scope, label) - if not self.app_secrets: - secret = self.model.get_secret(id=secret_id) - content = secret.get_content() - self.app_secrets = content - logger.debug(f"Retrieved secret {key} for app from juju") + if not secret: + logger.debug("Getting a secret when secret is not added in juju") + return - return self.app_secrets.get(key) + value = secret.get_content().get(key) + return value def _get_secret_from_databag(self, scope: str, key: str) -> Optional[str]: """Retrieve and return the secret from the peer relation databag.""" @@ -574,7 +589,7 @@ def _get_secret_from_databag(self, scope: str, key: str) -> Optional[str]: return self.app_peer_data.get(key) def get_secret( - self, scope: str, key: str, fallback_key: Optional[str] = None + self, scope: Scopes, key: str, fallback_key: Optional[str] = None ) -> Optional[str]: """Get secret from the secret storage. @@ -583,7 +598,7 @@ def get_secret( account for cases where secrets are stored in peer databag but the charm is then refreshed to a newer revision. """ - if scope not in ["unit", "app"]: + if scope not in ["app", "unit"]: raise MySQLSecretError(f"Invalid secret scope: {scope}") if ops.jujuversion.JujuVersion.from_environ().has_secrets: @@ -595,68 +610,53 @@ def get_secret( scope, fallback_key ) - def _set_secret_in_databag(self, scope: str, key: str, value: Optional[str]) -> None: + def _set_secret_in_databag(self, scope: Scopes, key: str, value: Optional[str]) -> None: """Set secret in the peer relation databag.""" if not value: - if scope == "unit": - del self.unit_peer_data[key] - else: - del self.app_peer_data[key] - return - - if scope == "unit": - self.unit_peer_data[key] = value - return + try: + self._peer_data(scope).pop(key) + return + except KeyError: + logger.error(f"Non-existing secret {scope}:{key} was attempted to be removed.") + return - self.app_peer_data[key] = value + self._peer_data(scope)[key] = value - def _set_secret_in_juju(self, scope: str, key: str, value: Optional[str]) -> None: + def _set_secret_in_juju(self, scope: Scopes, key: str, value: Optional[str]) -> None: """Set the secret in the juju secret storage.""" - if scope == "unit": - secret_id = self.unit_peer_data.get(SECRET_ID_KEY) - else: - secret_id = self.app_peer_data.get(SECRET_ID_KEY) - - if secret_id: - secret = self.model.get_secret(id=secret_id) + # Charm could have been upgraded since last run + # We make an attempt to remove potential traces from the databag + self._peer_data(scope).pop(key, None) + + label = generate_secret_label(self, scope) + secret = self._safe_get_secret(scope, label) + if not secret and value: + self.secrets.add(label, {key: value}, scope) + return - if scope == "unit": - content = self.unit_secrets or secret.get_content() - else: - content = self.app_secrets or secret.get_content() + content = secret.get_content() if secret else None - if not value: - del content[key] + if not value: + if content and key in content: + content.pop(key, None) else: - content[key] = value - - secret.set_content(content) - logger.debug(f"Updated {scope} secret {secret_id} for {key}") - elif not value: - return + logger.error(f"Non-existing secret {scope}:{key} was attempted to be removed.") + return else: - content = { - key: value, - } + content.update({key: value}) - if scope == "unit": - secret = self.unit.add_secret(content) - self.unit_peer_data[SECRET_ID_KEY] = secret.id - else: - secret = self.app.add_secret(content) - self.app_peer_data[SECRET_ID_KEY] = secret.id - logger.debug(f"Added {scope} secret {secret.id} for {key}") - - if scope == "unit": - self.unit_secrets = content + # Temporary solution: this should come from the shared lib + # Improved after https://warthogs.atlassian.net/browse/DPE-3056 is resolved + if content: + secret.set_content(content) else: - self.app_secrets = content + secret.meta.remove_all_revisions() def set_secret( - self, scope: str, key: str, value: Optional[str], fallback_key: Optional[str] = None + self, scope: Scopes, key: str, value: Optional[str], fallback_key: Optional[str] = None ) -> None: """Set a secret in the secret storage.""" - if scope not in ["unit", "app"]: + if scope not in ["app", "unit"]: raise MySQLSecretError(f"Invalid secret scope: {scope}") if scope == "app" and not self.unit.is_leader(): diff --git a/poetry.lock b/poetry.lock index 60bab0368..d01a19757 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "appnope" @@ -400,13 +400,13 @@ files = [ [[package]] name = "cosl" -version = "0.0.5" +version = "0.0.7" description = "Utils for COS Lite charms" optional = false python-versions = ">=3.8" files = [ - {file = "cosl-0.0.5-py3-none-any.whl", hash = "sha256:84666fde29b792299827d65a1b9b2e3c56029c769e892c8244b50ce793458894"}, - {file = "cosl-0.0.5.tar.gz", hash = "sha256:31c131d1f04c061d3fbef49a4e0a175d4cb481deeb06d0cb3c7b242e4c5416be"}, + {file = "cosl-0.0.7-py3-none-any.whl", hash = "sha256:ed7cf980b47f4faa0e65066d65e5b4274f1972fb6cd3533441a90edae360b4a7"}, + {file = "cosl-0.0.7.tar.gz", hash = "sha256:edf07a81d152720c3ee909a1201063e5b1a35c49f574a7ec1deb989a8bc6fada"}, ] [package.dependencies] @@ -943,6 +943,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1095,6 +1105,20 @@ files = [ {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] +[[package]] +name = "parameterized" +version = "0.9.0" +description = "Parameterized testing with any Python test framework" +optional = false +python-versions = ">=3.7" +files = [ + {file = "parameterized-0.9.0-py2.py3-none-any.whl", hash = "sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b"}, + {file = "parameterized-0.9.0.tar.gz", hash = "sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1"}, +] + +[package.extras] +dev = ["jinja2"] + [[package]] name = "paramiko" version = "2.12.0" @@ -1701,6 +1725,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1708,8 +1733,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1726,6 +1758,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1733,6 +1766,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2201,4 +2235,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "9fc0beeceef5a91a10688697520402b7d1194b1604c4b614a4fbc2e6dfbd3ce5" +content-hash = "16b7b8c0313e7413d11b43d2fc405e30d33bccaf9b1f25e6d41050dbc6e01a1b" diff --git a/pyproject.toml b/pyproject.toml index ec8fd00a1..5b493e16a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ poetry-core = "*" # grafana_agent/v0/cos_agent.py requires pydantic <2 pydantic = "^1.10, <2" # grafana_agent/v0/cos_agent.py -cosl = "*" +cosl = ">=0.0.7" # tls_certificates_interface/v1/tls_certificates.py cryptography = "*" jsonschema = "*" @@ -58,6 +58,7 @@ shellcheck-py = "^0.9.0.5" pytest = "^7.4.0" pytest-mock = "^3.11.1" coverage = {extras = ["toml"], version = "^7.2.7"} +parameterized = "^0.9.0" [tool.poetry.group.integration.dependencies] pytest = "^7.4.0" diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 4d8b872cf..8730e041e 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -1,6 +1,7 @@ # Copyright 2022 Canonical Ltd. # See LICENSE file for licensing details. +import logging import unittest from unittest.mock import patch @@ -13,6 +14,7 @@ ) from ops.model import ActiveStatus, BlockedStatus, MaintenanceStatus, WaitingStatus from ops.testing import Harness +from parameterized import parameterized from tenacity import Retrying, stop_after_attempt from charm import MySQLOperatorCharm @@ -40,6 +42,10 @@ def setUp(self): self.harness.add_relation_unit(self.db_router_relation_id, "app/0") self.harness.add_relation("restart", "restart") + @pytest.fixture + def use_caplog(self, caplog): + self._caplog = caplog + @patch_network_get(private_address="1.1.1.1") @patch("upgrade.MySQLVMUpgrade.cluster_state", return_value="idle") @patch("socket.getfqdn", return_value="test-hostname") @@ -109,9 +115,7 @@ def test_on_leader_elected_sets_mysql_passwords_secret(self): self.harness.set_leader(True) # ensure passwords set in the peer relation databag - secret_id = self.harness.get_relation_data(self.peer_relation_id, self.harness.charm.app)[ - "secret-id" - ] + secret_data = self.harness.model.get_secret(label="mysql.app").get_content() expected_peer_relation_databag_keys = [ "root-password", @@ -121,7 +125,6 @@ def test_on_leader_elected_sets_mysql_passwords_secret(self): "backups-password", ] - secret_data = self.harness.model.get_secret(id=secret_id).get_content() self.assertEqual(sorted(secret_data.keys()), sorted(expected_peer_relation_databag_keys)) @patch_network_get(private_address="1.1.1.1") @@ -345,10 +348,7 @@ def test_set_secret(self, _): self.peer_relation_id, self.charm.app.name ) self.charm.set_secret("app", "password", "test-password") - secret_id = self.harness.get_relation_data(self.peer_relation_id, self.charm.app.name)[ - "secret-id" - ] - secret_data = self.harness.model.get_secret(id=secret_id).get_content() + secret_data = self.harness.model.get_secret(label="mysql.app").get_content() assert secret_data["password"] == "test-password" assert "password" not in self.harness.get_relation_data( @@ -360,10 +360,7 @@ def test_set_secret(self, _): self.peer_relation_id, self.charm.unit.name ) self.charm.set_secret("unit", "password", "test-password") - secret_id = self.harness.get_relation_data(self.peer_relation_id, self.charm.unit.name)[ - "secret-id" - ] - secret_data = self.harness.model.get_secret(id=secret_id).get_content() + secret_data = self.harness.model.get_secret(label="mysql.app").get_content() assert secret_data["password"] == "test-password" assert "password" not in self.harness.get_relation_data( @@ -450,3 +447,136 @@ def test_on_update( _get_cluster_primary_address.assert_called_once() self.assertTrue(isinstance(self.harness.model.unit.status, BlockedStatus)) + + @parameterized.expand([("app"), ("unit")]) + @pytest.mark.usefixtures("with_juju_secrets") + def test_set_reset_new_secret(self, scope): + """NOTE: currently ops.testing seems to allow for non-leader to set secrets too!""" + # Getting current password + self.harness.set_leader() + self.harness.charm.set_secret(scope, "new-secret", "bla") + assert self.harness.charm.get_secret(scope, "new-secret") == "bla" + + # Reset new secret + self.harness.charm.set_secret(scope, "new-secret", "blablabla") + assert self.harness.charm.get_secret(scope, "new-secret") == "blablabla" + + # Set another new secret + self.harness.charm.set_secret(scope, "new-secret2", "blablabla") + assert self.harness.charm.get_secret(scope, "new-secret2") == "blablabla" + + @parameterized.expand([("app"), ("unit")]) + @pytest.mark.usefixtures("with_juju_secrets") + def test_invalid_secret(self, scope): + with self.assertRaises(TypeError): + self.harness.charm.set_secret("unit", "somekey", 1) + + self.harness.charm.set_secret("unit", "somekey", "") + assert self.harness.charm.get_secret(scope, "somekey") is None + + @pytest.mark.usefixtures("with_juju_secrets") + def test_migartion(self): + """Check if we're moving on to use secrets when live upgrade to Secrets usage.""" + # Getting current password + self.harness.set_leader() + entity = getattr(self.charm, "app") + self.harness.update_relation_data(self.peer_relation_id, entity.name, {"my-secret": "bla"}) + assert self.harness.charm.get_secret("app", "my-secret") == "bla" + + # Reset new secret + self.harness.charm.set_secret("app", "my-secret", "blablabla") + assert self.harness.charm.model.get_secret(label="mysql.app") + assert self.harness.charm.get_secret("app", "my-secret") == "blablabla" + + @pytest.mark.usefixtures("with_juju_secrets") + def test_migartion_unit(self): + """Check if we're moving on to use secrets when live upgrade to Secrets usage.""" + # Getting current password + entity = getattr(self.charm, "unit") + self.harness.update_relation_data(self.peer_relation_id, entity.name, {"my-secret": "bla"}) + assert self.harness.charm.get_secret("unit", "my-secret") == "bla" + + # Reset new secret + self.harness.charm.set_secret("unit", "my-secret", "blablabla") + assert self.harness.charm.model.get_secret(label="mysql.unit") + assert self.harness.charm.get_secret("unit", "my-secret") == "blablabla" + + @pytest.mark.usefixtures("without_juju_secrets") + @pytest.mark.usefixtures("use_caplog") + def test_delete_password(self): + """NOTE: currently ops.testing seems to allow for non-leader to remove secrets too!""" + self.harness.set_leader() + self.harness.update_relation_data( + self.peer_relation_id, self.charm.app.name, {"replication": "somepw"} + ) + self.harness.charm.set_secret("app", "replication", "") + assert self.harness.charm.get_secret("app", "replication") is None + + self.harness.update_relation_data( + self.peer_relation_id, self.charm.unit.name, {"somekey": "somevalue"} + ) + self.harness.charm.set_secret("unit", "somekey", "") + assert self.harness.charm.get_secret("unit", "somekey") is None + + with self._caplog.at_level(logging.ERROR): + self.harness.charm.set_secret("app", "replication", "") + assert ( + "Non-existing secret app:replication was attempted to be removed." + in self._caplog.text + ) + + self.harness.charm.set_secret("unit", "somekey", "") + assert ( + "Non-existing secret unit:somekey was attempted to be removed." + in self._caplog.text + ) + + self.harness.charm.set_secret("app", "non-existing-secret", "") + assert ( + "Non-existing secret app:non-existing-secret was attempted to be removed." + in self._caplog.text + ) + + self.harness.charm.set_secret("unit", "non-existing-secret", "") + assert ( + "Non-existing secret unit:non-existing-secret was attempted to be removed." + in self._caplog.text + ) + + @pytest.mark.usefixtures("with_juju_secrets") + @pytest.mark.usefixtures("use_caplog") + def test_delete_existing_password_secrets(self): + """NOTE: currently ops.testing seems to allow for non-leader to remove secrets too!""" + self.harness.set_leader() + self.harness.charm.set_secret("app", "replication", "somepw") + self.harness.charm.set_secret("app", "replication", "") + assert self.harness.charm.get_secret("app", "replication") is None + + self.harness.charm.set_secret("unit", "somekey", "somesecret") + self.harness.charm.set_secret("unit", "somekey", "") + assert self.harness.charm.get_secret("unit", "somekey") is None + + with self._caplog.at_level(logging.ERROR): + self.harness.charm.set_secret("app", "replication", "") + assert ( + "Non-existing secret app:replication was attempted to be removed." + in self._caplog.text + ) + + self.harness.charm.set_secret("unit", "somekey", "") + assert ( + "Non-existing secret unit:somekey was attempted to be removed." + in self._caplog.text + ) + + self.harness.charm.set_secret("app", "non-existing-secret", "") + assert ( + "Non-existing secret app:non-existing-secret was attempted to be removed." + in self._caplog.text + ) + + self.harness.charm.set_secret("unit", "non-existing-secret", "") + assert ( + "Non-existing secret unit:non-existing-secret was attempted to be removed." + in self._caplog.text + ) diff --git a/tests/unit/test_relation_mysql_legacy.py b/tests/unit/test_relation_mysql_legacy.py index d03a0d0a6..9aea4971a 100644 --- a/tests/unit/test_relation_mysql_legacy.py +++ b/tests/unit/test_relation_mysql_legacy.py @@ -118,11 +118,7 @@ def test_maria_db_relation_created_with_secrets( _does_mysql_user_exist.assert_called_once_with("mysql", "%") maria_db_relation = self.charm.model.get_relation(LEGACY_MYSQL) - peer_relation = self.charm.model.get_relation(PEER) - secret_id = self.harness.get_relation_data(peer_relation.id, self.harness.charm.app.name)[ - "secret-id" - ] - root_pw = self.harness.model.get_secret(id=secret_id).get_content()["root-password"] + root_pw = self.harness.model.get_secret(label="mysql.app").get_content()["root-password"] # confirm that the relation databag is populated self.assertEqual(