Update data_interfaces lib to include opensearch provider

.github/workflows/ci.yaml

@@ -88,9 +88,7 @@ jobs:
       - name: Run integration tests
         run: |
           # set sysctl values in case the cloudinit-userdata not applied
-          sudo sysctl -w vm.max_map_count=262144
-          sudo sysctl -w vm.swappiness=0
-          sudo sysctl -w net.ipv4.tcp_retries2=5
+          sudo sysctl -w vm.max_map_count=262144 vm.swappiness=0 net.ipv4.tcp_retries2=5
 
           tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}'
         env:

.gitignore

@@ -9,5 +9,6 @@ __pycache__/
 .vscode
 
 *.tar.gz
+*.tar.xz
 cloudinit-userdata.yaml
 /.pytest_cache/

CONTRIBUTING.md

@@ -63,14 +63,12 @@ cloudinit-userdata: |
     - [ 'sysctl', '-w', 'fs.file-max=1048576' ]
 EOF
 ```
+
 or in a single machine:
 ```
-sudo sysctl -w vm.max_map_count=262144
-sudo sysctl -w vm.swappiness=0
-sudo sysctl -w net.ipv4.tcp_retries2=5
+sudo sysctl -w vm.max_map_count=262144 vm.swappiness=0 net.ipv4.tcp_retries2=5
 ```
 
-
 Then create a new model and set the previously generated file in it.
 ```bash
 # Create a model
@@ -101,7 +99,7 @@ juju deploy -n 1 ./opensearch_ubuntu-22.04-amd64.charm --series jammy --show-log
 juju relate tls-certificates-operator opensearch
 ```
 
-**Note:** The TLS settings shown here are for self-signed-certificates, which are not recommended for production clusters. The TLS Certificates Operator offers a variety of configurations. Read more on the TLS Certificates Operator [here](https://charmhub.io/tls-certificates-operator). 
+**Note:** The TLS settings shown here are for self-signed-certificates, which are not recommended for production clusters. The TLS Certificates Operator offers a variety of configurations. Read more on the TLS Certificates Operator [here](https://charmhub.io/tls-certificates-operator).
 
 
 ## Canonical Contributor Agreement

README.md

@@ -18,7 +18,7 @@ Bootstrap a [lxd controller](https://juju.is/docs/olm/lxd#heading--create-a-cont
 juju add-model opensearch
 ```
 
-Configure the system settings required by [OpenSearch](https://opensearch.org/docs/2.3/opensearch/install/important-settings/), 
+Configure the system settings required by [OpenSearch](https://opensearch.org/docs/2.3/opensearch/install/important-settings/),
 we'll do that by creating and setting a [`cloudinit-userdata.yaml` file](https://juju.is/docs/olm/juju-model-config) on the model.
 ```
 cat <<EOF > cloudinit-userdata.yaml
@@ -34,9 +34,7 @@ juju model-config ./cloudinit-userdata.yaml
 ```
 or in a single machine:
 ```
-sudo sysctl -w vm.max_map_count=262144
-sudo sysctl -w vm.swappiness=0
-sudo sysctl -w net.ipv4.tcp_retries2=5
+sudo sysctl -w vm.max_map_count=262144 vm.swappiness=0 net.ipv4.tcp_retries2=5
 ```
 
 ### Basic Usage
@@ -55,10 +53,10 @@ Supported [relations](https://juju.is/docs/olm/relations):
 The Charmed OpenSearch Operator also supports TLS encryption on the HTTP and Transport layers. TLS is enabled by default:
 
 ```shell
-# Deploy the TLS Certificates Operator. 
+# Deploy the TLS Certificates Operator.
 juju deploy tls-certificates-operator --channel=edge
 # Add the necessary configurations for TLS.
-juju config tls-certificates-operator generate-self-signed-certificates="true" ca-common-name="Test CA" 
+juju config tls-certificates-operator generate-self-signed-certificates="true" ca-common-name="Test CA"
 # Enable TLS via relation.
 juju relate opensearch tls-certificates-operator
 # Disable TLS by removing relation.

lib/charms/data_platform_libs/v0/data_interfaces.py

@@ -304,7 +304,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent):
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 4
+LIBPATCH = 6
 
 logger = logging.getLogger(__name__)
 
@@ -659,7 +659,7 @@ def replset(self) -> Optional[str]:
     def uris(self) -> Optional[str]:
         """Returns the connection URIs.
 
-        MongoDB, Redis, OpenSearch.
+        MongoDB, Redis.
         """
         return self.relation.data[self.relation.app].get("uris")
 
@@ -969,7 +969,7 @@ class KafkaRequiresEvent(RelationEvent):
 
     @property
     def bootstrap_server(self) -> Optional[str]:
-        """Returns a a comma-seperated list of broker uris."""
+        """Returns a a comma-separated list of broker uris."""
         return self.relation.data[self.relation.app].get("endpoints")
 
     @property
@@ -1049,7 +1049,7 @@ def set_zookeeper_uris(self, relation_id: int, zookeeper_uris: str) -> None:
 
         Args:
             relation_id: the identifier for a particular relation.
-            zookeeper_uris: comma-seperated list of ZooKeeper server uris.
+            zookeeper_uris: comma-separated list of ZooKeeper server uris.
         """
         self._update_relation_data(relation_id, {"zookeeper-uris": zookeeper_uris})
 
@@ -1096,7 +1096,7 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None:
             # “endpoints_changed“ event if “topic_created“ is triggered.
             return
 
-        # Emit an endpoints (bootstap-server) changed event if the Kakfa endpoints
+        # Emit an endpoints (bootstrap-server) changed event if the Kakfa endpoints
         # added or changed this info in the relation databag.
         if "endpoints" in diff.added or "endpoints" in diff.changed:
             # Emit the default event (the one without an alias).
@@ -1105,3 +1105,164 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None:
                 event.relation, app=event.app, unit=event.unit
             )  # here check if this is the right design
             return
+
+
+# Opensearch related events
+
+
+class OpenSearchProvidesEvent(RelationEvent):
+    """Base class for OpenSearch events."""
+
+    @property
+    def index(self) -> Optional[str]:
+        """Returns the index that was requested."""
+        return self.relation.data[self.relation.app].get("index")
+
+
+class IndexRequestedEvent(OpenSearchProvidesEvent, ExtraRoleEvent):
+    """Event emitted when a new index is requested for use on this relation."""
+
+
+class OpenSearchProvidesEvents(CharmEvents):
+    """OpenSearch events.
+
+    This class defines the events that OpenSearch can emit.
+    """
+
+    index_requested = EventSource(IndexRequestedEvent)
+
+
+class OpenSearchRequiresEvent(DatabaseRequiresEvent):
+    """Base class for OpenSearch requirer events."""
+
+
+class IndexCreatedEvent(AuthenticationEvent, OpenSearchRequiresEvent):
+    """Event emitted when a new index is created for use on this relation."""
+
+
+class OpenSearchRequiresEvents(CharmEvents):
+    """OpenSearch events.
+
+    This class defines the events that the opensearch requirer can emit.
+    """
+
+    index_created = EventSource(IndexCreatedEvent)
+    endpoints_changed = EventSource(DatabaseEndpointsChangedEvent)
+    authentication_updated = EventSource(AuthenticationEvent)
+
+
+# OpenSearch Provides and Requires Objects
+
+
+class OpenSearchProvides(DataProvides):
+    """Provider-side of the OpenSearch relation."""
+
+    on = OpenSearchProvidesEvents()
+
+    def __init__(self, charm: CharmBase, relation_name: str) -> None:
+        super().__init__(charm, relation_name)
+
+    def _on_relation_changed(self, event: RelationChangedEvent) -> None:
+        """Event emitted when the relation has changed."""
+        # Only the leader should handle this event.
+        if not self.local_unit.is_leader():
+            return
+
+        # Check which data has changed to emit customs events.
+        diff = self._diff(event)
+
+        # Emit an index requested event if the setup key (index name and optional extra user roles)
+        # have been added to the relation databag by the application.
+        if "index" in diff.added:
+            self.on.index_requested.emit(event.relation, app=event.app, unit=event.unit)
+
+    def set_index(self, relation_id: int, index: str) -> None:
+        """Set the index in the application relation databag.
+
+        Args:
+            relation_id: the identifier for a particular relation.
+            index: the index as it is _created_ on the provider charm. This needn't match the
+                requested index, and can be used to present a different index name if, for example,
+                the requested index is invalid.
+        """
+        self._update_relation_data(relation_id, {"index": index})
+
+    def set_endpoints(self, relation_id: int, endpoints: str) -> None:
+        """Set the endpoints in the application relation databag.
+
+        Args:
+            relation_id: the identifier for a particular relation.
+            endpoints: the endpoint addresses for opensearch nodes.
+        """
+        self._update_relation_data(relation_id, {"endpoints": endpoints})
+
+    def set_version(self, relation_id: int, version: str) -> None:
+        """Set the database version in the application relation databag.
+
+        Args:
+            relation_id: the identifier for a particular relation.
+            version: database version.
+        """
+        self._update_relation_data(relation_id, {"version": version})
+
+
+class OpenSearchRequires(DataRequires):
+    """Requires-side of the OpenSearch relation."""
+
+    on = OpenSearchRequiresEvents()
+
+    def __init__(
+            self, charm, relation_name: str, index: str, extra_user_roles: Optional[str] = None
+        ):
+        """Manager of OpenSearch client relations."""
+        super().__init__(charm, relation_name, extra_user_roles)
+        self.charm = charm
+        self.index = index
+
+    def _on_relation_joined_event(self, event: RelationJoinedEvent) -> None:
+        """Event emitted when the application joins the OpenSearch relation."""
+        # Sets both index and extra user roles in the relation if the roles are provided.
+        # Otherwise, sets only the index.
+        data = {"index": self.index}
+        if self.extra_user_roles:
+            data["extra-user-roles"] = self.extra_user_roles
+
+        self._update_relation_data(event.relation.id, data)
+
+    def _on_relation_changed_event(self, event: RelationChangedEvent) -> None:
+        """Event emitted when the OpenSearch relation has changed.
+
+        This event triggers individual custom events depending on the changing relation.
+        """
+        # Check which data has changed to emit customs events.
+        diff = self._diff(event)
+
+        # Check if the index is created
+        # (the OpenSearch charm shares the credentials).
+        if "username" in diff.added and "password" in diff.added:
+            # Emit the default event (the one without an alias).
+            logger.info("index created at: %s", datetime.now())
+            self.on.index_created.emit(event.relation, app=event.app, unit=event.unit)
+
+            # To avoid unnecessary application restarts do not trigger
+            # “endpoints_changed“ or "authentication_updated" event if “index_created“ is
+            # triggered.
+            return
+
+        # Check if authentication has updated, emit event if so
+        updates = {"password", "tls", "tls-ca"}
+        if len(set(diff._asdict().keys()) - updates) < len(diff):
+            logger.info("authentication updated at: %s", datetime.now())
+            self.on.authentication_updated.emit(event.relation, app=event.app, unit=event.unit)
+
+            return
+
+        # Emit a endpoints changed event if the OpenSearch application added or changed this info
+        # in the relation databag.
+        if "endpoints" in diff.added or "endpoints" in diff.changed:
+            # Emit the default event (the one without an alias).
+            logger.info("endpoints changed on %s", datetime.now())
+            self.on.endpoints_changed.emit(
+                event.relation, app=event.app, unit=event.unit
+            )  # here check if this is the right design
+            return

lib/charms/opensearch/v0/opensearch_base_charm.py

@@ -332,7 +332,7 @@ def _on_opensearch_data_storage_detaching(self, _: StorageDetachingEvent):
     def _on_update_status(self, event: UpdateStatusEvent):
         """On update status event.
 
-        We want to periodically check for 3 things:
+        We want to periodically check for the following:
         1- Do we have users that need to be deleted, and if so we need to delete them.
         2- The system requirements are still met
         3- every 6 hours check if certs are expiring soon (in 7 days),
@@ -355,6 +355,9 @@ def _on_update_status(self, event: UpdateStatusEvent):
             self.opensearch_exclusions.cleanup()
             self._apply_cluster_health()
 
+        for relation in self.model.relations.get(ClientRelationName, []):
+            self.opensearch_provider.update_endpoints(relation)
+
         self.user_manager.remove_users_and_roles()
 
         # If relation broken - leave
@@ -472,8 +475,11 @@ def _is_tls_fully_configured(self) -> bool:
     def _start_opensearch(self, event: EventBase) -> None:  # noqa: C901
         """Start OpenSearch, with a generated or passed conf, if all resources configured."""
         if self.opensearch.is_started():
-            self._post_start_init()
-            self.status.clear(WaitingToStart)
+            try:
+                self._post_start_init()
+                self.status.clear(WaitingToStart)
+            except OpenSearchHttpError:
+                event.defer()
             return
 
         if not self._can_service_start():
@@ -842,8 +848,8 @@ def _check_certs_expiration(self, event: UpdateStatusEvent) -> None:
         certs = self.secrets.get_unit_certificates()
 
         # keep certificates that are expiring in less than 24h
-        for cert_type, cert in certs.items():
-            hours = cert_expiration_remaining_hours(cert)
+        for cert_type in list(certs.keys()):
+            hours = cert_expiration_remaining_hours(certs[cert_type])
             if hours > 24 * 7:
                 del certs[cert_type]