Documentation updates

Signed-off-by: Michael Edgar <[email protected]>
streamshub · May 16, 2024 · 5471062 · 5471062
1 parent f55839a
commit 5471062
Show file tree

Hide file tree

Showing 8 changed files with 95 additions and 74 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -46,6 +46,7 @@ jobs:
           export QUARKUS_CONTAINER_IMAGE_USERNAME="${{ secrets.IMAGE_REPO_USERNAME }}"
           export QUARKUS_CONTAINER_IMAGE_PASSWORD="${{ secrets.IMAGE_REPO_PASSWORD }}"
           export QUARKUS_CONTAINER_IMAGE_PUSH="true"
+          export QUARKUS_CONTAINER_IMAGE_ADDITIONAL_TAGS=latest
           export GIT_REVISION=$(git rev-parse --short release)
           # Build and push the release images using the commit tagged in `release:prepare`
           mvn -B -P docker release:perform -f api/pom.xml
@@ -63,7 +64,8 @@ jobs:
           context: ui/
           push: true
           tags: |
-            ${{ secrets.IMAGE_REPO_HOSTNAME }}/${{ secrets.IMAGE_REPO_NAMESPACE }}/ui:${{steps.metadata.outputs.current-version}}
+            ${{ secrets.IMAGE_REPO_HOSTNAME }}/${{ secrets.IMAGE_REPO_NAMESPACE }}/console-ui:${{steps.metadata.outputs.current-version}}
+            ${{ secrets.IMAGE_REPO_HOSTNAME }}/${{ secrets.IMAGE_REPO_NAMESPACE }}/console-ui:latest
 
       - name: Push Release Tag
         run: |

diff --git a/README.md b/README.md
@@ -6,10 +6,16 @@ It is composed of two main parts:
 - a REST API backend developed with Java and [Quarkus](https://quarkus.io/)
 - a user interface (UI) built with [Next.js](https://nextjs.org/) and [PatternFly](https://patternfly.org)
 
-## Installing
+## Running the Application
 
+The console application may either be run in a Kubernetes cluster or locally to try it out.
+
+### Install to Kubernetes
 Please refer to the [installation README](./install/README.md) file for detailed information about how to install the latest release of the console in a Kubernetes cluster.
 
+### Run locally
+
+
 ## Contributing
 
 We welcome contributions of all forms. Please see the [CONTRIBUTING](./CONTRIBUTING.md) file for how to get started.

diff --git a/api/pom.xml b/api/pom.xml
@@ -6,7 +6,7 @@
     <groupId>com.github.eyefloaters</groupId>
     <artifactId>console-api</artifactId>
     <packaging>jar</packaging>
-    <version>0.0.36-SNAPSHOT</version>
+    <version>0.1.0-SNAPSHOT</version>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

diff --git a/api/src/main/java/com/github/eyefloaters/console/api/ClientFactory.java b/api/src/main/java/com/github/eyefloaters/console/api/ClientFactory.java
@@ -139,6 +139,8 @@ public ConsoleConfig produceConsoleConfig() {
             })
             .filter(Objects::nonNull)
             .map(url -> {
+                log.infof("Loading console configuration from %s", url);
+
                 ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
 
                 try (InputStream stream = url.openStream()) {
@@ -147,7 +149,10 @@ public ConsoleConfig produceConsoleConfig() {
                     throw new UncheckedIOException(e);
                 }
             })
-            .orElseGet(ConsoleConfig::new);
+            .orElseGet(() -> {
+                log.infof("Console configuration not specified");
+                return new ConsoleConfig();
+            });
     }
 
     /**
@@ -349,6 +354,17 @@ Map<String, Object> buildConfig(Set<String> configNames,
             .map(Optional::get)
             .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
 
+        if (!cfg.containsKey(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG)) {
+            Optional.ofNullable(cluster.getStatus())
+                .map(KafkaStatus::getListeners)
+                .map(Collection::stream)
+                .orElseGet(Stream::empty)
+                .filter(listener -> listener.getName().equals(config.getListener()))
+                .map(ListenerStatus::getBootstrapServers)
+                .findFirst()
+                .ifPresent(bootstrapServers -> cfg.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers));
+        }
+
         if (truststoreRequired(cfg)) {
             if (trustManager.isResolvable()) {
                 trustManager.get().trustClusterCertificate(cfg);

diff --git a/console-config-example.yaml b/console-config-example.yaml
@@ -0,0 +1,28 @@
+kafka:
+  clusters:
+    - name: my-kafka1 # name of the Strimzi Kafka CR
+      namespace: my-namespace1 # namespace of the Strimzi Kafka CR
+      listener: "secure" # name of the listener to use for connections from the console
+      # `properties` contains keys/values to use for any Kafka connection
+      properties:
+        security.protocol: SASL_SSL
+        sasl.mechanism: SCRAM-SHA-512
+        bootstrap.servers: my-kafka1.cloud.example.com:9093 # optional, if omitted the bootstrap servers from the Strimzi Kafka CR are used
+        sasl.jaas.config: org.apache.kafka.common.security.scram.ScramLoginModule required username="kafka1-user" password="sCr@m!";
+      # `adminProperties` contains keys/values to use for Admin client Kafka connections.
+      # Properties specified here override properties of the same name in `properties`
+      adminProperties: {}
+      # `consumerProperties` contains keys/values to use for Consumer client Kafka connections.
+      # Properties specified here override properties of the same name in `properties`
+      consumerProperties: {}
+      # `producerProperties` contains keys/values to use for Producer client Kafka connections.
+      # Properties specified here override properties of the same name in `properties`
+      producerProperties: {}
+
+    - name: my-kafka2
+      namespace: my-namespace2
+      listener: "secure"
+      properties:
+        security.protocol: SASL_SSL
+        sasl.mechanism: SCRAM-SHA-512
+        sasl.jaas.config: org.apache.kafka.common.security.scram.ScramLoginModule required username="kafka2-user" password="sCr@m!";
diff --git a/install/003-install-console.sh b/install/003-install-console.sh
@@ -7,6 +7,7 @@ RESOURCE_PATH=${CONSOLE_INSTALL_PATH}/resources
 
 export NAMESPACE="${1?Please provide the deployment namespace}"
 export CLUSTER_DOMAIN="${2?Please provide the base domain name for Kafka listener ingress}"
+export CONSOLE_CONFIG="${3?Please provide the path to a console configuration YAML. See console-config-example.yaml for sample}"
 
 source ${CONSOLE_INSTALL_PATH}/_common.sh
 
@@ -76,6 +77,11 @@ else
     echo -e "${WARN} Console Credential secret console-ui-secrets already exists, nothing applied"
 fi
 
+${KUBE} create secret generic console-config -n ${NAMESPACE} \
+    --dry-run=client \
+    --from-file=console-config.yaml="${CONSOLE_CONFIG}" \
+    -o yaml | ${KUBE} apply -n ${NAMESPACE} -f -
+
 if ${KUBE} get deployment console -n ${NAMESPACE} 1>/dev/null 2>&1 ; then
     ${KUBE} scale --replicas=0 deployment/console -n ${NAMESPACE}
 fi

diff --git a/install/README.md b/install/README.md
@@ -7,7 +7,7 @@ are available on the `PATH`.
 
 ## Prerequisites
 
-### Strimzi & Prometheus
+### 1. Strimzi & Prometheus
 
 The console requires that the Strimzi Kafka Operator is installed and available in the cluster before
 deployment. Strimzi may be installed either using Operator Lifecycle Manager (OLM, preferred) or directly
@@ -19,21 +19,25 @@ is not available, the cluster metrics graphs on the Kafka cluster overview scree
 
 Users who do not previously have Strimzi and Promethus installed may use the `000-install-dependency-operators.sh`
 and `001-deploy-prometheus.sh` scripts to bootstrap the environment. The scripts will install either the community-supported
-or commercially supported (i.e. AMQ Streams) version of the two operators using OLM and deploy a Prometheus instance
-configured to scrape metrics from any Kafka clusters deployed by Strimzi within the cluster.
+or commercially supported (i.e. streams for Apache Kafka) version of the two operators using OLM and deploy a Prometheus instance
+configured to scrape metrics from Kafka clusters deployed by Strimzi within the cluster.
 
 ```shell
 000-install-dependency-operators.sh ${TARGET_NAMESPACE}
 001-deploy-prometheus.sh ${TARGET_NAMESPACE}
 ```
 
-### Apache Kafka Cluster
+### 2. Apache Kafka Cluster
 
 Once the two prerequisite components have been installed, the demo Kafka cluster may be created using the
 `002-deploy-console-kafka.sh` script. This script will create a Strimzi `Kafka` custom resource as well as a
 `KafkaUser` custom resource for a user to access the cluster. Additionally, the Kafka cluster will be configured via
 a ConfigMap to export metrics in the way expected by the Prometheus instance created earlier.
 
+```shell
+002-deploy-console-kafka.sh ${TARGET_NAMESPACE} ${CLUSTER_DOMAIN}
+```
+
 ### Authorization
 
 In order to allow the necessary access for the console to function, a minimum level of authorization must be configured
@@ -45,47 +49,19 @@ of ACL types are:
 1. `READ`, `DESCRIBE`, `DESCRIBE_CONFIGS` for all `TOPIC` resources
 1. `READ`, `DESCRIBE` for all `GROUP` resources
 
-## Installation
+## 3. Installation
 
 With the prerequisites met, the console can be deployed using the `003-install-console.sh` script. This script will
 create the role, role binding, service account, services, and ingress (or route in OpenShift) necessary to run the console.
 Finally, the console deployment is applied to the Kubernetes/OpenShift cluster. A link to access the application will
 be printed to the script's output if no errors are encountered.
 
-The configurations used by the console to connect to Kafka may be customized by altering the environment variables
-for the `console-api` container in `resources/console/console.deployment.yaml`. The format used for the variables
-is as follows.
-
-Configurations that apply to all Kafka connections should use the format `KAFKA_CONFIG_WITH_UNDERSCORES`. For example,
-if all clusters are configured to use `SASL_SSL` for the Kafka `security.protocol` property, you may set env
-`KAFKA_SECURITY_PROTOCOL` to `SASL_SSL`.
-
-Each individual cluster must be configured with a variable like `CONSOLE_KAFKA_CLUSTER1` where `CLUSTER1` is a unique
-name or identifier for each cluster and the value of the env is the `${namespace}/${name}` of the `Kafka` CR that
-represents the cluster.
-
-Configurations that apply to an individual Kafka connection should use the format `CONSOLE_KAFKA_CLUSTER1_CONFIG_WITH_UNDERSCORES`.
-Using the example above, if you would like to configure one cluster to use `SASL_SSL` for the Kafka `security.protocol` property,
-the following env settings would be needed:
-
-```yaml
-- name: CONSOLE_KAFKA_EXAMPLE
-  value: example-ns/console-kafka
-- name: CONSOLE_KAFKA_EXAMPLE_SECURITY_PROTOCOL
-  value: SASL_SSL
-- name: CONSOLE_KAFKA_EXAMPLE_BOOTSTRAP_SERVERS
-  value: bootstrap.console-kafka.example.com:443
-```
-
-As always, configuration properties that contain sensitive information may be mounted from a `Secret`. For example, to
-set the `sasl.jaas.config` property, you could use an env entry such as the following.
+The configurations used by the console to connect to Kafka may be customized by providing your own configuration.
+See [console-config-example.yaml](../console-config-example.yaml) in the repository root for an example. The path to
+the customized configuration must be provided as the third argument of `003-install-console.sh`
 
-```yaml
-- name: CONSOLE_KAFKA_EXAMPLE_SASL_JAAS_CONFIG
-  valueFrom:
-    secretKeyRef:
-      name: console-kafka-user1
-      key: sasl.jaas.config
+```shell
+003-install-console.sh ${TARGET_NAMESPACE} ${CLUSTER_DOMAIN} ${CONSOLE_CONFIG}
 ```
 
 ## References

diff --git a/install/resources/console/console.deployment.yaml b/install/resources/console/console.deployment.yaml
@@ -15,56 +15,43 @@ spec:
     spec:
       serviceAccountName: console-server
       volumes:
-      - emptyDir: {}
-        name: cache
+      - name: cache
+        emptyDir: {}
+      - name: config
+        secret:
+          secretName: console-config
       containers:
       ### API
       - name: console-api
-        image: quay.io/eyefloaters/console-api:0.0.35
+        image: quay.io/eyefloaters/console-api:latest
+        imagePullPolicy: Always
         ports:
         - containerPort: 8080
-        # Adjust KAFKA and CONSOLE_KAFKA variables to match your Kafka cluster
+        volumeMounts:
+        - name: config
+          mountPath: /deployments/console-config.yaml
+          subPath: console-config.yaml
         env:
-        - name: KAFKA_SECURITY_PROTOCOL
-          value: SASL_SSL
-        - name: KAFKA_SASL_MECHANISM
-          value: SCRAM-SHA-512
-        - name: CONSOLE_KAFKA_EXAMPLE
-          value: ${NAMESPACE}/console-kafka
-        - name: CONSOLE_KAFKA_EXAMPLE_BOOTSTRAP_SERVERS
-          # Replace with your Kafka's boostrap address
-          value: bootstrap.console-kafka.${CLUSTER_DOMAIN}:443
-        - name: CONSOLE_KAFKA_EXAMPLE_SASL_JAAS_CONFIG
-          valueFrom:
-            secretKeyRef:
-              name: console-kafka-user1
-              key: sasl.jaas.config
+        - name: CONSOLE_CONFIG_PATH
+          value: /deployments/console-config.yaml
       ### User Interface
       - name: console-ui
-        image: quay.io/eyefloaters/ui:0.0.35
+        image: quay.io/eyefloaters/console-ui:latest
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 3000
         volumeMounts:
-        - mountPath: /app/.next/cache
-          name: cache
+        - name: cache
+          mountPath: /app/.next/cache
         env:
         - name: NEXTAUTH_SECRET
           valueFrom:
             secretKeyRef:
               name: console-ui-secrets
               key: NEXTAUTH_SECRET
-        - name: SESSION_SECRET
-          valueFrom:
-            secretKeyRef:
-              name: console-ui-secrets
-              key: SESSION_SECRET
         - name: NEXTAUTH_URL
           value: 'https://${CONSOLE_HOSTNAME}'
         - name: BACKEND_URL
           value: 'http://127.0.0.1:8080'
         - name: CONSOLE_METRICS_PROMETHEUS_URL
           value: 'http://prometheus-operated.${NAMESPACE}.svc.cluster.local:9090'
-        - name: CONSOLE_MODE
-          value: read-only
-        - name: LOG_LEVEL
-          value: info
-        ports:
-        - containerPort: 3000