Use YAML configuration for Kafka connections, improve local runtime s…

…upport (#749)

* Load cluster configuration from YAML
* Add local build and compose scripts/configurations
* Documentation updates
* Cleanup config model, test updates, add instructions for running locally
* Optionally deploy Prometheus ingress, set Prometheus URL in compose
* Use host network for compose, document service account setup
* Update root README instructions for running locally

---------

Signed-off-by: Michael Edgar <[email protected]>

.github/workflows/release.yml

@@ -46,6 +46,7 @@ jobs:
           export QUARKUS_CONTAINER_IMAGE_USERNAME="${{ secrets.IMAGE_REPO_USERNAME }}"
           export QUARKUS_CONTAINER_IMAGE_PASSWORD="${{ secrets.IMAGE_REPO_PASSWORD }}"
           export QUARKUS_CONTAINER_IMAGE_PUSH="true"
+          export QUARKUS_CONTAINER_IMAGE_ADDITIONAL_TAGS=latest
           export GIT_REVISION=$(git rev-parse --short release)
           # Build and push the release images using the commit tagged in `release:prepare`
           mvn -B -P docker release:perform -f api/pom.xml
@@ -63,7 +64,8 @@ jobs:
           context: ui/
           push: true
           tags: |
-            ${{ secrets.IMAGE_REPO_HOSTNAME }}/${{ secrets.IMAGE_REPO_NAMESPACE }}/ui:${{steps.metadata.outputs.current-version}}
+            ${{ secrets.IMAGE_REPO_HOSTNAME }}/${{ secrets.IMAGE_REPO_NAMESPACE }}/console-ui:${{steps.metadata.outputs.current-version}}
+            ${{ secrets.IMAGE_REPO_HOSTNAME }}/${{ secrets.IMAGE_REPO_NAMESPACE }}/console-ui:latest
 
       - name: Push Release Tag
         run: |

.gitignore

@@ -21,6 +21,10 @@ release.properties
 .factorypath
 .vscode
 
-
 # env files
 .env*
+
+# User-provided and generated compose configurations
+/compose-runtime.env
+/console-config.yaml
+/compose.env

Makefile

@@ -0,0 +1,39 @@
+
+.PHONY: container-image-api container-image-ui container-images
+
+include *compose.env
+
+CONSOLE_API_IMAGE ?= quay.io/eyefloaters/console-api:latest
+CONSOLE_UI_IMAGE ?= quay.io/eyefloaters/console-api:latest
+CONSOLE_UI_NEXTAUTH_SECRET ?= $(shell openssl rand -base64 32)
+CONSOLE_METRICS_PROMETHEUS_URL ?= 
+CONTAINER_RUNTIME ?= $(shell which podman || which docker)
+
+container-image-api:
+	mvn package -f api/pom.xml -Pdocker -DskipTests -Dquarkus.container-image.image=$(CONSOLE_API_IMAGE)
+
+container-image-api-push: container-image-api
+	$(CONTAINER_RUNTIME) push $(CONSOLE_API_IMAGE)
+
+container-image-ui:
+	$(CONTAINER_RUNTIME) build -t $(CONSOLE_UI_IMAGE) ./ui -f ./ui/Dockerfile
+
+container-image-ui-push: container-image-ui
+	$(CONTAINER_RUNTIME) push $(CONSOLE_UI_IMAGE)
+
+container-images: container-image-api container-image-ui
+
+container-images-push: container-image-api-push container-image-ui-push
+
+compose-up:
+	> compose-runtime.env
+	echo "CONSOLE_API_IMAGE=$(CONSOLE_API_IMAGE)" >> compose-runtime.env
+	echo "CONSOLE_API_SERVICE_ACCOUNT_TOKEN=$(CONSOLE_API_SERVICE_ACCOUNT_TOKEN)" >> compose-runtime.env
+	echo "CONSOLE_API_KUBERNETES_API_SERVER_URL=$(CONSOLE_API_KUBERNETES_API_SERVER_URL)" >> compose-runtime.env 
+	echo "CONSOLE_UI_IMAGE=$(CONSOLE_UI_IMAGE)" >> compose-runtime.env
+	echo "CONSOLE_UI_NEXTAUTH_SECRET=$(CONSOLE_UI_NEXTAUTH_SECRET)" >> compose-runtime.env
+	echo "CONSOLE_METRICS_PROMETHEUS_URL=$(CONSOLE_METRICS_PROMETHEUS_URL)" >> compose-runtime.env
+	$(CONTAINER_RUNTIME) compose --env-file compose-runtime.env up -d
+
+compose-down:
+	$(CONTAINER_RUNTIME) compose --env-file compose-runtime.env down

README.md

@@ -6,10 +6,68 @@ It is composed of two main parts:
 - a REST API backend developed with Java and [Quarkus](https://quarkus.io/)
 - a user interface (UI) built with [Next.js](https://nextjs.org/) and [PatternFly](https://patternfly.org)
 
-## Installing
+#### Roadmap / Goals
+
+The future goals of this project are to provide a user interface to interact with and manage additional data streaming components such as:
+
+- [Apicurio Registry](https://www.apicur.io/registry/) for message serialization and de-serialization + validation
+- [Kroxylicious](https://kroxylicious.io/)
+- [Apache Flink](https://flink.apache.org/)
+
+Contributions and discussions around use cases for these (and other relevant) components are both welcome and encouraged.
+
+## Running the Application
+
+The console application may either be run in a Kubernetes cluster or locally to try it out.
+
+### Install to Kubernetes
 
 Please refer to the [installation README](./install/README.md) file for detailed information about how to install the latest release of the console in a Kubernetes cluster.
 
+### Run locally
+
+Running the console locally requires the use of a remote or locally-running Kubernetes cluster that hosts the Strimzi Kafka operator
+and any Apache Kafka™ clusters that will be accessed from the console. To get started, you will need to provide a console configuration
+file and credentials to connect to the Kubernetes cluster where Strimzi and Kafka are available.
+
+1. Using the [console-config-example.yaml](./console-config-example.yaml) file as an example, create your own configuration
+   in a file `console-config.yaml` in the repository root. The `compose.yaml` file expects this location to be used and
+   and difference in name or location requires an adjustment to the compose file.
+
+2. Install the prerequisite software into the Kubernetes cluster. This step assumes none have yet been installed.
+   ```shell
+   ./install/000-install-dependency-operators.sh <your namespace>
+   ./install/001-deploy-prometheus.sh <your namespace> <your cluster base domain>
+   ./install/002-deploy-console-kafka.sh <your namespace> <your cluster base domain>
+   ```
+   Note that the Prometheus instance will be available at `http://console-prometheus.<your cluster base domain>` when this step
+   completes.
+
+3. Provide the Prometheus endpoint, the API server endpoint, and the service account token that you would like to use to connect to the Kubernetes cluster. These may be placed in a `compose.env` file that will be detected when starting the console.
+   ```
+   CONSOLE_API_SERVICE_ACCOUNT_TOKEN=<TOKEN>
+   CONSOLE_API_KUBERNETES_API_SERVER_URL=https://my-kubernetes-api.example.com:6443
+   CONSOLE_METRICS_PROMETHEUS_URL=http://console-prometheus.<your cluster base domain>
+   ```
+   The service account token may be obtain using the `kubectl create token` command. For example, to create a service account
+   named "console-server" (from [console-server.serviceaccount.yaml](./install/resources/console/console-server.serviceaccount.yaml)
+   with the correct permissions and a token that expires in 1 year ([yq](https://github.com/mikefarah/yq/releases) required):
+   ```shell
+   export NAMESPACE=<service account namespace>
+   kubectl apply -n ${NAMESPACE} -f ./install/resources/console/console-server.clusterrole.yaml
+   kubectl apply -n ${NAMESPACE} -f ./install/resources/console/console-server.serviceaccount.yaml
+   yq '.subjects[0].namespace = strenv(NAMESPACE)' ./install/resources/console/console-server.clusterrolebinding.yaml | kubectl apply -n ${NAMESPACE} -f -
+   kubectl create token console-server -n ${NAMESPACE} --duration=$((365*24))h
+   ```
+
+4. By default, the provided configuration will use the latest console release container images. If you would like to
+   build your own images with changes you've made locally, you may also set the `CONSOLE_API_IMAGE` and `CONSOLE_UI_IMAGE`
+   in your `compose.env` and build them with `make container-images`
+
+5. Start the environment with `make compose-up`.
+
+6. When finished with the local console process, you may run `make compose-down` to clean up.
+
 ## Contributing
 
 We welcome contributions of all forms. Please see the [CONTRIBUTING](./CONTRIBUTING.md) file for how to get started.

api/pom.xml

@@ -6,7 +6,7 @@
     <groupId>com.github.eyefloaters</groupId>
     <artifactId>console-api</artifactId>
     <packaging>jar</packaging>
-    <version>0.0.36-SNAPSHOT</version>
+    <version>0.1.0-SNAPSHOT</version>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -322,6 +322,8 @@
                         </goals>
                         <configuration>
                             <systemProperties>
+                                <quarkus.jacoco.reuse-data-file>true</quarkus.jacoco.reuse-data-file>
+                                <quarkus.jacoco.report>false</quarkus.jacoco.report>
                                 <quarkus.docker.dockerfile-jvm-path>src/main/docker/Dockerfile</quarkus.docker.dockerfile-jvm-path>
                             </systemProperties>
                         </configuration>