Merge pull request #569 from cmgrote/master

Mapping change and draft lab chart extension
odpi · Jul 6, 2021 · 874f578 · 874f578
2 parents b5b68c5 + f85b9a9
commit 874f578
Show file tree

Hide file tree

Showing 16 changed files with 523 additions and 34 deletions.
diff --git a/charts/ec-ibmis-lab-ext/Chart.yaml b/charts/ec-ibmis-lab-ext/Chart.yaml
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright Contributors to the Egeria project.
+---
+name: ec-ibmis-lab-ext
+description: Egeria lab extension to include IBM Information Server connectors
+apiVersion: v1
+version: 0.1
+icon: https://raw.githubusercontent.com/odpi/egeria/99016e77167fa30dcfade809b061358a92a59973/assets/img/egeria.png
+keywords:
+  - egeria, open-metadata, connector, lab, ibm, information-server
+sources:
+  - https://github.com/odpi/egeria-connector-ibm-information-server
+home: https://github.com/odpi/egeria-connector-ibm-information-server
+maintainers:
+  - name: Christopher Grote
+    email: [email protected]
diff --git a/charts/ec-ibmis-lab-ext/requirements.yaml b/charts/ec-ibmis-lab-ext/requirements.yaml
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright Contributors to the Egeria project.
+---
+dependencies:
+  - name: odpi-egeria-lab
+    version: 1.0.2
+    repository: https://odpi.github.io/egeria-charts
diff --git a/charts/ec-ibmis-lab-ext/templates/_helpers.tpl b/charts/ec-ibmis-lab-ext/templates/_helpers.tpl
@@ -0,0 +1,15 @@
+{{/* <!-- SPDX-License-Identifier: Apache-2.0 --> */}}
+{{/* Copyright Contributors to the Egeria project. */}}{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "myapp.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "myapp.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
diff --git a/charts/ec-ibmis-lab-ext/templates/configmap.yaml b/charts/ec-ibmis-lab-ext/templates/configmap.yaml
@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright Contributors to the Egeria project.
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-configmap
+  labels:
+    app.kubernetes.io/name: {{ include "myapp.name" . }}
+    helm.sh/chart: {{ include "myapp.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+  # General cluster configuration - for core egeria, kafka - these must be used
+  EGERIA_USER: {{ .Values.egeria.user }}
+  EGERIA_COHORT: {{ .Values.egeria.cohort }}
+  KAFKA_ENDPOINT: {{ .Release.Name }}-kafka:9092
+
+  # Used by IBM Information Server
+  STRICT_SSL: "false"
+  LOGGING_LEVEL_ROOT: {{ .Values.logging.level }}
+  PROXY_ENDPOINT: https://{{ .Release.Name }}-infosvr-proxy:9443
+  IBM_HOST: infosvr
+  IBM_PORT: "9446"
+  IBM_KAFKA_ENDPOINT: infosvr:59092
+  IBM_USER: {{ .Values.ibm.user }}
+  IBM_PASS: {{ .Values.ibm.password }}
+  IBM_IGC_SERVER: {{ .Values.ibm.igc }}
+  IBM_DATASTAGE_SERVER: {{ .Values.ibm.datastage }}
+  OMAS_SERVER: "ibm_omas"
+
+  # Used for downloads
+  CONNECTOR_JAR: egeria-connector-ibm-information-server-package-{{  .Values.connector.version }}-jar-with-dependencies.jar
+  CONNECTOR_URL: {{  .Values.connector.url }}
diff --git a/charts/ec-ibmis-lab-ext/templates/ibm-infosvr.yaml b/charts/ec-ibmis-lab-ext/templates/ibm-infosvr.yaml
@@ -0,0 +1,154 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright Contributors to the Egeria project.
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: {{ include "myapp.name" . }}
+    helm.sh/chart: {{ include "myapp.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/component: infosvr-proxy
+  name: {{ .Release.Name }}-infosvr-proxy
+
+spec:
+  ports:
+    - name: proxy
+      port: 9443
+    - name: web
+      port: 9446
+    - name: zookeeper
+      port: 52181
+    - name: kafka
+      port: 59092
+  selector:
+    app.kubernetes.io/name: {{ include "myapp.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: infosvr-proxy
+...
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-infosvr
+  labels:
+    app.kubernetes.io/name: {{ include "myapp.name" . }}
+    helm.sh/chart: {{ include "myapp.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/component: infosvr-proxy
+
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "myapp.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: infosvr-proxy
+  strategy: {}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "myapp.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: infosvr-proxy
+    spec:
+      # Fix the hostname to what the image requires of itself and define the secrets to use
+      # for pulling the container image from a private repository
+      hostname: infosvr
+      imagePullSecrets:
+        - name: regcred
+      volumes:
+        - name: egeria-igc-connector-volume
+          emptyDir: {}
+      initContainers:
+        - name: init-connector
+          image: "{{ if (.Values.image.configure.registry | default .Values.imageDefaults.registry) }}{{ .Values.image.configure.registry | default .Values.imageDefaults.registry }}/{{ end }}\
+                  {{ if (.Values.image.configure.namespace | default .Values.imageDefaults.namespace) }}{{ .Values.image.configure.namespace | default .Values.imageDefaults.namespace }}/{{ end }}\
+                  {{ .Values.image.configure.name }}\
+                  :{{ .Values.image.configure.tag | default .Values.imageDefaults.tag }}"
+          imagePullPolicy: {{ .Values.image.configure.pullPolicy | default .Values.imageDefaults.pullPolicy }}
+          envFrom:
+            - configMapRef:
+                name: {{ .Release.Name }}-configmap
+          command:
+            - "/bin/bash"
+            - "-c"
+            - >
+                cd /opt/egeria/connectors &&
+                curl --location ${CONNECTOR_URL} --output ${CONNECTOR_JAR}
+          volumeMounts:
+            - mountPath: /opt/egeria/connectors
+              name: egeria-igc-connector-volume
+      containers:
+        - name: proxy
+          image: "{{ if (.Values.image.egeria.registry | default .Values.imageDefaults.registry) }}{{ .Values.image.egeria.registry | default .Values.imageDefaults.registry }}/{{ end }}\
+                  {{ if (.Values.image.egeria.namespace | default .Values.imageDefaults.namespace) }}{{ .Values.image.egeria.namespace | default .Values.imageDefaults.namespace }}/{{ end }}\
+                  {{ .Values.image.egeria.name }}\
+                  :{{ .Values.image.egeria.tag | default .Values.imageDefaults.tag }}"
+          imagePullPolicy: {{ .Values.image.egeria.pullPolicy | default .Values.imageDefaults.pullPolicy }}
+          envFrom:
+            - configMapRef:
+                name: {{ .Release.Name }}-configmap
+          env:
+              - name: "LOADER_PATH"
+                value: "/opt/egeria/connectors"
+              - name: "LOGGING_LEVEL_ORG_ODPI_EGERIA_CONNECTORS_IBM_IGC"
+                value: "DEBUG"
+          ports:
+            - containerPort: 9443
+          readinessProbe:
+            tcpSocket:
+              port: 9443
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            failureThreshold: 6
+          resources:
+            requests:
+              memory: "1Gi"
+              cpu: "250m"
+            limits:
+              memory: "2Gi"
+              cpu: "1000m"
+          volumeMounts:
+            - mountPath: /opt/egeria/connectors
+              name: egeria-igc-connector-volume
+              readOnly: true
+        - name: infosvr
+          image: "{{ if (.Values.image.infosvr.registry | default .Values.imageDefaults.registry) }}{{ .Values.image.infosvr.registry | default .Values.imageDefaults.registry }}/{{ end }}\
+                  {{ if (.Values.image.infosvr.namespace | default .Values.imageDefaults.namespace) }}{{ .Values.image.infosvr.namespace | default .Values.imageDefaults.namespace }}/{{ end }}\
+                  {{ .Values.image.infosvr.name }}\
+                  :{{ .Values.image.infosvr.tag }}"
+          imagePullPolicy: {{ .Values.image.infosvr.pullPolicy | default .Values.imageDefaults.pullPolicy }}
+          securityContext:
+            privileged: true
+          ports:
+            - containerPort: 9446
+            - containerPort: 52181
+            - containerPort: 59092
+          livenessProbe:
+            httpGet:
+              path: /ibm/iis/launchpad
+              port: 9446
+              scheme: "HTTPS"
+            initialDelaySeconds: 720
+            periodSeconds: 20
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /ibm/iis/launchpad
+              port: 9446
+              scheme: "HTTPS"
+            initialDelaySeconds: 300
+            periodSeconds: 20
+            failureThreshold: 12
+          resources:
+            requests:
+              memory: "4Gi"
+              cpu: "2000m"
+            limits:
+              memory: "8Gi"
+              cpu: "6000m"
+
+...
diff --git a/charts/ec-ibmis-lab-ext/templates/init.yaml b/charts/ec-ibmis-lab-ext/templates/init.yaml
@@ -0,0 +1,138 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright Contributors to the Egeria project.
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Release.Name }}-init
+  labels:
+    app.kubernetes.io/name: {{ include "myapp.name" . }}
+    helm.sh/chart: {{ include "myapp.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/component: init
+
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "myapp.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: init
+    spec:
+      restartPolicy: Never
+      serviceAccountName: {{ .Release.Name }}-lab
+      # Wait for all of the services to be up before attempting initial configuration
+      initContainers:
+        - name: wait-for-igcproxy
+          image: "{{ if (.Values.image.configure.registry | default .Values.imageDefaults.registry) }}{{ .Values.image.configure.registry | default .Values.imageDefaults.registry }}/{{ end }}\
+                  {{ if (.Values.image.configure.namespace | default .Values.imageDefaults.namespace) }}{{ .Values.image.configure.namespace | default .Values.imageDefaults.namespace }}/{{ end }}\
+                  {{ .Values.image.configure.name }}\
+                  :{{ .Values.image.configure.tag | default .Values.imageDefaults.tag }}"
+          imagePullPolicy: {{ .Values.image.configure.pullPolicy | default .Values.imageDefaults.pullPolicy }}
+          env:
+            - name: SERVICE
+              value: {{ .Release.Name }}-infosvr-proxy
+        - name: wait-for-kafka
+          image: "{{ if (.Values.image.configure.registry | default .Values.imageDefaults.registry) }}{{ .Values.image.configure.registry | default .Values.imageDefaults.registry }}/{{ end }}\
+                  {{ if (.Values.image.configure.namespace | default .Values.imageDefaults.namespace) }}{{ .Values.image.configure.namespace | default .Values.imageDefaults.namespace }}/{{ end }}\
+                  {{ .Values.image.configure.name }}\
+                  :{{ .Values.image.configure.tag | default .Values.imageDefaults.tag }}"
+          imagePullPolicy: {{ .Values.image.configure.pullPolicy | default .Values.imageDefaults.pullPolicy }}
+          env:
+            - name: SERVICE
+              value: {{ .Release.Name }}-kafka
+        - name: wait-for-ibm-igc
+          image: "{{ if (.Values.image.configure.registry | default .Values.imageDefaults.registry) }}{{ .Values.image.configure.registry | default .Values.imageDefaults.registry }}/{{ end }}\
+                  {{ if (.Values.image.configure.namespace | default .Values.imageDefaults.namespace) }}{{ .Values.image.configure.namespace | default .Values.imageDefaults.namespace }}/{{ end }}\
+                  {{ .Values.image.configure.name }}\
+                  :{{ .Values.image.configure.tag | default .Values.imageDefaults.tag }}"
+          imagePullPolicy: {{ .Values.image.configure.pullPolicy | default .Values.imageDefaults.pullPolicy }}
+          envFrom:
+            - configMapRef:
+                name: {{ .Release.Name }}-configmap
+          command:
+            - "/bin/bash"
+            - "-c"
+            - >
+                until $(curl -c /tmp/cookies.txt -s -f -k --connect-timeout 5 -X POST -u ${IBM_USER}:${IBM_PASS} --url https://{{ .Release.Name }}-infosvr-proxy:${IBM_PORT}/ibm/iis/igc-rest/v1/search -H "Content-Type: application/json" -d '{"types":["category","term","information_governance_policy","information_governance_rule"],"pageSize":1,"workflowMode":"draft"}' | grep 'numTotal' >/dev/null); do
+                curl -b /tmp/cookies.txt -s -f -k --connect-timeout 5 -X GET -u ${IBM_USER}:${IBM_PASS} --url https://{{ .Release.Name }}-infosvr-proxy:${IBM_PORT}/ibm/iis/igc-rest/v1/logout;
+                echo waiting for {{ .Release.Name }}-infosvr-proxy to be accessible;
+                sleep 2;
+                done;
+      # Then configure and startup the DataStage connector
+      containers:
+        - name: init
+          image: "{{ if (.Values.image.configure.registry | default .Values.imageDefaults.registry) }}{{ .Values.image.configure.registry | default .Values.imageDefaults.registry }}/{{ end }}\
+                  {{ if (.Values.image.configure.namespace | default .Values.imageDefaults.namespace) }}{{ .Values.image.configure.namespace | default .Values.imageDefaults.namespace }}/{{ end }}\
+                  {{ .Values.image.configure.name }}\
+                  :{{ .Values.image.configure.tag | default .Values.imageDefaults.tag }}"
+          imagePullPolicy: {{ .Values.image.configure.pullPolicy | default .Values.imageDefaults.pullPolicy }}
+          envFrom:
+            - configMapRef:
+                name: {{ .Release.Name }}-configmap
+          command:
+            - "/bin/bash"
+            - "-c"
+            - >
+                curl -f -k -w "\n  (%{http_code} - %{url_effective})\n" --silent --basic admin:admin -X POST
+                ${PROXY_ENDPOINT}/open-metadata/admin-services/users/${EGERIA_USER}/servers/${IBM_IGC_SERVER}/server-type?typeName=IGC
+                &&
+                curl -f -k -w "\n  (%{http_code} - %{url_effective})\n" --silent --basic admin:admin -X POST
+                ${PROXY_ENDPOINT}/open-metadata/admin-services/users/${EGERIA_USER}/servers/${IBM_IGC_SERVER}/organization-name?name=IBM
+                &&
+                curl -f -k -w "\n  (%{http_code} - %{url_effective})\n" --silent --basic admin:admin -X POST
+                ${PROXY_ENDPOINT}/open-metadata/admin-services/users/${EGERIA_USER}/servers/${IBM_IGC_SERVER}/server-url-root?url=${PROXY_ENDPOINT}
+                &&
+                curl -f -k -w "\n  (%{http_code} - %{url_effective})\n" --silent --basic admin:admin -X POST
+                --header "Content-Type: application/json"
+                ${PROXY_ENDPOINT}/open-metadata/admin-services/users/${EGERIA_USER}/servers/${IBM_IGC_SERVER}/event-bus
+                --data '{"producer": {"bootstrap.servers": "'"${KAFKA_ENDPOINT}"'"}, "consumer": {"bootstrap.servers": "'"${KAFKA_ENDPOINT}"'"} }'
+                &&
+                curl -f -k -w "\n  (%{http_code} - %{url_effective})\n" --silent --basic admin:admin -X POST
+                --header "Content-Type: application/json"
+                ${PROXY_ENDPOINT}/open-metadata/admin-services/users/${EGERIA_USER}/servers/${IBM_IGC_SERVER}/local-repository/mode/repository-proxy/connection
+                --data '{"class":"Connection","connectorType":{"class":"ConnectorType","connectorProviderClassName":"org.odpi.egeria.connectors.ibm.igc.repositoryconnector.IGCOMRSRepositoryConnectorProvider"},"endpoint":{"class":"Endpoint","address":"'"${IBM_HOST}:${IBM_PORT}"'","protocol":"https"},"userId":"'"${IBM_USER}"'","clearPassword":"'"${IBM_PASS}"'","configurationProperties":{"defaultZones":["default"]}}'
+                &&
+                curl -f -k -w "\n  (%{http_code} - %{url_effective})\n" --silent --basic admin:admin -X POST
+                ${PROXY_ENDPOINT}/open-metadata/admin-services/users/${EGERIA_USER}/servers/${IBM_IGC_SERVER}/cohorts/${EGERIA_COHORT}
+                &&
+                curl -f -k -w "\n  (%{http_code} - %{url_effective})\n" --silent --basic admin:admin -X POST
+                ${PROXY_ENDPOINT}/open-metadata/admin-services/users/${EGERIA_USER}/servers/${IBM_IGC_SERVER}/instance
+                &&
+                curl -f -k -w "\n  (%{http_code} - %{url_effective})\n" --silent --basic admin:admin -X POST
+                ${PROXY_ENDPOINT}/open-metadata/admin-services/users/${EGERIA_USER}/servers/${OMAS_SERVER}/local-repository/mode/local-graph-repository
+                &&
+                curl -f -k -w "\n  (%{http_code} - %{url_effective})\n" --silent --basic admin:admin -X POST
+                ${PROXY_ENDPOINT}/open-metadata/admin-services/users/${EGERIA_USER}/servers/${OMAS_SERVER}/server-url-root?url=${PROXY_ENDPOINT}
+                &&
+                curl -f -k -w "\n  (%{http_code} - %{url_effective})\n" --silent --basic admin:admin -X POST
+                --header "Content-Type: application/json"
+                ${PROXY_ENDPOINT}/open-metadata/admin-services/users/${EGERIA_USER}/servers/${OMAS_SERVER}/event-bus
+                --data '{"producer": {"bootstrap.servers": "'"${KAFKA_ENDPOINT}"'"}, "consumer": {"bootstrap.servers": "'"${KAFKA_ENDPOINT}"'"} }'
+                &&
+                curl -f -k -w "\n  (%{http_code} - %{url_effective})\n" --silent --basic admin:admin -X POST
+                ${PROXY_ENDPOINT}/open-metadata/admin-services/users/${EGERIA_USER}/servers/${OMAS_SERVER}/access-services/data-engine
+                &&
+                curl -f -k -w "\n  (%{http_code} - %{url_effective})\n" --silent --basic admin:admin -X POST
+                ${PROXY_ENDPOINT}/open-metadata/admin-services/users/${EGERIA_USER}/servers/${OMAS_SERVER}/instance
+                &&
+                curl -f -k -w "\n  (%{http_code} - %{url_effective})\n" --silent --basic admin:admin -X POST
+                ${PROXY_ENDPOINT}/open-metadata/admin-services/users/${EGERIA_USER}/servers/${IBM_DATASTAGE_SERVER}/local-repository/mode/in-memory-repository
+                &&
+                curl -f -k -w "\n  (%{http_code} - %{url_effective})\n" --silent --basic admin:admin -X POST
+                --header "Content-Type: application/json"
+                ${PROXY_ENDPOINT}/open-metadata/admin-services/users/${EGERIA_USER}/servers/${IBM_DATASTAGE_SERVER}/data-engine-proxy-service/configuration
+                --data '{"class":"DataEngineProxyConfig","accessServiceRootURL":"'"${PROXY_ENDPOINT}"'","accessServiceServerName":"'"${OMAS_SERVER}"'","eventsClientEnabled":true,"dataEngineConnection":{"class":"Connection","connectorType":{"class":"ConnectorType","connectorProviderClassName":"org.odpi.egeria.connectors.ibm.datastage.dataengineconnector.DataStageConnectorProvider"},"endpoint":{"class":"Endpoint","address":"'"${IBM_HOST}:${IBM_PORT}"'","protocol":"https"},"userId":"'"${IBM_USER}"'","clearPassword":"'"${IBM_PASS}"'","configurationProperties":{"limitToProjects":["minimal"]}},"pollIntervalInSeconds":60}'
+                &&
+                curl -f -k -w "\n  (%{http_code} - %{url_effective})\n" --silent --basic admin:admin -X POST
+                ${PROXY_ENDPOINT}/open-metadata/admin-services/users/${EGERIA_USER}/servers/${IBM_DATASTAGE_SERVER}/instance
+          resources:
+            requests:
+              memory: "16Mi"
+              cpu: "100m"
+            limits:
+              memory: "64Mi"
+              cpu: "500m"
+...