diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/Chart.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/Chart.yaml new file mode 100644 index 0000000..1e244f9 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/Chart.yaml @@ -0,0 +1,13 @@ +name: zeppelin-spark +version: 0.0.24 +description: An umbrella Helm chart for Zeppelin and Spark +home: https://banzaicloud.com +sources: + - https://github.com/banzaicloud/zeppelin + - https://github.com/banzaicloud/spark +keywords: +- spark +- zeppelin +maintainers: +- name: Banzai Cloud + email: info@banzaicloud.com diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/Chart.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/Chart.yaml new file mode 100644 index 0000000..a4a4e7d --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/Chart.yaml @@ -0,0 +1,13 @@ +name: spark +version: 0.0.19 +description: A Helm chart for Spark in Kubernetes +home: https://banzaicloud.com +sources: + - https://github.com/banzaicloud/spark +keywords: +- spark +- spotguide +icon: https://spark.apache.org/images/spark-logo-trademark.png +maintainers: +- name: Banzai Cloud + email: info@banzaicloud.com diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/.helmignore b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/Chart.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/Chart.yaml new file mode 100644 index 0000000..b05981a --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/Chart.yaml @@ -0,0 +1,12 @@ +name: spark-hs +version: 0.2.3 +description: A Helm chart for Spark HS in Kubernetes +home: https://banzaicloud.com +sources: + - https://github.com/banzaicloud/spark +keywords: +- spark +- history-server +maintainers: +- name: Banzai Cloud + email: info@banzaicloud.com diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/NOTES.txt b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/NOTES.txt new file mode 100644 index 0000000..e69de29 diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/_helpers.tpl b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/_helpers.tpl new file mode 100644 index 0000000..5593335 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "spark-hs.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "spark-hs.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "spark-hs.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/deployment.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/deployment.yaml new file mode 100644 index 0000000..35e695b --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/deployment.yaml @@ -0,0 +1,94 @@ +{{- define "logDirectoryTag" }} + {{- if eq .Values.sparkEventLogStorage.cloudProvider "amazon"}} + {{- printf "s3a://%s" .Values.sparkEventLogStorage.logDirectory }} + {{- else if eq .Values.sparkEventLogStorage.cloudProvider "alibaba"}} + {{- printf "oss://%s" .Values.sparkEventLogStorage.logDirectory }} + {{- else if eq .Values.sparkEventLogStorage.cloudProvider "azure"}} + {{- printf "wasb://%s@%s.blob.core.windows.net" .Values.sparkEventLogStorage.logDirectory .Values.sparkEventLogStorage.azureStorageAccountName }} + {{- else if eq .Values.sparkEventLogStorage.cloudProvider "google"}} + {{- printf "gs://%s" .Values.sparkEventLogStorage.logDirectory }} + {{- else if eq .Values.sparkEventLogStorage.cloudProvider "oracle"}} + {{- printf "oci://%s@%s" .Values.sparkEventLogStorage.logDirectory .Values.sparkEventLogStorage.oracleNamespace }} + {{- end }} +{{- end }} +{{- $schema := .Values.sparkEventLogStorage.cloudProvider }} +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ template "spark-hs.fullname" . }} + labels: + app: {{ template "spark-hs.fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + app: {{ template "spark-hs.fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + spec: + initContainers: + - name: {{ .Chart.Name }}-init-config + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + envFrom: + - secretRef: + name: {{ default (printf "%s" (include "spark-hs.fullname" .)) .Values.sparkEventLogStorage.secretName }} + command: + - '/bin/sh' + - '-c' + - > + envsubst < /opt/spark/prepared_conf/spark-defaults.conf > /opt/spark/conf/spark-defaults.conf; + volumeMounts: + - name: config-volume + mountPath: /opt/spark/conf/ + - name: config-init-volume + mountPath: /opt/spark/prepared_conf/spark-defaults.conf + subPath: spark-defaults.conf + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - '/opt/spark/sbin/start-history-server.sh' + env: + - name: SPARK_NO_DAEMONIZE + value: "false" + - name: SPARK_HISTORY_OPTS + value: {{ printf "-Dspark.history.fs.logDirectory=%s" (include "logDirectoryTag" . )}} + volumeMounts: + - name: config-volume + mountPath: /opt/spark/conf/spark-defaults.conf + subPath: spark-defaults.conf + {{- if eq $schema "google" }} + - name: secret-volume + mountPath: /opt/spark/conf/secret/google.json + subPath: google.json + {{- else if eq $schema "oracle" }} + - name: secret-volume + mountPath: /opt/spark/conf/secret/api_key + subPath: api_key + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.internalPort }} + protocol: TCP + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + resources: +{{ toYaml .Values.resources | indent 12 }} + volumes: + - name: config-volume + emptyDir: {} + - name: config-init-volume + configMap: + name: {{ template "spark-hs.fullname" . }}-spark-hs-config + {{- if or (eq $schema "google") ((eq $schema "oracle")) }} + - name: secret-volume + secret: + secretName: {{ default (printf "%s" (include "spark-hs.fullname" .)) .Values.sparkEventLogStorage.secretName }} + {{- end }} diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/ingress.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/ingress.yaml new file mode 100644 index 0000000..6bbc4f2 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/ingress.yaml @@ -0,0 +1,39 @@ +{{- if .Values.ingress.enabled -}} +{{- $serviceName := include "spark-hs.fullname" . -}} +{{- $servicePort := .Values.service.externalPort -}} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ template "spark-hs.fullname" . }} + labels: + app: {{ template "spark-hs.name" . }} + chart: {{ template "spark-hs.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.ingress.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + {{- $url := splitList "/" . }} + - host: {{ first $url }} + http: + paths: + - path: /{{ rest $url | join "/" }} + backend: + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end}} +{{- end }} diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/secret.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/secret.yaml new file mode 100644 index 0000000..8e2504b --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/secret.yaml @@ -0,0 +1,34 @@ +{{- if eq .Values.sparkEventLogStorage.secretName "" }} +{{- $schema := .Values.sparkEventLogStorage.cloudProvider }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "spark-hs.fullname" . }} + labels: + app: {{ template "spark-hs.fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + {{- if eq $schema "amazon" }} + AWS_ACCESS_KEY_ID: {{ .Values.sparkEventLogStorage.awsAccessKeyId | b64enc | quote }} + AWS_SECRET_ACCESS_KEY: {{ .Values.sparkEventLogStorage.awsSecretAccessKey | b64enc | quote }} + {{- if ne .Values.sparkEventLogStorage.endpoint "" }} + ENDPOINT: {{ .Values.sparkEventLogStorage.endpoint | b64enc | quote }} + {{- end}} + {{- else if eq $schema "azure" }} + storageAccount: {{ .Values.sparkEventLogStorage.azureStorageAccountName | b64enc | quote }} + accessKey: {{ .Values.sparkEventLogStorage.azureStorageAccessKey | b64enc | quote }} + {{- else if eq $schema "alibaba" }} + ALIBABA_ACCESS_KEY_ID: {{ .Values.sparkEventLogStorage.aliAccessKeyId | b64enc | quote }} + ALIBABA_ACCESS_KEY_SECRET: {{ .Values.sparkEventLogStorage.aliSecretAccessKey | b64enc | quote }} + {{- else if eq $schema "google" }} + google.json: {{ .Values.sparkEventLogStorage.googleJson | quote }} + {{- else if eq $schema "oracle" }} + api_key: {{ .Values.sparkEventLogStorage.apiKey | b64enc | quote }} + tenancy_ocid: {{ .Values.sparkEventLogStorage.oracleTenancyId | b64enc | quote }} + user_ocid: {{ .Values.sparkEventLogStorage.oracleUserId | b64enc | quote }} + api_key_fingerprint: {{ .Values.sparkEventLogStorage.oracleApiKeyFingerprint | b64enc | quote }} + {{- end }} +{{- end }} diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/service.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/service.yaml new file mode 100644 index 0000000..5252056 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "spark-hs.fullname" . }} + labels: + app: {{ template "spark-hs.fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.externalPort }} + targetPort: {{ .Values.service.internalPort }} + protocol: TCP + name: {{ .Chart.Name }} + selector: + app: {{ template "spark-hs.fullname" . }} diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/spark-configmap.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/spark-configmap.yaml new file mode 100644 index 0000000..e1b6e62 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/templates/spark-configmap.yaml @@ -0,0 +1,38 @@ +{{- $schema := .Values.sparkEventLogStorage.cloudProvider }} +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ template "spark-hs.fullname" . }}-spark-hs-config +data: + spark-defaults.conf: |- + {{- if .Values.sparkEventLogStorage.logDirectory }} + + {{- if eq $schema "amazon" }} + spark.hadoop.fs.s3a.access.key=$AWS_ACCESS_KEY_ID + spark.hadoop.fs.s3a.secret.key=$AWS_SECRET_ACCESS_KEY + {{- if .Values.sparkEventLogStorage.endpoint }} + spark.hadoop.fs.s3a.endpoint=$ENDPOINT + {{- end}} + {{- else if eq $schema "alibaba" }} + spark.hadoop.fs.oss.accessKeyId=$ALIBABA_ACCESS_KEY_ID + spark.hadoop.fs.oss.accessKeySecret=$ALIBABA_ACCESS_KEY_SECRET + spark.hadoop.fs.oss.impl=org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem + spark.hadoop.fs.oss.endpoint={{ default (printf "oss-%s.aliyuncs.com" .Values.sparkEventLogStorage.aliOssRegion) .Values.sparkEventLogStorage.aliOssEndpoint }} + + {{- else if eq $schema "azure" }} + spark.hadoop.fs.azure.account.key.$storageAccount.blob.core.windows.net=$accessKey + + {{- else if eq $schema "google" }} + spark.hadoop.google.cloud.auth.service.account.enable=true + spark.hadoop.google.cloud.auth.service.account.json.keyfile=/opt/spark/conf/secret/google.json + + {{- else if eq $schema "oracle" }} + spark.hadoop.fs.oci.client.hostname={{ default (printf "https://objectstorage.%s.oraclecloud.com" .Values.sparkEventLogStorage.oracleRegion) .Values.sparkEventLogStorage.oracleHost }} + spark.hadoop.fs.oci.client.auth.tenantId=$tenancy_ocid + spark.hadoop.fs.oci.client.auth.userId=$user_ocid + spark.hadoop.fs.oci.client.auth.fingerprint=$api_key_fingerprint + spark.hadoop.fs.oci.client.auth.pemfilepath=/opt/spark/conf/secret/api_key + + {{- end }} + + {{- end }} diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/values.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/values.yaml new file mode 100644 index 0000000..3c6d310 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/charts/spark-hs/values.yaml @@ -0,0 +1,76 @@ +# Default values for spark-hs +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: banzaicloud/spark + tag: v2.3.2.146 + pullPolicy: IfNotPresent + +service: + externalPort: 80 + internalPort: 18080 + type: ClusterIP + +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: + cpu: 500m + memory: 1024Mi + + requests: + cpu: 500m + memory: 512Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +ingress: + enabled: false + annotations: {} + #kubernetes.io/ingress.class: traefik + #ingress.kubernetes.io/ssl-redirect: "false" + #traefik.frontend.rule.type: PathPrefix + hosts: + - "/" + # - "domain.com/xyz" + # - "domain.com" + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +sparkEventLogStorage: + cloudProvider: "" + logDirectory: "" + secretName: "" + + awsAccessKeyId: "" + awsSecretAccessKey: "" + + aliAccessKeyId: "" + aliSecretAccessKey: "" + + aliOssEndpoint: "" + aliOssRegion: "" + + azureStorageAccessKey: "" + azureStorageAccountName: "" + + googleJson: "" + + oracleRegion: "" + oracleHost: "" + apiKey: "" + oracleTenancyId: "" + oracleUserId: "" + oracleApiKeyFingerprint: "" + oracleNamespace: "" diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/requirements.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/requirements.yaml new file mode 100644 index 0000000..a4618a2 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/requirements.yaml @@ -0,0 +1,4 @@ +dependencies: +- name: spark-hs + version: 0.2.3 + condition: historyServer.enabled diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/templates/role.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/templates/role.yaml new file mode 100644 index 0000000..52698f4 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/templates/role.yaml @@ -0,0 +1,240 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: "{{ .Chart.Name }}-{{ .Release.Name }}" + labels: + app: "{{ .Chart.Name }}-{{ .Release.Name }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: + - "" + resources: + - pods/attach + - pods/exec + - pods/portforward + - pods/proxy + - secrets + - services/proxy + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - impersonate +- apiGroups: + - "" + resources: + - pods + - pods/attach + - pods/exec + - pods/portforward + - pods/proxy + verbs: + - create + - delete + - deletecollection + - patch + - update +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - persistentvolumeclaims + - replicationcontrollers + - replicationcontrollers/scale + - secrets + - serviceaccounts + - services + - services/proxy + verbs: + - create + - delete + - deletecollection + - patch + - update +- apiGroups: + - apps + resources: + - daemonsets + - deployments + - deployments/rollback + - deployments/scale + - replicasets + - replicasets/scale + - statefulsets + - statefulsets/scale + verbs: + - create + - delete + - deletecollection + - patch + - update +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - create + - delete + - deletecollection + - patch + - update +- apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - create + - delete + - deletecollection + - patch + - update +- apiGroups: + - extensions + resources: + - daemonsets + - deployments + - deployments/rollback + - deployments/scale + - ingresses + - networkpolicies + - replicasets + - replicasets/scale + - replicationcontrollers/scale + verbs: + - create + - delete + - deletecollection + - patch + - update +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - create + - delete + - deletecollection + - patch + - update +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - deletecollection + - patch + - update +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - persistentvolumeclaims + - pods + - replicationcontrollers + - replicationcontrollers/scale + - serviceaccounts + - services + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - bindings + - events + - limitranges + - namespaces/status + - pods/log + - pods/status + - replicationcontrollers/status + - resourcequotas + - resourcequotas/status + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - controllerrevisions + - daemonsets + - deployments + - deployments/scale + - replicasets + - replicasets/scale + - statefulsets + - statefulsets/scale + verbs: + - get + - list + - watch +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - get + - list + - watch +- apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - daemonsets + - deployments + - deployments/scale + - ingresses + - networkpolicies + - replicasets + - replicasets/scale + - replicationcontrollers/scale + verbs: + - get + - list + - watch +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/templates/rolebinding.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/templates/rolebinding.yaml new file mode 100644 index 0000000..8cba100 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/templates/rolebinding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: "{{ .Chart.Name }}-{{ .Release.Name }}" + labels: + app: "{{ .Chart.Name }}-{{ .Release.Name }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +subjects: + - kind: ServiceAccount + name: spark + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ .Chart.Name }}-{{ .Release.Name }} + apiGroup: rbac.authorization.k8s.io diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/templates/secret.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/templates/secret.yaml new file mode 100644 index 0000000..0f6b32c --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/templates/secret.yaml @@ -0,0 +1,19 @@ +{{ if .Values.monitoring.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ .Chart.Name }}-{{ .Release.Name }}" + labels: + app: "{{ .Chart.Name }}-{{ .Release.Name }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + {{ if .Values.monitoring.metricsProperties }} + metrics.properties: {{ .Values.monitoring.metricsProperties | b64enc | quote }} + {{ end }} + {{ if .Values.monitoring.jmxCollector }} + jmxCollector.yaml: {{ .Values.monitoring.jmxCollector | b64enc | quote }} + {{ end }} +{{ end }} \ No newline at end of file diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/templates/serviceaccount.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/templates/serviceaccount.yaml new file mode 100644 index 0000000..95e32bf --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: spark + namespace: {{ .Release.Namespace }} + labels: + spark-version: {{ .Values.spark.version }} + app: "{{ .Chart.Name }}-{{ .Release.Name }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/values.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/values.yaml new file mode 100644 index 0000000..e8dab9e --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/spark/values.yaml @@ -0,0 +1,34 @@ +# Default values for Spark +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +spark: + version: 2.2.0 + +ingress: + enabled: false + +pipelineIngress: + enabled: false + +historyServer: + enabled: false + +monitoring: + enabled: false + jmxCollector: | + lowercaseOutputName: false + lowercaseOutputLabelNames: false + whitelistObjectNames: ["*:*"] + metricsProperties: | + # Enable Prometheus for all instances by class name + *.sink.prometheus.class=com.banzaicloud.spark.metrics.sink.PrometheusSink + # Prometheus pushgateway address + *.sink.prometheus.pushgateway-address=prometheus-pushgateway:9091 + *.sink.prometheus.pushgateway-enable-timestamp=true + *.sink.prometheus.enable-dropwizard-collector=false + *.sink.prometheus.enable-jmx-collector=true + *.sink.prometheus.jmx-collector-config=/opt/spark/conf/monitoring/jmxCollector.yaml + *.sink.jmx.class=org.apache.spark.metrics.sink.JmxSink + # Enable JVM metrics source for all instances by class name + *.source.jvm.class=org.apache.spark.metrics.source.JvmSource diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/.helmignore b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/Chart.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/Chart.yaml new file mode 100644 index 0000000..804014f --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/Chart.yaml @@ -0,0 +1,12 @@ +name: zeppelin +version: 0.0.18 +description: A Helm chart for Kubernetes +home: https://banzaicloud.com +sources: + - https://github.com/banzaicloud/zeppelin +keywords: +- zeppelin +- spotguide +maintainers: +- name: Banzai Cloud + email: info@banzaicloud.com diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/NOTES.txt b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/NOTES.txt new file mode 100644 index 0000000..e69de29 diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/_helpers.tpl b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/_helpers.tpl new file mode 100644 index 0000000..308975b --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/_helpers.tpl @@ -0,0 +1,16 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} \ No newline at end of file diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/configmap.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/configmap.yaml new file mode 100644 index 0000000..0a04676 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/configmap.yaml @@ -0,0 +1,74 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ template "fullname" . }}-config +data: + log4j.properties: |- + log4j.rootLogger = INFO, stdout, {{- if .Values.logService.host }}, syslog{{- end }} + + log4j.appender.stdout = org.apache.log4j.ConsoleAppender + log4j.appender.stdout.Threshold = {{ .Values.logService.zeppelinLogLevel }} + log4j.appender.stdout.layout = org.apache.log4j.PatternLayout + log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n + + log4j.logger.org.apache.zeppelin.interpreter = {{ .Values.logService.zeppelinLogLevel }} + + {{- if .Values.logService.host }} + log4j.appender.syslog=org.apache.log4j.net.SyslogAppender + log4j.appender.syslog.threshold={{ .Values.logService.zeppelinLogLevel }} + log4j.appender.syslog.syslogHost={{ .Values.logService.host }}:{{ .Values.logService.zeppelinLogPort }} + log4j.appender.syslog.facility={{ .Values.logService.zeppelinFacility }} + log4j.appender.syslog.layout=org.apache.log4j.PatternLayout + log4j.appender.syslog.layout.conversionPattern={{ .Values.logService.zeppelinLogPattern }} + {{- end }} + + log4j_k8s_cluster.properties: |- + log4j.rootLogger = INFO, stdout{{- if .Values.logService.host }}, syslog{{- end }} + + log4j.appender.stdout = org.apache.log4j.ConsoleAppender + log4j.appender.stdout.layout = org.apache.log4j.PatternLayout + log4j.appender.stdout.layout.ConversionPattern=%5p XXX [%d] ({%t} %F[%M]:%L) - %m%n + + {{- if .Values.logService.host }} + log4j.appender.syslog=org.apache.log4j.net.SyslogAppender + log4j.appender.syslog.threshold={{ .Values.logService.sparkLogLevel }} + log4j.appender.syslog.syslogHost={{ .Values.logService.host }}:{{ .Values.logService.sparkLogPort }} + log4j.appender.syslog.facility={{ .Values.logService.sparkFacility }} + log4j.appender.syslog.layout=org.apache.log4j.PatternLayout + log4j.appender.syslog.layout.conversionPattern={{ .Values.logService.sparkLogPattern }} + + log4j.appender.{{ .Values.logService.applicationLoggerName }}=org.apache.log4j.net.SyslogAppender + log4j.appender.{{ .Values.logService.applicationLoggerName }}.threshold={{ .Values.logService.applicationLogLevel }} + log4j.appender.{{ .Values.logService.applicationLoggerName }}.syslogHost={{ .Values.logService.host }}:{{ .Values.logService.applicationLogPort }} + log4j.appender.{{ .Values.logService.applicationLoggerName }}.facility={{ .Values.logService.applicationFacility }} + log4j.appender.{{ .Values.logService.applicationLoggerName }}.layout=org.apache.log4j.PatternLayout + log4j.appender.{{ .Values.logService.applicationLoggerName }}.layout.conversionPattern={{ .Values.logService.applicationLogPattern }} + {{- end }} + + shiro.ini.template: |- + [users] + ADMIN_USER_NAME = ADMIN_PASSWORD, admin + + [main] + sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager + cookie = org.apache.shiro.web.servlet.SimpleCookie + cookie.name = JSESSIONID + cookie.httpOnly = true + sessionManager.sessionIdCookie = $cookie + securityManager.sessionManager = $sessionManager + securityManager.sessionManager.globalSessionTimeout = 86400000 + shiro.loginUrl = /api/login + passwordMatcher = org.apache.shiro.authc.credential.PasswordMatcher + iniRealm.credentialsMatcher = $passwordMatcher + + [roles] + admin = * + + [urls] + /api/version = anon + /api/interpreter/setting/restart/** = authc + /api/interpreter/** = authc, roles[admin] + /api/configurations/** = authc, roles[admin] + /api/credential/** = authc, roles[admin] + #/** = anon + /** = authc diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/deployment.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/deployment.yaml new file mode 100644 index 0000000..d2f1122 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/deployment.yaml @@ -0,0 +1,146 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ template "fullname" . }} + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + spec: + serviceAccountName: spark + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + volumes: + - name: config-init-volume + configMap: + name: {{ template "fullname" . }}-config + items: + - key: log4j.properties + path: log4j.properties + - key: log4j_k8s_cluster.properties + path: log4j_k8s_cluster.properties + - key: shiro.ini.template + path: shiro.ini.template + - name: zeppelin-site-config-volume + configMap: + name: {{ template "fullname" . }}-site-config + items: + - key: zeppelin-site.xml + path: zeppelin-site.xml + - name: interpreter-config-volume + configMap: + name: {{ template "fullname" . }}-interpreter-config + items: + - key: interpreter.json + path: interpreter.json + - name: spark-config-init-volume + configMap: + name: {{ template "fullname" . }}-spark-config + items: + - key: spark-defaults.conf + path: spark-defaults.conf + - name: config-volume + emptyDir: {} + - name: spark-config-volume + emptyDir: {} + initContainers: + - name: {{ .Chart.Name }}-init-config + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: ZEPPELIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ default (printf "%s-password" (include "fullname" .)) .Values.userCredentialSecretName }} + key: password + - name: ADMIN_USER_NAME + valueFrom: + secretKeyRef: + name: {{ default (printf "%s-password" (include "fullname" .)) .Values.userCredentialSecretName }} + key: username + {{- if .Values.sparkEventLogStorage.logDirectory }} + envFrom: + - secretRef: + name: {{ default (printf "%s-spark-eventlog-storage" (include "fullname" .)) .Values.sparkEventLogStorage.secretName }} + {{- end }} + command: + - '/bin/sh' + - '-c' + - > + cp conf/* prepared_conf; + export ADMIN_PASSWORD=$(java -jar /opt/zeppelin/bin/shiro-tools-hasher-1.3.2-cli.jar -gs -i 500000 -f shiro1 -a SHA-256 $ZEPPELIN_PASSWORD); + sed 's|ADMIN_USER_NAME|'$ADMIN_USER_NAME'|g;s|ADMIN_PASSWORD|'$ADMIN_PASSWORD'|g' /opt/zeppelin/prepared_conf/shiro.ini.template > /opt/zeppelin/prepared_conf/shiro.ini; + envsubst < /opt/spark/prepared_conf/spark-defaults.conf > /opt/spark/conf/spark-defaults.conf; + volumeMounts: + - name: config-init-volume + mountPath: /opt/zeppelin/conf/log4j.properties + subPath: log4j.properties + - name: config-init-volume + mountPath: /opt/zeppelin/conf/log4j_k8s_cluster.properties + subPath: log4j_k8s_cluster.properties + - name: config-init-volume + mountPath: /opt/zeppelin/conf/shiro.ini.template + subPath: shiro.ini.template + - name: spark-config-volume + mountPath: /opt/spark/conf + - name: zeppelin-site-config-volume + mountPath: /opt/zeppelin/conf/zeppelin-site.xml + subPath: zeppelin-site.xml + - name: interpreter-config-volume + mountPath: /opt/zeppelin/conf/interpreter.json + subPath: interpreter.json + - name: config-volume + mountPath: /opt/zeppelin/prepared_conf + - name: spark-config-init-volume + mountPath: /opt/spark/prepared_conf + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: ZEPPELIN_MEM + value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1 -Xms{{ .Values.resources.requests.memory }}M + ports: + - containerPort: {{ .Values.service.internalPort }} + livenessProbe: + httpGet: + path: {{ .Values.ingress.baseUrl }} + port: {{ .Values.service.internalPort }} + initialDelaySeconds: 30 + timeoutSeconds: 1 + readinessProbe: + httpGet: + path: {{ .Values.ingress.baseUrl }} + port: {{ .Values.service.internalPort }} + initialDelaySeconds: 20 + timeoutSeconds: 1 + volumeMounts: + - name: config-volume + mountPath: /opt/zeppelin/conf + - name: spark-config-volume + mountPath: /opt/spark/conf + resources: + limits: + cpu: {{ .Values.resources.limits.cpu }} + memory: {{ .Values.resources.limits.memory }}Mi + requests: + cpu: {{ .Values.resources.requests.cpu }} + memory: {{ .Values.resources.requests.memory }}Mi diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/ingress.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/ingress.yaml new file mode 100644 index 0000000..4156343 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/ingress.yaml @@ -0,0 +1,39 @@ +{{- if .Values.ingress.enabled -}} +{{- $serviceName := include "fullname" . -}} +{{- $servicePort := .Values.service.externalPort -}} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ template "fullname" . }} + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.ingress.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + {{- $url := splitList "/" . }} + - host: {{ first $url }} + http: + paths: + - path: /{{ rest $url | join "/" }} + backend: + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end}} +{{- end }} diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/interpreter-settings-config.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/interpreter-settings-config.yaml new file mode 100644 index 0000000..15bf5d1 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/interpreter-settings-config.yaml @@ -0,0 +1,605 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ template "fullname" . }}-interpreter-config +data: + interpreter.json: |- + { + "interpreterSettings": { + "2CYM9BXZH": { + "id": "2CYM9BXZH", + "name": "sh", + "group": "sh", + "properties": { + "zeppelin.shell.principal": { + "name": "zeppelin.shell.principal", + "value": "", + "type": "string" + }, + "shell.working.directory.user.home": { + "name": "shell.working.directory.user.home", + "value": false, + "type": "checkbox" + }, + "zeppelin.shell.auth.type": { + "name": "zeppelin.shell.auth.type", + "value": "", + "type": "string" + }, + "zeppelin.shell.keytab.location": { + "name": "zeppelin.shell.keytab.location", + "value": "", + "type": "string" + }, + "shell.command.timeout.millisecs": { + "name": "shell.command.timeout.millisecs", + "value": "60000", + "type": "number" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "sh", + "class": "org.apache.zeppelin.shell.ShellInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sh", + "editOnDblClick": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "2CXX5TW2H": { + "id": "2CXX5TW2H", + "name": "spark", + "group": "spark", + "properties": { + "spark.executor.memory": { + "name": "spark.executor.memory", + "value": "", + "type": "string" + }, + "zeppelin.spark.concurrentSQL": { + "name": "zeppelin.spark.concurrentSQL", + "value": false, + "type": "checkbox" + }, + "zeppelin.R.knitr": { + "name": "zeppelin.R.knitr", + "value": true, + "type": "checkbox" + }, + "zeppelin.R.cmd": { + "name": "zeppelin.R.cmd", + "value": "R", + "type": "string" + }, + "spark.app.name": { + "name": "spark.app.name", + "value": "zri", + "type": "string" + }, + "zeppelin.R.image.width": { + "name": "zeppelin.R.image.width", + "value": "100%", + "type": "number" + }, + "spark.submit.deployMode": { + "name": "spark.submit.deployMode", + "value": "cluster", + "type": "string" + }, + "spark.kubernetes.namespace": { + "name": "spark.kubernetes.namespace", + "value": "{{ .Values.sparkSubmitOptions.k8sNameSpace }}", + "type": "string" + }, + "spark.executor.instances": { + "name": "spark.executor.instances", + "value": "2", + "type": "number" + }, + "zeppelin.spark.importImplicit": { + "name": "zeppelin.spark.importImplicit", + "value": true, + "type": "checkbox" + }, + "zeppelin.dep.additionalRemoteRepository": { + "name": "zeppelin.dep.additionalRemoteRepository", + "value": "spark-packages,http://dl.bintray.com/spark-packages/maven,false;", + "type": "textarea" + }, + "zeppelin.spark.maxResult": { + "name": "zeppelin.spark.maxResult", + "value": "1000", + "type": "number" + }, + "master": { + "name": "master", + "value": "k8s://https://kubernetes.default.svc.cluster.local:443", + "type": "string" + }, + "zeppelin.pyspark.python": { + "name": "zeppelin.pyspark.python", + "value": "python", + "type": "string" + }, + "args": { + "name": "args", + "value": "", + "type": "string" + }, + "zeppelin.dep.localrepo": { + "name": "zeppelin.dep.localrepo", + "value": "local-repo", + "type": "string" + }, + "zeppelin.spark.sql.stacktrace": { + "name": "zeppelin.spark.sql.stacktrace", + "value": false, + "type": "checkbox" + }, + "zeppelin.spark.useHiveContext": { + "name": "zeppelin.spark.useHiveContext", + "value": true, + "type": "checkbox" + }, + "zeppelin.spark.unSupportedVersionCheck": { + "name": "zeppelin.spark.unSupportedVersionCheck", + "value": true, + "type": "checkbox" + }, + "zeppelin.R.render.options": { + "name": "zeppelin.R.render.options", + "value": "out.format \u003d \u0027html\u0027, comment \u003d NA, echo \u003d FALSE, results \u003d \u0027asis\u0027, message \u003d F, warning \u003d F", + "type": "textarea" + }, + "zeppelin.spark.printREPLOutput": { + "name": "zeppelin.spark.printREPLOutput", + "value": true, + "type": "checkbox" + }, + "spark.cores.max": { + "name": "spark.cores.max", + "value": "", + "type": "number" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "spark", + "class": "org.apache.zeppelin.spark.SparkInterpreter", + "defaultInterpreter": true, + "editor": { + "language": "scala" + } + }, + { + "name": "sql", + "class": "org.apache.zeppelin.spark.SparkSqlInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sql" + } + }, + { + "name": "dep", + "class": "org.apache.zeppelin.spark.DepInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "scala" + } + }, + { + "name": "pyspark", + "class": "org.apache.zeppelin.spark.PySparkInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "python" + } + }, + { + "name": "r", + "class": "org.apache.zeppelin.spark.SparkRInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "r" + } + }, + { + "name": "ipyspark", + "class": "org.apache.zeppelin.spark.IPySparkInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "python", + "editOnDblClick": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "perNote": "isolated", + "perUser": "", + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "2CYVXG82Y": { + "id": "2CYVXG82Y", + "name": "angular", + "group": "angular", + "properties": {}, + "status": "READY", + "interpreterGroup": [ + { + "name": "angular", + "class": "org.apache.zeppelin.angular.AngularInterpreter", + "defaultInterpreter": false, + "editor": { + "editOnDblClick": true + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "2D1B3T53H": { + "id": "2D1B3T53H", + "name": "groovy", + "group": "groovy", + "properties": { + "GROOVY_CLASSES": { + "name": "GROOVY_CLASSES", + "value": "", + "type": "textarea" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "groovy", + "class": "org.apache.zeppelin.groovy.GroovyInterpreter", + "defaultInterpreter": false + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "2D245VY96": { + "id": "2D245VY96", + "name": "file", + "group": "file", + "properties": { + "hdfs.url": { + "name": "hdfs.url", + "value": "http://localhost:50070/webhdfs/v1/", + "type": "url" + }, + "hdfs.user": { + "name": "hdfs.user", + "value": "hdfs", + "type": "string" + }, + "hdfs.maxlength": { + "name": "hdfs.maxlength", + "value": "1000", + "type": "number" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "hdfs", + "class": "org.apache.zeppelin.file.HDFSFileInterpreter", + "defaultInterpreter": false, + "editor": { + "editOnDblClick": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "2D1KRUTHV": { + "id": "2D1KRUTHV", + "name": "jdbc", + "group": "jdbc", + "properties": { + "default.url": { + "name": "default.url", + "value": "jdbc:postgresql://localhost:5432/", + "type": "string" + }, + "default.driver": { + "name": "default.driver", + "value": "org.postgresql.Driver", + "type": "string" + }, + "zeppelin.jdbc.principal": { + "name": "zeppelin.jdbc.principal", + "value": "", + "type": "string" + }, + "default.completer.ttlInSeconds": { + "name": "default.completer.ttlInSeconds", + "value": "120", + "type": "number" + }, + "default.password": { + "name": "default.password", + "value": "", + "type": "password" + }, + "default.completer.schemaFilters": { + "name": "default.completer.schemaFilters", + "value": "", + "type": "textarea" + }, + "default.splitQueries": { + "name": "default.splitQueries", + "value": false, + "type": "checkbox" + }, + "default.user": { + "name": "default.user", + "value": "gpadmin", + "type": "string" + }, + "zeppelin.jdbc.concurrent.max_connection": { + "name": "zeppelin.jdbc.concurrent.max_connection", + "value": "10", + "type": "number" + }, + "common.max_count": { + "name": "common.max_count", + "value": "1000", + "type": "number" + }, + "default.precode": { + "name": "default.precode", + "value": "", + "type": "textarea" + }, + "zeppelin.jdbc.auth.type": { + "name": "zeppelin.jdbc.auth.type", + "value": "", + "type": "string" + }, + "zeppelin.jdbc.concurrent.use": { + "name": "zeppelin.jdbc.concurrent.use", + "value": true, + "type": "checkbox" + }, + "zeppelin.jdbc.keytab.location": { + "name": "zeppelin.jdbc.keytab.location", + "value": "", + "type": "string" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "sql", + "class": "org.apache.zeppelin.jdbc.JDBCInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sql", + "editOnDblClick": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "2D2A8488A": { + "id": "2D2A8488A", + "name": "md", + "group": "md", + "properties": { + "markdown.parser.type": { + "name": "markdown.parser.type", + "value": "pegdown", + "type": "string" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "md", + "class": "org.apache.zeppelin.markdown.Markdown", + "defaultInterpreter": false, + "editor": { + "language": "markdown", + "editOnDblClick": true + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "2CXW23SU3": { + "id": "2CXW23SU3", + "name": "python", + "group": "python", + "properties": { + "zeppelin.python": { + "name": "zeppelin.python", + "value": "python", + "type": "string" + }, + "zeppelin.ipython.launch.timeout": { + "name": "zeppelin.ipython.launch.timeout", + "value": "30000", + "type": "number" + }, + "zeppelin.python.useIPython": { + "name": "zeppelin.python.useIPython", + "value": true, + "type": "checkbox" + }, + "zeppelin.python.maxResult": { + "name": "zeppelin.python.maxResult", + "value": "1000", + "type": "number" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "python", + "class": "org.apache.zeppelin.python.PythonInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "python", + "editOnDblClick": false + } + }, + { + "name": "ipython", + "class": "org.apache.zeppelin.python.IPythonInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "python", + "editOnDblClick": false, + "completionKey": "TAB" + } + }, + { + "name": "sql", + "class": "org.apache.zeppelin.python.PythonInterpreterPandasSql", + "defaultInterpreter": false, + "editor": { + "language": "sql", + "editOnDblClick": false, + "completionKey": "TAB" + } + }, + { + "name": "conda", + "class": "org.apache.zeppelin.python.PythonCondaInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sh", + "editOnDblClick": false + } + }, + { + "name": "docker", + "class": "org.apache.zeppelin.python.PythonDockerInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sh", + "editOnDblClick": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + } + }, + "interpreterBindings": { + "2CYJ77JJ3": [ + "2CXX5TW2H", + "2D2A8488A", + "2CYVXG82Y", + "2CYM9BXZH", + "2D245VY96", + "2CXW23SU3", + "2D1KRUTHV", + "2D1B3T53H" + ] + }, + "interpreterRepositories": [ + { + "id": "central", + "type": "default", + "url": "http://repo1.maven.org/maven2/", + "releasePolicy": { + "enabled": true, + "updatePolicy": "daily", + "checksumPolicy": "warn" + }, + "snapshotPolicy": { + "enabled": true, + "updatePolicy": "daily", + "checksumPolicy": "warn" + }, + "mirroredRepositories": [], + "repositoryManager": false + }, + { + "id": "local", + "type": "default", + "url": "file:///root/.m2/repository", + "releasePolicy": { + "enabled": true, + "updatePolicy": "daily", + "checksumPolicy": "warn" + }, + "snapshotPolicy": { + "enabled": true, + "updatePolicy": "daily", + "checksumPolicy": "warn" + }, + "mirroredRepositories": [], + "repositoryManager": false + } + ] + } diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/service.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/service.yaml new file mode 100644 index 0000000..0635d78 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: {{ template "fullname" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "fullname" . }} +spec: + ports: + - name: http + port: {{ .Values.service.externalPort }} + protocol: TCP + targetPort: {{ .Values.service.internalPort }} + - name: rpc + port: {{ .Values.service.rpcPort }} + protocol: TCP + targetPort: {{ .Values.service.rpcPort }} + selector: + app: {{ template "fullname" . }} + type: {{ .Values.service.type }} diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/spark-configmap.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/spark-configmap.yaml new file mode 100644 index 0000000..cfb79b1 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/spark-configmap.yaml @@ -0,0 +1,66 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ template "fullname" . }}-spark-config +data: + spark-defaults.conf: |- + spark.kubernetes.authenticate.driver.serviceAccountName=spark + spark.driver.cores={{ .Values.sparkSubmitOptions.sparkDriverCores }} + spark.kubernetes.driver.limit.cores={{ .Values.sparkSubmitOptions.sparkDriverLimitCores }} + spark.executor.cores={{ .Values.sparkSubmitOptions.sparkExecutorCores }} + spark.driver.memory={{ .Values.sparkSubmitOptions.sparkDriverMemory }} + spark.executor.memory={{ .Values.sparkSubmitOptions.sparkExecutorMemory }} + spark.kubernetes.container.image={{- printf "%s:%s" .Values.sparkSubmitOptions.sparkImage.name .Values.sparkSubmitOptions.sparkImage.tag }} + spark.metrics.conf=/opt/spark/conf/monitoring/metrics.properties + spark.local.dir={{ .Values.sparkSubmitOptions.sparkLocalDir }} + spark.kubernetes.driver.secrets.spark-{{ .Release.Name }}=/opt/spark/conf/monitoring + spark.kubernetes.executor.secrets.spark-{{ .Release.Name }}=/opt/spark/conf/monitoring + spark.kubernetes.driver.label.app={{ template "fullname" . }}-driver + spark.kubernetes.executor.label.app={{ template "fullname" . }}-executor + + {{- if .Values.sparkEventLogStorage.logDirectory }} + + spark.eventLog.enabled=true + {{- $schema := .Values.sparkEventLogStorage.cloudProvider }} + + {{- if eq $schema "amazon" }} + + spark.eventLog.dir={{- printf "s3a://%s//" .Values.sparkEventLogStorage.logDirectory }} + spark.hadoop.fs.s3a.access.key=$AWS_ACCESS_KEY_ID + spark.hadoop.fs.s3a.secret.key=$AWS_SECRET_ACCESS_KEY + + {{- else if eq $schema "alibaba" }} + + spark.eventLog.dir={{- printf "oss://%s//" .Values.sparkEventLogStorage.logDirectory }} + spark.hadoop.fs.oss.accessKeyId=$ALIBABA_ACCESS_KEY_ID + spark.hadoop.fs.oss.accessKeySecret=$ALIBABA_ACCESS_KEY_SECRET + spark.hadoop.fs.oss.impl=org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem + spark.hadoop.fs.oss.endpoint={{ default (printf "oss-%s.aliyuncs.com" .Values.sparkEventLogStorage.aliOssRegion) .Values.sparkEventLogStorage.aliOssEndpoint }} + + {{- else if eq $schema "azure" }} + + spark.eventLog.dir={{- printf "wasb://%s@$storageAccount.blob.core.windows.net" .Values.sparkEventLogStorage.logDirectory }} + spark.hadoop.fs.azure.account.key.$storageAccount.blob.core.windows.net=$accessKey + + {{- else if eq $schema "google" }} + + spark.eventLog.dir={{- printf "gs://%s" .Values.sparkEventLogStorage.logDirectory }} + spark.kubernetes.driver.secrets.{{ default (printf "%s" (include "fullname" .)) .Values.sparkEventLogStorage.secretName }}=/opt/spark/conf/secret + spark.kubernetes.executor.secrets.{{ default (printf "%s" (include "fullname" .)) .Values.sparkEventLogStorage.secretName }}=/opt/spark/conf/secret + spark.hadoop.google.cloud.auth.service.account.enable=true + spark.hadoop.google.cloud.auth.service.account.json.keyfile=/opt/spark/conf/secret/google.json + + {{- else if eq $schema "oracle" }} + + spark.eventLog.dir={{- printf "oci://%s@%s" .Values.sparkEventLogStorage.logDirectory .Values.sparkEventLogStorage.oracleNamespace }} + spark.hadoop.fs.oci.client.hostname={{ default (printf "https://objectstorage.%s.oraclecloud.com" .Values.sparkEventLogStorage.oracleRegion) .Values.sparkEventLogStorage.oracleHost }} + spark.hadoop.fs.oci.client.auth.tenantId=$tenancy_ocid + spark.hadoop.fs.oci.client.auth.userId=$user_ocid + spark.hadoop.fs.oci.client.auth.fingerprint=$api_key_fingerprint + spark.hadoop.fs.oci.client.auth.pemfilepath=/opt/spark/conf/secret/api_key + spark.kubernetes.driver.secrets.{{ default (printf "%s-spark-eventlog-storage" (include "fullname" .)) .Values.sparkEventLogStorage.secretName }}=/opt/spark/conf/secret + spark.kubernetes.executor.secrets.{{ default (printf "%s-spark-eventlog-storage" (include "fullname" .)) .Values.sparkEventLogStorage.secretName }}=/opt/spark/conf/secret + + {{- end }} + + {{- end }} diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/zeppelin-secret.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/zeppelin-secret.yaml new file mode 100644 index 0000000..d284413 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/zeppelin-secret.yaml @@ -0,0 +1,52 @@ +{{- if eq .Values.userCredentialSecretName "" }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "fullname" . }}-password + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + username: {{ .Values.username | b64enc | quote }} + password: {{ .Values.password | b64enc | quote }} +{{- end }} + +--- +{{- if .Values.sparkEventLogStorage.logDirectory }} + +{{- if eq .Values.sparkEventLogStorage.secretName "" }} +{{- $schema := .Values.sparkEventLogStorage.cloudProvider }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "fullname" . }}-spark-eventlog-storage + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + {{- if eq $schema "amazon" }} + AWS_ACCESS_KEY_ID: {{ .Values.sparkEventLogStorage.awsAccessKeyId | b64enc | quote }} + AWS_SECRET_ACCESS_KEY: {{ .Values.sparkEventLogStorage.awsSecretAccessKey | b64enc | quote }} + {{- else if eq $schema "alibaba" }} + ALIBABA_ACCESS_KEY_ID: {{ .Values.sparkEventLogStorage.aliAccessKeyId | b64enc | quote }} + ALIBABA_ACCESS_KEY_SECRET: {{ .Values.sparkEventLogStorage.aliSecretAccessKey | b64enc | quote }} + {{- else if eq $schema "azure" }} + storageAccount: {{ .Values.sparkEventLogStorage.azureStorageAccountName | b64enc | quote }} + accessKey: {{ .Values.sparkEventLogStorage.azureStorageAccessKey | b64enc | quote }} + {{- else if eq $schema "google" }} + google.json: {{ .Values.sparkEventLogStorage.googleJson | quote }} + {{- else if eq $schema "oracle" }} + api_key: {{ .Values.sparkEventLogStorage.apiKey | b64enc | quote }} + tenancy_ocid: {{ .Values.sparkEventLogStorage.oracleTenancyId | b64enc | quote }} + user_ocid: {{ .Values.sparkEventLogStorage.oracleUserId | b64enc | quote }} + api_key_fingerprint: {{ .Values.sparkEventLogStorage.oracleApiKeyFingerprint | b64enc | quote }} + {{- end }} +{{- end }} + +{{- end }} diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/zeppelin-site-config.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/zeppelin-site-config.yaml new file mode 100644 index 0000000..d7d7dfd --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/templates/zeppelin-site-config.yaml @@ -0,0 +1,384 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ template "fullname" . }}-site-config +data: + zeppelin-site.xml: |- + + + + + + + + zeppelin.server.addr + 0.0.0.0 + Server address + + + + zeppelin.server.rpc.hostName + {{ template "fullname" . }} + RPC server address + + + + zeppelin.server.rpc.portRange + {{ .Values.service.rpcPort }}:{{ .Values.service.rpcPort }} + RPC Server address port + + + + + zeppelin.server.port + {{ .Values.service.internalPort }} + Server port. + + + + zeppelin.server.ssl.port + 8443 + Server ssl port. (used when ssl property is set to true) + + + + zeppelin.server.context.path + {{- if .Values.ingress.enabled }}{{ .Values.ingress.baseUrl }}{{- end }} + Context Path of the Web Application + + + + zeppelin.war.tempdir + webapps + Location of jetty temporary directory + + + + zeppelin.notebook.dir + notebook + path or URI for notebook persist + + + + zeppelin.notebook.homescreen + + id of notebook to be displayed in homescreen. ex) 2A94M5J1Z Empty value displays default home screen + + + + zeppelin.notebook.homescreen.hide + false + hide homescreen notebook from list when this value set to true + + + {{- if eq .Values.notebookStorage.type "s3" }} + + zeppelin.notebook.s3.bucket + {{ .Values.notebookStorage.path }} + S3 bucket path + + + zeppelin.notebook.storage + org.apache.zeppelin.notebook.repo.S3NotebookRepo + versioned notebook persistence layer implementation + + {{- else if eq .Values.notebookStorage.type "gs" }} + + zeppelin.notebook.gcs.dir + gs://{{ .Values.notebookStorage.path }} + GCS bucket path + + + zeppelin.notebook.storage + org.apache.zeppelin.notebook.repo.GCSNotebookRepo + versioned notebook persistence layer implementation + + {{- else if eq .Values.notebookStorage.type "azure" }} + + zeppelin.notebook.azure.share + {{ .Values.notebookStorage.path }} + Azure FileShare name + + + zeppelin.notebook.storage + org.apache.zeppelin.notebook.repo.AzureNotebookRepo + versioned notebook persistence layer implementation + + + zeppelin.notebook.azure.connectionString + DefaultEndpointsProtocol=https;AccountName={{ .Values.azureStorageAccountName }};AccountKey={{ .Values.azureStorageAccessKey }} + Connection string to Azure File + + {{- else }} + + zeppelin.notebook.storage + org.apache.zeppelin.notebook.repo.GitNotebookRepo + versioned notebook persistence layer implementation + + {{- end }} + + + zeppelin.notebook.one.way.sync + false + If there are multiple notebook storages, should we treat the first one as the only source of truth? + + + + zeppelin.interpreter.dir + interpreter + Interpreter implementation base directory + + + + zeppelin.interpreter.localRepo + local-repo + Local repository for interpreter's additional dependency loading + + + + zeppelin.interpreter.dep.mvnRepo + http://repo1.maven.org/maven2/ + Remote principal repository for interpreter's additional dependency loading + + + + zeppelin.dep.localrepo + local-repo + Local repository for dependency loader + + + + zeppelin.helium.node.installer.url + https://nodejs.org/dist/ + Remote Node installer url for Helium dependency loader + + + + zeppelin.helium.npm.installer.url + http://registry.npmjs.org/ + Remote Npm installer url for Helium dependency loader + + + + zeppelin.helium.yarnpkg.installer.url + https://github.com/yarnpkg/yarn/releases/download/ + Remote Yarn package installer url for Helium dependency loader + + + + zeppelin.interpreters + org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter,org.apache.zeppelin.groovy.GroovyInterpreter + Comma separated interpreter configurations. First interpreter become a default + + + + zeppelin.interpreter.group.order + spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery,beam,groovy + + + + + zeppelin.interpreter.connect.timeout + {{ .Values.interpreterConnectTimeout }} + Interpreter process connect timeout in msec. + + + + zeppelin.interpreter.output.limit + 102400 + Output message from interpreter exceeding the limit will be truncated + + + + zeppelin.ssl + false + Should SSL be used by the servers? + + + + zeppelin.ssl.client.auth + false + Should client authentication be used for SSL connections? + + + + zeppelin.ssl.keystore.path + keystore + Path to keystore relative to Zeppelin configuration directory + + + + zeppelin.ssl.keystore.type + JKS + The format of the given keystore (e.g. JKS or PKCS12) + + + + zeppelin.ssl.keystore.password + change me + Keystore password. Can be obfuscated by the Jetty Password tool + + + + + + zeppelin.ssl.truststore.path + truststore + Path to truststore relative to Zeppelin configuration directory. Defaults to the keystore path + + + + zeppelin.ssl.truststore.type + JKS + The format of the given truststore (e.g. JKS or PKCS12). Defaults to the same type as the keystore type + + + + + + zeppelin.server.allowed.origins + * + Allowed sources for REST and WebSocket requests (i.e. http://onehost:8080,http://otherhost.com). If you leave * you are vulnerable to https://issues.apache.org/jira/browse/ZEPPELIN-173 + + + + zeppelin.anonymous.allowed + true + Anonymous user allowed by default + + + + zeppelin.notebook.default.owner.username + + Set owner role by default + + + + zeppelin.notebook.public + true + Make notebook public by default when created, private otherwise + + + + zeppelin.websocket.max.text.message.size + 1024000 + Size in characters of the maximum text message to be received by websocket. Defaults to 1024000 + + + + zeppelin.server.default.dir.allowed + false + Enable directory listings on server. + + + + zeppelin.interpreter.lifecyclemanager.class + org.apache.zeppelin.interpreter.lifecycle.TimeoutLifecycleManager + LifecycleManager class for managing the lifecycle of interpreters, by default interpreter will + be closed after timeout + + + + zeppelin.interpreter.lifecyclemanager.timeout.checkinterval + 60000 + Milliseconds of the interval to checking whether interpreter is time out + + + + zeppelin.interpreter.lifecyclemanager.timeout.threshold + 3600000 + Milliseconds of the interpreter timeout threshold, by default it is 1 hour + + + + + + + + + + + + + + + + + + + diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/values.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/values.yaml new file mode 100755 index 0000000..be53a2b --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/charts/zeppelin/values.yaml @@ -0,0 +1,110 @@ +# Default values for zeppelin +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. +replicaCount: 1 + +image: + repository: banzaicloud/zeppelin-server + tag: v0.9.0-k8s-1.0.76 + pullPolicy: IfNotPresent + +resources: + limits: + cpu: 2 + memory: 2048 + requests: + cpu: 500m + memory: 1024 + +additionalOptions: + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +service: + type: ClusterIP + internalPort: 8080 + externalPort: 8080 + rpcPort: 38853 + +userCredentialSecretName: "" +username: "admin" +password: "zeppelin" + +interpreterConnectTimeout: 120000 + +ingress: + baseUrl: "/zeppelin" + enabled: false + annotations: {} + #kubernetes.io/ingress.class: traefik + #ingress.kubernetes.io/ssl-redirect: "false" + #traefik.frontend.rule.type: PathPrefix + hosts: + - "/" + # - "domain.com/xyz" + # - "domain.com" + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +logService: + zeppelinLogLevel: DEBUG + zeppelinFacility: LOCAL4 + zeppelinLogPattern: "%5p [%d] ({%t} %F[%M]:%L) - %m%n" + sparkLogLevel: INFO + sparkFacility: LOCAL4 + sparkLogPattern: "[%p] XXX %c:%L - %m%n" + applicationLoggerName: application + applicationLogLevel: INFO + applicationFacility: LOCAL4 + applicationLogPattern: "[%p] XXX %c:%L - %m%n" + +notebookStorage: + type: "" + path: "" +azureStorageAccountName: "" +azureStorageAccessKey: "" + +sparkSubmitOptions: + sparkImage: + name: banzaicloud/zeppelin-spark + tag: v0.9.0-k8s-1.0.76 + k8sNameSpace: default + sparkDriverCores: 1 + sparkDriverLimitCores: 1 + sparkExecutorCores: 1 + sparkDriverMemory: 1G + sparkExecutorMemory: 1G + dynamicAllocation: false + shuffleService: false + driverServiceAccountName: "spark" + sparkLocalDir: /tmp/spark-local + +sparkEventLogStorage: + cloudProvider: "" + logDirectory: "" + secretName: "" + + awsAccessKeyId: "" + awsSecretAccessKey: "" + + aliOssEndpoint: "" + aliOssRegion: "" + aliAccessKeyId: "" + aliSecretAccessKey: "" + + azureStorageAccessKey: "" + azureStorageAccountName: "" + + googleJson: "" + + oracleRegion: "" + oracleHost: "" + oracleTenancyId: "" + oracleUserId: "" + oracleApiKeyFingerprint: "" diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/requirements.yaml b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/requirements.yaml new file mode 100644 index 0000000..d154f69 --- /dev/null +++ b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/requirements.yaml @@ -0,0 +1,6 @@ +dependencies: +- name: spark + version: 0.0.19 + +- name: zeppelin + version: 0.0.18 diff --git a/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/templates/.gitkeep b/.banzaicloud/charts/spotguide-zeppelin/charts/zeppelin-spark/templates/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/.banzaicloud/charts/spotguide-zeppelin/requirements.yaml b/.banzaicloud/charts/spotguide-zeppelin/requirements.yaml index 4322fb0..4710a44 100644 --- a/.banzaicloud/charts/spotguide-zeppelin/requirements.yaml +++ b/.banzaicloud/charts/spotguide-zeppelin/requirements.yaml @@ -1,4 +1,3 @@ dependencies: - name: zeppelin-spark - repository: https://kubernetes-charts.banzaicloud.com version: 0.0.24 diff --git a/.banzaicloud/charts/spotguide-zeppelin/templates/NOTES.txt b/.banzaicloud/charts/spotguide-zeppelin/templates/NOTES.txt index 0301b28..07a3430 100644 --- a/.banzaicloud/charts/spotguide-zeppelin/templates/NOTES.txt +++ b/.banzaicloud/charts/spotguide-zeppelin/templates/NOTES.txt @@ -2,12 +2,27 @@ Congratulations, you have deployed Zeppelin Spotguide to Kubernetes! Your releas ### Zeppelin -To access the zeppelin ui: - {{ $hosts := index .Values "zeppelin-spark" "zeppelin" "ingress" "hosts" }} + +{{ if (has "/zeppelin" $hosts) }} +Get Zeppelin Web UI URL by running this command: + +{{- if eq .Values.banzaicloud.cluster.cloud "amazon" }} +``` +echo https://$(kubectl get ing --namespace {{ .Release.Namespace }} -l app={{ .Release.Name }},release={{ .Release.Name }} -o jsonpath='{.items[0].status.loadBalancer.ingress[0].hostname}')/zeppelin +``` +{{- else}} +``` +echo https://$(kubectl get ing --namespace {{ .Release.Namespace }} -l app={{ .Release.Name }},release={{ .Release.Name }} -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')/zeppelin +``` +{{- end }} + +{{ else }} +Zeppelin Web UI: {{ range $hosts }} - [{{ . }}](https://{{ . }}) -{{- end }} +{{- end}} +{{- end}} - [User secret]({{ $.Values.banzaicloud.organization.name }}/secret?name={{ .Values.banzaicloud.secret.zeppelin }}) @@ -15,25 +30,40 @@ To access the zeppelin ui: Spark logs are stored in the provided bucket for *Spark History Server* after your Spark context is closed. You can close the session by restarting 'spark' interpreter group on *Zeppelin* interpreter settings page here: -{{- end }} - -{{ $hosts := index .Values "zeppelin-spark" "zeppelin" "ingress" "hosts" }} +{{ if (has "/zeppelin" $hosts) }} +https://Zeppelin_Web_UI_URL/interpreter +{{- else }} {{ range $hosts }} - [{{ . }}/#/interpreter](https://{{ . }}/#/interpreter) {{- end }} - -{{- if .Values.historyServer.enabled }} +{{- end }} In case you already opened *Spark History Server*, you need to refresh browser page to reload completed application logs. ### Spark History Server -To access logs created by your spark job, we are using spark history server which can be accessed from here: +You can view your event logs of your spark job, in Spark History Server: {{ $hosts := index .Values "zeppelin-spark" "spark" "spark-hs" "ingress" "hosts" }} + +{{ if not (has "/" $hosts) }} {{ range $hosts }} - [{{ . }}](https://{{ . }}) {{- end }} +{{ else }} + +Get the application URL by running this command: + +{{- if eq .Values.banzaicloud.cluster.cloud "amazon" }} +``` +echo https://$(kubectl get ing --namespace {{ .Release.Namespace }} -l app=spark-hs,release={{ .Release.Name }} -o jsonpath='{.items[0].status.loadBalancer.ingress[0].hostname}') +``` +{{- else}} +``` +echo https://$(kubectl get ing --namespace {{ .Release.Namespace }} -l app=spark-hs,release={{ .Release.Name }} -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') +``` +{{- end}} +{{- end}} - [User secret]({{ $.Values.banzaicloud.organization.name }}/secret?name={{ .Values.banzaicloud.secret.historyServer }}) {{- end}} diff --git a/.banzaicloud/pipeline.yaml b/.banzaicloud/pipeline.yaml index 3d813a3..5c1e9a7 100644 --- a/.banzaicloud/pipeline.yaml +++ b/.banzaicloud/pipeline.yaml @@ -37,8 +37,7 @@ pipeline: image: lachlanevenson/k8s-helm:latest commands: - helm init -c - - helm repo add banzaicloud-stable https://kubernetes-charts.banzaicloud.com - - helm package -u ./.banzaicloud/charts/spotguide-zeppelin + - helm package ./.banzaicloud/charts/spotguide-zeppelin deploy_application: image: banzaicloud/ci-pipeline-client:0.11 @@ -55,11 +54,25 @@ pipeline: enabled: zeppelin-spark: zeppelin: + {{{{- if eq .pipeline.deploy_application.deployment.values.banzaicloud.spark.version "2.3.2" }}}} + image: + tag: v0.9.0-k8s-1.0.74 + sparkSubmitOptions: + sparkImage: + tag: v0.9.0-k8s-1.0.74 + {{{{- else if eq .pipeline.deploy_application.deployment.values.banzaicloud.spark.version "2.4.3" }}}} + image: + tag: v0.9.0-k8s-1.0.79 + sparkSubmitOptions: + sparkImage: + tag: v0.9.0-k8s-1.0.79 + {{{{- end }}}} userCredentialSecretName: '{{ .CICD_REPO_NAME }}-zeppelin-auth' sparkEventLogStorage: secretName: '{{ .CICD_REPO_NAME }}-hs-bucket' ingress: enabled: true + {{{{- if .platform.AutoDNSEnabled }}}} baseUrl: annotations: kubernetes.io/ingress.class: traefik @@ -67,12 +80,27 @@ pipeline: ingress.kubernetes.io/ssl-redirect: 'true' hosts: - 'zeppelin-{{ printf "%s-%s" .CICD_REPO_NAME .CLUSTER_NAME | sha1sum | trunc 10}}.{{.ORG_NAME}}.{{.DOMAIN_NAME}}' + {{{{- else }}}} + baseUrl: /zeppelin + annotations: + kubernetes.io/ingress.class: traefik + traefik.frontend.rule.type: PathPrefix + ingress.kubernetes.io/ssl-redirect: 'true' + hosts: + - '/zeppelin' + {{{{- end }}}} spark: monitoring: enabled: jmxCollector: metricsProperties: spark-hs: + image: + {{{{- if eq .pipeline.deploy_application.deployment.values.banzaicloud.spark.version "2.3.2" }}}} + tag: v2.3.2.146 + {{{{- else if eq .pipeline.deploy_application.deployment.values.banzaicloud.spark.version "2.4.3" }}}} + tag: v2.4.3.198 + {{{{- end }}}} sparkEventLogStorage: secretName: '{{ .CICD_REPO_NAME }}-hs-bucket' ingress: @@ -83,5 +111,7 @@ pipeline: traefik.frontend.rule.type: PathPrefixStrip traefik.ingress.kubernetes.io/auth-type: basic ingress.kubernetes.io/ssl-redirect: 'true' + {{{{- if .platform.AutoDNSEnabled }}}} hosts: - 'shs-{{ printf "%s-%s" .CICD_REPO_NAME .CLUSTER_NAME | sha1sum | trunc 10}}.{{.ORG_NAME}}.{{.DOMAIN_NAME}}' + {{{{- end }}}} diff --git a/.banzaicloud/spotguide.yaml b/.banzaicloud/spotguide.yaml index e3dc924..d785812 100644 --- a/.banzaicloud/spotguide.yaml +++ b/.banzaicloud/spotguide.yaml @@ -10,20 +10,83 @@ resources: minNodes: 1 maxNodes: 2 questions: +- type: pipeline + dataType: string + label: 'Namespace' + required: true + targets: + - deploy_application.deployment.namespace + - install_zeppelin_auth_secrets.clusterSecret.namespace + - install_history_server_bucket_secrets.clusterSecret.namespace + - install_history_server_auth_secrets.clusterSecret.namespace + - deploy_application.deployment.values.zeppelin-spark.zeppelin.sparkSubmitOptions.k8sNameSpace - type: secret dataType: password name: zeppelinpass required: true - group: Zeppelin - label: 'Zeppelin server auth creds' + label: 'Zeppelin server auth credentials' targets: - name: install_zeppelin_auth_secrets.clusterSecret.sourceSecretName - name: deploy_application.deployment.values.banzaicloud.secret.zeppelin +- type: pipeline + dataType: enum + key: spark_version_enum + label: 'Spark Version' + description: '' + default: '2.4.3' + required: true + options: + - '2.3.2' + - '2.4.3' + targets: + - deploy_application.deployment.values.banzaicloud.spark.version +- type: pipeline + dataType: boolean + default: true + label: 'Spark History Server' + required: true + key: historyServer + targets: + - deploy_application.deployment.values.historyServer.enabled +- type: secret + dataType: htpasswd + name: historyserverpass + label: 'History Server Auth Creds' + required: true + showIf: + properties: + historyServer: + const: true + targets: + - name: install_history_server_auth_secrets.clusterSecret.sourceSecretName + - name: deploy_application.deployment.values.banzaicloud.secret.historyServer +- type: bucket + name: eventBucket + label: 'Spark Event Log Bucket Name' + description: 'This bucket will be used to store spark logs.' + required: true + showIf: + properties: + historyServer: + const: true + targets: + - name: deploy_application.deployment.values.zeppelin-spark.spark.spark-hs.sparkEventLogStorage.logDirectory + - name: deploy_application.deployment.values.zeppelin-spark.zeppelin.sparkEventLogStorage.logDirectory + - cloud: deploy_application.deployment.values.zeppelin-spark.spark.spark-hs.sparkEventLogStorage.cloudProvider + - cloud: deploy_application.deployment.values.zeppelin-spark.zeppelin.sparkEventLogStorage.cloudProvider + - location: deploy_application.deployment.values.zeppelin-spark.spark.spark-hs.sparkEventLogStorage.aliOssRegion + - location: deploy_application.deployment.values.zeppelin-spark.spark.spark-hs.sparkEventLogStorage.oracleRegion + - location: deploy_application.deployment.values.zeppelin-spark.zeppelin.sparkEventLogStorage.aliOssRegion + - location: deploy_application.deployment.values.zeppelin-spark.zeppelin.sparkEventLogStorage.oracleRegion + - secret.accessName: install_history_server_bucket_secrets.clusterSecret.sourceSecretName + - secret.accessName: deploy_application.deployment.values.zeppelin-spark.spark.spark-hs.sparkEventLogStorage.pipelineSecretName + - aks.storageAccount: deploy_application.deployment.values.zeppelin-spark.spark.spark-hs.sparkEventLogStorage.azureStorageAccountName + - oracle.namespace: deploy_application.deployment.values.zeppelin-spark.zeppelin.sparkEventLogStorage.oracleNamespace + - oracle.namespace: deploy_application.deployment.values.zeppelin-spark.spark.spark-hs.sparkEventLogStorage.oracleNamespace - type: pipeline dataType: boolean - label: "Monitoring" + label: "Spark Monitoring" required: true - group: Monitoring key: monitoring default: true targets: @@ -47,29 +110,12 @@ questions: *.sink.jmx.class=org.apache.spark.metrics.sink.JmxSink # Enable JVM metrics source for all instances by class name *.source.jvm.class=org.apache.spark.metrics.source.JvmSource - group: Monitoring showIf: properties: monitoring: const: true targets: - deploy_application.deployment.values.zeppelin-spark.spark.monitoring.metricsProperties -- type: pipeline - dataType: enum - key: spark_version_enum - label: 'Spark Version' - description: '' - default: '2.3.2' - required: true - group: Spark - options: - - label: '2.3.2' - value: v0.9.0-k8s-1.0.74 - - label: '2.4.0' - value: v0.9.0-k8s-1.0.76 - targets: - - deploy_application.deployment.values.zeppelin-spark.zeppelin.image.tag - - deploy_application.deployment.values.zeppelin-spark.zeppelin.sparkSubmitOptions.sparkImage.tag - type: pipeline dataType: string controlType: code @@ -81,49 +127,9 @@ questions: lowercaseOutputName: false lowercaseOutputLabelNames: false whitelistObjectNames: ["*:*"] - group: Monitoring showIf: properties: monitoring: const: true targets: - deploy_application.deployment.values.zeppelin-spark.spark.monitoring.jmxCollector -- type: pipeline - dataType: boolean - default: true - label: 'History Server' - required: true - key: historyServer - group: History Server - targets: - - deploy_application.deployment.values.historyServer.enabled -- type: secret - dataType: htpasswd - name: historyserverpass - group: History Server - label: 'History Server Auth Creds' - required: true - showIf: - properties: - historyServer: - const: true - targets: - - name: install_history_server_auth_secrets.clusterSecret.sourceSecretName - - name: deploy_application.deployment.values.banzaicloud.secret.historyServer -- type: bucket - name: eventBucket - label: 'Bucket Name' - description: 'This bucket will be used to store spark logs.' - required: true - group: History Server - showIf: - properties: - historyServer: - const: true - targets: - - name: deploy_application.deployment.values.zeppelin-spark.spark.spark-hs.sparkEventLogStorage.logDirectory - - cloud: deploy_application.deployment.values.zeppelin-spark.spark.spark-hs.sparkEventLogStorage.cloudProvider - - secret.accessName: install_history_server_bucket_secrets.clusterSecret.sourceSecretName - - cloud: deploy_application.deployment.values.zeppelin-spark.zeppelin.sparkEventLogStorage.cloudProvider - - name: deploy_application.deployment.values.zeppelin-spark.zeppelin.sparkEventLogStorage.logDirectory - - aks.storageAccount: deploy_application.deployment.values.zeppelin-spark.spark.spark-hs.sparkEventLogStorage.azureStorageAccountName