Search Validation (#130)

This PR validates Search capability.
Azure · Jul 23, 2024 · ddca75a · ddca75a
1 parent df9b1e7
commit ddca75a
Show file tree

Hide file tree

Showing 31 changed files with 1,294 additions and 506 deletions.
diff --git a/bicep/modules/blade_common.bicep b/bicep/modules/blade_common.bicep
@@ -312,7 +312,7 @@ module configStorage './storage-account/main.bicep' = {
     keyVaultName: keyvault.outputs.name
     storageAccountSecretName: 'tbl-storage'
     storageAccountKeySecretName: 'tbl-storage-key'
-    storageAccountEndpointSecretName: 'tbl-storage-endpoint'
+    storageAccountTableEndpointSecretName: 'tbl-storage-endpoint'
 
     // Use as System Storage Account
     isSystem: true

diff --git a/bicep/modules/blade_partition.bicep b/bicep/modules/blade_partition.bicep
@@ -72,6 +72,7 @@ var partitionLayerConfig = {
   secrets: {
     storageAccountName: 'storage'
     storageAccountKey: 'storage-key'
+    storageAccountBlob: 'storage-account-blob-endpoint'
     cosmosConnectionString: 'cosmos-connection'
     cosmosEndpoint: 'cosmos-endpoint'
     cosmosPrimaryKey: 'cosmos-primary-key'
@@ -469,6 +470,7 @@ module partitionStorage './storage-account/main.bicep' = [for (partition, index)
     keyVaultName: kvName
     storageAccountSecretName: '${partition.name}-${partitionLayerConfig.secrets.storageAccountName}'
     storageAccountKeySecretName: '${partition.name}-${partitionLayerConfig.secrets.storageAccountKey}'
+    storageAccountBlobEndpointSecretName: '${partition.name}-${partitionLayerConfig.secrets.storageAccountBlob}'
   }
 }]
 

diff --git a/bicep/modules/keyvault_secrets_partition.bicep b/bicep/modules/keyvault_secrets_partition.bicep
@@ -27,6 +27,15 @@ resource serviceBusConnection 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = i
   }
 }
 
+resource serviceBusNamespace 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = if (serviceBusName != 'null') {
+  name: '${partitionName}-sb-namespace'
+  parent: keyVault
+
+  properties: {
+    value: serviceBus.name
+  }
+}
+
 resource elasticEndpoint 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = {
   name: '${partitionName}-elastic-endpoint'
   parent: keyVault
@@ -54,4 +63,13 @@ resource elasticUserPassword 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = {
   }
 }
 
+resource elasticKey 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = {
+  name: '${partitionName}-elastic-key'
+  parent: keyVault
+
+  properties: {
+    value: uniqueString(keyVault.id, partitionName, subscription().id, resourceGroup().id)
+  }
+}
+
 output keyVaultName string = keyVault.name
diff --git a/bicep/modules/storage-account/main.bicep b/bicep/modules/storage-account/main.bicep
@@ -447,8 +447,11 @@ param storageAccountSecretName string = ''
 @description('Optional: To save storage account key into vault set the secret name.')
 param storageAccountKeySecretName string = ''
 
-@description('Optional: To save storage account endpoint into vault set the secret name.')
-param storageAccountEndpointSecretName string = ''
+@description('Optional: To save storage account table endpoint into vault set the secret name.')
+param storageAccountTableEndpointSecretName string = ''
+
+@description('Optional: To save storage account blob endpoint into vault set the secret name.')
+param storageAccountBlobEndpointSecretName string = ''
 
 @description('Optional: To save storage account connectionstring into vault set the secret name.')
 param storageAccountConnectionString string = ''
@@ -526,15 +529,24 @@ module secretStorageAccountKey '.bicep/keyvault_secrets.bicep' =  if (!empty(key
 }
 
 
-module secretStorageAccountEndpoint '.bicep/keyvault_secrets.bicep' =  if (!empty(keyVaultName) && !empty(storageAccountEndpointSecretName)) {
-  name: '${deployment().name}-secret-endpoint'
+module secretStorageAccountTableEndpoint '.bicep/keyvault_secrets.bicep' =  if (!empty(keyVaultName) && !empty(storageAccountTableEndpointSecretName)) {
+  name: '${deployment().name}-secret-table-endpoint'
   params: {
     keyVaultName: keyVaultName
-    name: storageAccountEndpointSecretName
+    name: storageAccountTableEndpointSecretName
     value: storage.properties.primaryEndpoints.table
   }
 }
 
+module secretStorageAccountBlobEndpoint '.bicep/keyvault_secrets.bicep' =  if (!empty(keyVaultName) && !empty(storageAccountBlobEndpointSecretName)) {
+  name: '${deployment().name}-secret-blob-endpoint'
+  params: {
+    keyVaultName: keyVaultName
+    name: storageAccountBlobEndpointSecretName
+    value: storage.properties.primaryEndpoints.blob
+  }
+}
+
 module secretStorageAccountConnection '.bicep/keyvault_secrets.bicep' =  if (!empty(keyVaultName) && !empty(storageAccountConnectionString)) {
   name: '${deployment().name}-secret-connectionstring'
   params: {

diff --git a/charts/osdu-developer-init/templates/elastic-init.yaml b/charts/osdu-developer-init/templates/elastic-init.yaml
@@ -6,11 +6,16 @@ kind: Job
 metadata:
   name: init-elastic
   namespace: {{ $namespace }}
+  annotations:
+    sidecar.istio.io/inject: "false"  # Disable Istio sidecar injection
 spec:
   ttlSecondsAfterFinished: 120
   activeDeadlineSeconds: 600 # Timeout set to 10 minutes
   backoffLimit: 2            # Job will fail after 2 unsuccessful retries
   template:
+    metadata:
+      annotations:
+        sidecar.istio.io/inject: "false"  # Disable Istio sidecar injection
     spec:
       serviceAccountName: workload-identity-sa  # Specify the service account here
       initContainers:

diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml
@@ -93,8 +93,8 @@ data:
               "value": "sb-connection"
           },
           "sb-namespace": {
-              "sensitive": false,
-              "value": "{{ .Values.SERVICE_BUS_NAME }}"
+              "sensitive": true,
+              "value": "sb-namespace"
           },
           "storage-account-key": {
               "sensitive": true,

diff --git a/software/applications/elastic-search/elastic-job.yaml b/software/applications/elastic-search/elastic-job.yaml
@@ -0,0 +1,28 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: elastic-search-init
+  namespace: elastic-search
+  annotations:
+    clusterconfig.azure.com/use-managed-source: "true"
+spec:
+  targetNamespace: elastic-search
+  chart:
+    spec:
+      chart: ./charts/osdu-developer-init
+      sourceRef:
+        kind: GitRepository
+        name: flux-system
+        namespace: flux-system
+  interval: 5m0s
+  install:
+    remediation:
+      retries: 3
+  values:
+    jobs:
+      partitionInit: false
+      entitlementInit: false
+      userInit: false
+      schemaInit: false
+      elasticInit: true
diff --git a/software/applications/elastic-search/elastic-search.yaml b/software/applications/elastic-search/elastic-search.yaml
@@ -6,7 +6,14 @@ metadata:
   annotations:
     eck.k8s.elastic.co/downward-node-labels: "topology.kubernetes.io/zone"
 spec:
-  version: 8.5.3
+  version: 7.17.22
+  http:
+    tls:
+      selfSignedCertificate:
+        disabled: true
+    service:
+      spec:
+        type: ClusterIP
   nodeSets:
     - name: default
       count: 3
@@ -77,58 +84,4 @@ spec:
                 matchLabels:
                   elasticsearch.k8s.elastic.co/cluster-name: elasticsearch
                   elasticsearch.k8s.elastic.co/statefulset-name: elasticsearch-es-default
-  http:
-    tls:
-      selfSignedCertificate:
-        disabled: true
-    service:
-      spec:
-        type: ClusterIP
----
-apiVersion: kibana.k8s.elastic.co/v1
-kind: Kibana
-metadata:
-  name: kibana
-  namespace: elastic-search
-spec:
-  version: 8.5.3
-  http:
-    tls:
-      selfSignedCertificate:
-        disabled: true
-  elasticsearchRef:
-    name: "elasticsearch"
-  count: 3
-  podTemplate:
-    spec:
-      tolerations:
-        - effect: NoSchedule
-          key: app
-          value: "cluster"
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: agentpool
-                    operator: In
-                    values:
-                      - poolz1
-                      - poolz2
-                      - poolz3
-              - matchExpressions:
-                  - key: topology.kubernetes.io/zone
-                    operator: In
-                    values:
-                      - "$(REGION)-1"
-                      - "$(REGION)-2"
-                      - "$(REGION)-3"
-      containers:
-        - name: kibana
-          env:
-            - name: SERVER_BASEPATH
-              value: "/kibana" 
-            - name: REGION
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.labels['topology.kubernetes.io/region']
+
diff --git a/software/applications/elastic-search/ingress.yaml b/software/applications/elastic-search/ingress.yaml
diff --git a/software/applications/elastic-search/kibana.yaml b/software/applications/elastic-search/kibana.yaml
@@ -0,0 +1,49 @@
+---
+apiVersion: kibana.k8s.elastic.co/v1
+kind: Kibana
+metadata:
+  name: kibana
+  namespace: elastic-search
+spec:
+  version: 7.17.22
+  elasticsearchRef:
+    name: "elasticsearch"
+  count: 3
+  podTemplate:
+    spec:
+      tolerations:
+        - effect: NoSchedule
+          key: app
+          value: "cluster"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: agentpool
+                    operator: In
+                    values:
+                      - poolz1
+                      - poolz2
+                      - poolz3
+              - matchExpressions:
+                  - key: topology.kubernetes.io/zone
+                    operator: In
+                    values:
+                      - "$(REGION)-1"
+                      - "$(REGION)-2"
+                      - "$(REGION)-3"
+      containers:
+        - name: kibana
+          env:
+            - name: xpack.encryptedSavedObjects.encryptionKey
+              valueFrom:
+                secretKeyRef:
+                  name: elasticsearch-credentials
+                  key: key
+            - name: ELASTICSEARCH_HOSTS
+              value: "http://elasticsearch-es-http.elastic-search:9200"  # Adjust this URL to match your 
+            - name: REGION
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.labels['topology.kubernetes.io/region']
diff --git a/software/applications/elastic-search/namespace.yaml b/software/applications/elastic-search/namespace.yaml
@@ -2,3 +2,5 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: elastic-search
+  labels:
+    toolkit.fluxcd.io/tenant: dev-team