New version of opensearch in automate

.studio/common

@@ -362,6 +362,7 @@ function build_all_changed_components() {
   hab pkg install core/coreutils >/dev/null
   mkdir -p /usr/bin
   ln -sv "$(hab pkg path core/coreutils)/bin/env" /usr/bin/env 2>/dev/null
+  cp "$(hab pkg path core/coreutils)/bin/env" /usr/bin/env
 
   pushd /src >/dev/null
     for component in $(./scripts/changed_components.rb)

.studio/deployment-service

@@ -10,6 +10,8 @@ function start_deployment_service {
   local local_manifest=""
   local_manifest=$(find_local_manifest)
 
+  cp "$(hab pkg path core/coreutils)/bin/env" /usr/bin/env
+
   chef-automate dev configure-deployment --config dev/config.toml --manifest-dir "$local_manifest"
 
   restart_gateway_if_up

.studiorc

@@ -159,6 +159,9 @@ DOC
 start_all_services() {
   start_deployment_service
   chef-automate dev deployinate
+
+  cp "$(hab pkg path core/coreutils)/bin/env" /usr/bin/env
+
   if [[ -f "/src/dev/license.jwt" ]]; then
      chef-automate license apply "/src/dev/license.jwt"
   fi

components/automate-opensearch/habitat/config/opensearch.yml

@@ -162,6 +162,7 @@ plugins.security.authcz.admin_dn:
 plugins.security.audit.type: internal_opensearch
 plugins.security.enable_snapshot_restore_privilege: true
 plugins.security.check_snapshot_restore_write_privileges: true
+plugins.security.restapi.admin.enabled: true
 plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
 plugins.security.system_indices.enabled: true
 plugins.security.system_indices.indices: [".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opendistro-notifications-*", ".opendistro-notebooks", ".opensearch-observability", ".opendistro-asynchronous-search-response*", ".replication-metadata-store"]

components/automate-opensearch/habitat/config/securityconfig/internal_users.yml

@@ -15,6 +15,7 @@ admin:
   reserved: true
   backend_roles:
   - "admin"
+  - "security_rest_api_access"
   description: "Demo admin user"
 
 kibanaserver:

components/automate-opensearch/habitat/hooks/post-run

@@ -2,9 +2,10 @@
 
 exec 2>&1
 
+sleep 80
 # Executing opensearch securityadmin
 echo "Executing opensearch securityadmin"
-hab pkg exec core/bash {{pkg.path}}/os/plugins/opensearch-security/tools/securityadmin.sh -cd {{pkg.svc_config_path}}/securityconfig -icl -key {{pkg.svc_config_path}}/admin-key.pem -cert {{pkg.svc_config_path}}/admin.pem -cacert {{pkg.svc_config_path}}/root-ca.pem -nhnv -h "localhost" -p {{cfg.transport.port}}
+hab pkg exec core/bash {{pkg.path}}/os/plugins/opensearch-security/tools/securityadmin.sh -cd {{pkg.svc_config_path}}/securityconfig -icl -key {{pkg.svc_config_path}}/admin-key.pem -cert {{pkg.svc_config_path}}/admin.pem -cacert {{pkg.svc_config_path}}/root-ca.pem -nhnv -h "localhost" -p {{cfg.network.port}}
 echo "Done !!!! Executing opensearch securityadmin"
 
 source {{pkg.svc_config_path}}/health_check
@@ -62,7 +63,7 @@ curl \
   -H 'Content-Type: application/json'\
   -X PUT "https://${HOST}:{{cfg.network.port}}/_all/_settings?preserve_existing=true" \
   -k \
-  -u admin:admin \
+  --cert {{pkg.svc_config_path}}/admin.pem --key {{pkg.svc_config_path}}/admin-key.pem \
   -d '{
     "index.number_of_replicas": "{{cfg.index.number_of_replicas}}",
     "index.refresh_interval": "{{cfg.index.refresh_interval}}"

components/automate-opensearch/habitat/plan.sh

@@ -5,12 +5,14 @@
 pkg_name="automate-opensearch"
 pkg_description="Wrapper package for core/elasticsearch"
 pkg_origin="chef"
-pkg_version="1.3.19"
+pkg_version="2.18.0"
 pkg_maintainer="Chef Software Inc. <[email protected]>"
 pkg_license=("Chef-MLSA")
 pkg_upstream_url="https://www.chef.io/automate"
-pkg_source="https://artifacts.opensearch.org/releases/bundle/opensearch/1.3.19/opensearch-1.3.19-linux-x64.tar.gz"
-pkg_shasum=af901097211df4c3d2ef75f0c3699452d82a87adac60c240520a3ca03f969595
+pkg_source="https://artifacts.opensearch.org/releases/bundle/opensearch/2.18.0/opensearch-2.18.0-linux-x64.tar.gz"
+pkg_shasum=ae3cb4107b2e0cdbb9b98bb4e5f2f019b736b68e995442c718e459d39ff01df1
+# pkg_source="https://artifacts.opensearch.org/releases/bundle/opensearch/2.11.1/opensearch-2.11.1-linux-x64.tar.gz"
+# pkg_shasum=926eb3f99f592e7518d5dc1e250ae11d1bb45ef766a90222a8df11bb2ee35a9c
 
 
 pkg_build_deps=(
@@ -52,7 +54,20 @@ do_build() {
   return 0
 }
 
+do_after() {
+  if [ ! -f /usr/bin/env ]; then
+    echo "inside the /usr/ib/env"
+    cp $(pkg_path_for core/coreutils)/bin/env /usr/bin/env
+  fi
+
+}
+
 do_install() {
+  if [ ! -f /usr/bin/env ]; then
+    echo "inside the /isr/ib/env"
+    cp $(pkg_path_for core/coreutils)/bin/env /usr/bin/env
+  fi
+
   cd "$HAB_CACHE_SRC_PATH/opensearch-${pkg_version}"
   chown -RL hab:hab ${pkg_prefix}
   mkdir -p "${pkg_prefix}/os"

components/config-mgmt-service/integration_test/suite_test.go

@@ -56,11 +56,11 @@ const (
 // multiple tests, consider putting it here so that we have them available globally
 //
 // This struct holds:
-// * A Ingest backend client, that you can leverate to do all sorts of ingestion.
-//   => Check this for the list of things this client can do:
-//      https://github.com/github.com/chef/automate/components/ingest-service/blob/master/backend/client.go#L1
-// * A Elasticsearch client, that you can use to throw ES queries.
-//   => Docs: https://godoc.org/gopkg.in/olivere/elastic.v5
+//   - A Ingest backend client, that you can leverate to do all sorts of ingestion.
+//     => Check this for the list of things this client can do:
+//     https://github.com/github.com/chef/automate/components/ingest-service/blob/master/backend/client.go#L1
+//   - A Elasticsearch client, that you can use to throw ES queries.
+//     => Docs: https://godoc.org/gopkg.in/olivere/elastic.v5
 type Suite struct {
 	ingest iBackend.Client
 	client *elastic.Client
@@ -128,16 +128,12 @@ func (s *Suite) GlobalTeardown() {
 	}
 
 	indicesToDelete := make([]string, 0)
-	for _, index := range indices {
-		//don't ever delete node run info.. we'll do that after each test when needed
-		if index != mappings.IndexNameNodeRunInfo {
-			indicesToDelete = append(indicesToDelete, index)
-		}
-	}
-	for i, v := range indicesToDelete {
-		if v == ".opendistro_security" {
-			indicesToDelete = append(indicesToDelete[:i], indicesToDelete[i+1:]...)
-			break
+
+	for _, v := range indices {
+		if v == ".plugins-ml-config" || v == ".opensearch-observability" || v == ".opendistro_security" || v != mappings.IndexNameNodeRunInfo {
+			continue
+		} else {
+			indicesToDelete = append(indicesToDelete, v)
 		}
 	}
 	time.Sleep(2 * time.Second)
@@ -221,30 +217,34 @@ func (s *Suite) Indices() []string {
 //
 // You should call this method on every single test as the following example:
 // ```
-//  func TestGrpcFunc(t *testing.T) {
-//    // Here we are ingesting a number of nodes
-//    suite.IngestNodes(nodes)
 //
-//    // Immediately after the ingestion add the hook to clean all documents,
-//    // by using `defer` you will ensure that the next test will have clean
-//    // data regardless if this test passes or fails
-//    defer suite.DeleteAllDocuments()
-//  }
+//	func TestGrpcFunc(t *testing.T) {
+//	  // Here we are ingesting a number of nodes
+//	  suite.IngestNodes(nodes)
+//
+//	  // Immediately after the ingestion add the hook to clean all documents,
+//	  // by using `defer` you will ensure that the next test will have clean
+//	  // data regardless if this test passes or fails
+//	  defer suite.DeleteAllDocuments()
+//	}
+//
 // ```
 func (s *Suite) DeleteAllDocuments() {
 	// ES Query to match all documents
 	q := elastic.RawStringQuery("{\"match_all\":{}}")
 
 	// Make sure we clean them all!
 	indices, _ := s.client.IndexNames()
-	for i, v := range indices {
-		if v == ".opendistro_security" {
-			indices = append(indices[:i], indices[i+1:]...)
-			break
+	indicesToDelete := make([]string, 0)
+	for _, v := range indices {
+		if v == ".plugins-ml-config" || v == ".opensearch-observability" || v == ".opendistro_security" {
+			continue
+		} else {
+			indicesToDelete = append(indicesToDelete, v)
 		}
 	}
 	_, err := s.client.DeleteByQuery().
-		Index(indices...).
+		Index(indicesToDelete...).
 		Query(q).
 		IgnoreUnavailable(true).
 		Refresh("true").

components/es-sidecar-service/integration_test/purge_test.go

@@ -32,7 +32,7 @@ func TestPurgeTimeSeriesIndicesByAge(t *testing.T) {
 			fmt.Sprintf("When starting with %v indices and removing older than %v days, %v remain",
 				test.startingCount, test.olderThanDays, test.expectedRemainingCount),
 			func(t *testing.T) {
-				doPurgeTest(t, TimeSeriesTestIndex, test.startingCount, test.olderThanDays, test.expectedRemainingCount+2)
+				doPurgeTest(t, TimeSeriesTestIndex, test.startingCount, test.olderThanDays, test.expectedRemainingCount+4)
 			},
 		)
 	}

components/es-sidecar-service/integration_test/suite_test.go

@@ -79,13 +79,16 @@ func (suite *Suite) DeleteAllIndices() {
 	if err != nil {
 		fmt.Printf("Could not retrieve index list: %s\n", err)
 	}
-	for i, v := range indices {
-		if v == ".opendistro_security" {
-			indices = append(indices[:i], indices[i+1:]...)
-			break
+
+	indicesToDelete := make([]string, 0)
+	for _, v := range indices {
+		if v == ".plugins-ml-config" || v == ".opensearch-observability" || v == ".opendistro_security" {
+			continue
+		} else {
+			indicesToDelete = append(indicesToDelete, v)
 		}
 	}
-	suite.esClient.DeleteIndex(indices...).Do(context.Background())
+	suite.esClient.DeleteIndex(indicesToDelete...).Do(context.Background())
 }
 
 func (suite Suite) GlobalTeardown() {

components/ingest-service/integration_test/suite_test.go

@@ -265,14 +265,16 @@ func (s *Suite) DeleteAllDocuments() {
 	q := elastic.RawStringQuery("{\"match_all\":{}}")
 	// Make sure we clean them all!
 	indices, _ := s.client.IndexNames()
-	for i, v := range indices {
-		if v == ".opendistro_security" {
-			indices = append(indices[:i], indices[i+1:]...)
-			break
+	indicesToDelete := make([]string, 0)
+	for _, v := range indices {
+		if v == ".plugins-ml-config" || v == ".opensearch-observability" || v == ".opendistro_security" {
+			continue
+		} else {
+			indicesToDelete = append(indicesToDelete, v)
 		}
 	}
 	_, err := s.client.DeleteByQuery().
-		Index(indices...).
+		Index(indicesToDelete...).
 		Query(q).
 		IgnoreUnavailable(true).
 		Refresh("true").