Add security plugin

.ci/opensearch/Dockerfile.opensearch

@@ -1,25 +1,27 @@
 ARG OPENSEARCH_VERSION
 FROM opensearchproject/opensearch:${OPENSEARCH_VERSION}
 
-ARG OPENSEARCH_VERSION
 ARG opensearch_path=/usr/share/opensearch
 ARG SECURE_INTEGRATION
 ENV SECURE_INTEGRATION=$SECURE_INTEGRATION
+ARG OPENSEARCH_INITIAL_ADMIN_PASSWORD
 
-# Starting in 2.12.0 security demo requires an initial admin password, which is set as myStrongPassword123!
+# Some opensearch secuirty settings are only present since 2.8.0 and causes older versions to brake if the setting is present
 # https://apple.stackexchange.com/a/123408/11374
 RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then \
       $opensearch_path/bin/opensearch-plugin remove opensearch-security; \
       else \
+        $opensearch_path/opensearch-onetime-setup.sh; \
+        echo "plugins.security.nodes_dn_dynamic_config_enabled: true"  | tee -a $opensearch_path/config/opensearch.yml > /dev/null; \
+        echo "plugins.security.unsupported.restapi.allow_securityconfig_modification: true"  | tee -a $opensearch_path/config/opensearch.yml > /dev/null; \
+        echo "plugins.security.ssl_cert_reload_enabled: true" | tee -a $opensearch_path/config/opensearch.yml > /dev/null; \
         function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; }; \
-        if [ $(version $OPENSEARCH_VERSION) -ge $(version "2.12.0") ] || [ $OPENSEARCH_VERSION == "latest" ]; then \
-          echo user admin:myStrongPassword123! > curl.conf ; \
-        else \
-          echo user admin:admin > curl.conf ; \
-        fi\
+        if [ $(version $OPENSEARCH_VERSION) -ge $(version "2.8.0") ] || [ $OPENSEARCH_VERSION == "latest" ]; then \
+          echo "plugins.security.restapi.admin.enabled: true" | tee -a $opensearch_path/config/opensearch.yml > /dev/null; \
+        fi \
       fi
 
 HEALTHCHECK --start-period=20s --interval=30s \
   CMD curl -sf -retry 5 --max-time 5 --retry-delay 5 --retry-max-time 30 \
-  $(if $SECURE_INTEGRATION; then echo "-K curl.conf -k https://"; fi)"localhost:9200" \
+  $(if $SECURE_INTEGRATION; then echo "--cert config/kirk.pem --key config/kirk-key.pem -k https://"; fi)"localhost:9200" \
   || bash -c 'kill -s 15 -1 && (sleep 10; kill -s 9 -1)'

.ci/opensearch/docker-compose.yml

@@ -11,6 +11,7 @@ services:
       args:
         - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false}
         - OPENSEARCH_VERSION=${OPENSEARCH_VERSION:-latest}
+        - OPENSEARCH_INITIAL_ADMIN_PASSWORD=myStrongPassword123!
     environment:
       - discovery.type=single-node
       - bootstrap.memory_lock=true

.github/workflows/test-compatibility.yml

@@ -55,7 +55,7 @@ jobs:
         run: |
           export OPENSEARCH_VERSION=${{ matrix.entry.opensearch_version }}
           export SECURE_INTEGRATION=${{ matrix.secured }}
-          make test-integ race=true
+          make cluster.get-cert test-integ race=true
 
       - name: Stop the OpenSearch cluster
         run: |

.github/workflows/test-integration.yml

@@ -26,11 +26,7 @@ jobs:
           for attempt in `seq 25`; do sleep 5; \
           if curl -s localhost:9200; \
           then echo '=====> ready'; break; fi; if [ $attempt == 25 ]; then exit 1; fi; echo '=====> waiting...'; done
-      - run: make test-integ race=true coverage=true
-      - uses: codecov/codecov-action@v4
-        with:
-          file: tmp/integ.cov
-          flags: integration
+      - run: make test-integ race=true
 
   secured:
     name: Tests against secure cluster
@@ -54,4 +50,8 @@ jobs:
           for attempt in `seq 25`; do sleep 5; \
           if curl -s -ku admin:myStrongPassword123! https://localhost:9200; \
           then echo '=====> ready'; break; fi; if [ $attempt == 25 ]; then exit 1; fi; echo '=====> waiting...'; done
-      - run: make test-integ-secure
+      - run: make cluster.get-cert test-integ-secure race=true coverage=true
+      - uses: codecov/codecov-action@v4
+        with:
+          file: tmp/integ.cov
+          flags: integration

.golangci.yml

@@ -172,4 +172,4 @@ issues:
       path: opensearchtransport/opensearchtransport.go
     - linters:
         - dupl
-      path: (-params\.go|api_indices|api_dangling\.go|api_point_in_time\.go|rethrottle\.go|api_cat-.*\.go)
+      path: (-params\.go|api_indices|api_dangling\.go|api_point_in_time\.go|rethrottle\.go|api_cat-.*\.go|plugins/security/api_\w+.go|plugins/security/api_.*-patch.go)

CHANGELOG.md

@@ -10,6 +10,10 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Adds new error types ([#512](https://github.com/opensearch-project/opensearch-go/pull/506))
 - Adds handling of non json errors to ParseError ([#512](https://github.com/opensearch-project/opensearch-go/pull/506))
 - Adds the `Failures` field to opensearchapi structs ([#510](https://github.com/opensearch-project/opensearch-go/pull/510))
+- Adds the `Fields` field containing the document fields to the `SearchHit` struct. ([#508](https://github.com/opensearch-project/opensearch-go/pull/508))
+- Adds security plugin ([#507](https://github.com/opensearch-project/opensearch-go/pull/507))
+- Adds security settings to container for security testing ([#507](https://github.com/opensearch-project/opensearch-go/pull/507))
+- Adds cluster.get-certs to copy admin certs out of the container ([#507](https://github.com/opensearch-project/opensearch-go/pull/507))
 
 ### Changed
 - Uses docker compose v2 instead of v1 ([#506](https://github.com/opensearch-project/opensearch-go/pull/506))
@@ -21,6 +25,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Adjusts and extent opensearch tests for better coverage ([#517](https://github.com/opensearch-project/opensearch-go/pull/517))
 - Bumps codecov action version to v4 ([#517](https://github.com/opensearch-project/opensearch-go/pull/517))
 - Changes bulk error/reason field and some cat response fields to pointer as they can be nil ([#510](https://github.com/opensearch-project/opensearch-go/pull/510))
+- Adjust workflows to work with security plugin ([#507](https://github.com/opensearch-project/opensearch-go/pull/507))
 
 ### Deprecated
 

Makefile

@@ -197,10 +197,16 @@ cluster.build:
 	docker compose --project-directory .ci/opensearch build;
 
 cluster.start:
-	docker compose --project-directory .ci/opensearch up -d ;
+	docker compose --project-directory .ci/opensearch up -d;
 
 cluster.stop:
-	docker compose --project-directory .ci/opensearch down ;
+	docker compose --project-directory .ci/opensearch down;
+
+cluster.get-cert:
+	@if [[ -v SECURE_INTEGRATION ]] && [[ $$SECURE_INTEGRATION == "true" ]]; then \
+		docker cp $$(docker compose --project-directory .ci/opensearch ps --format '{{.Name}}'):/usr/share/opensearch/config/kirk.pem admin.pem && \
+		docker cp $$(docker compose --project-directory .ci/opensearch ps --format '{{.Name}}'):/usr/share/opensearch/config/kirk-key.pem admin.key; \
+	fi
 
 
 cluster.clean: ## Remove unused Docker volumes and networks

opensearch.go

@@ -325,3 +325,8 @@ func addrsToURLs(addrs []string) ([]*url.URL, error) {
 
 	return urls, nil
 }
+
+// ToPointer converts any value to a pointer, mainly used for request parameters
+func ToPointer[V any](value V) *V {
+	return &value
+}

opensearch_internal_test.go

@@ -414,3 +414,9 @@ func TestParseElasticsearchVersion(t *testing.T) {
 		})
 	}
 }
+
+func TestToPointer(t *testing.T) {
+	testPointer := ToPointer(true)
+	assert.NotNil(t, testPointer)
+	assert.True(t, *testPointer)
+}

plugins/security/api.go

@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// The OpenSearch Contributors require contributions made to
+// this file be licensed under the Apache-2.0 license or a
+// compatible open source license.
+
+package security
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/opensearch-project/opensearch-go/v3"
+)
+
+// Config represents the client configuration
+type Config struct {
+	Client opensearch.Config
+}
+
+// Client represents the security Client summarizing all API calls
+type Client struct {
+	Client         *opensearch.Client
+	Account        accountClient
+	ActionGroups   actiongroupsClient
+	Audit          auditClient
+	InternalUsers  internalusersClient
+	NodesDN        nodesdnClient
+	Roles          rolesClient
+	RolesMapping   rolesmappingClient
+	SecurityConfig securityconfigClient
+	SSL            sslClient
+	Tenants        tenantsClient
+}
+
+// clientInit inits the Client with all sub clients
+func clientInit(rootClient *opensearch.Client) *Client {
+	client := &Client{
+		Client: rootClient,
+	}
+	client.Account = accountClient{apiClient: client}
+	client.ActionGroups = actiongroupsClient{apiClient: client}
+	client.Audit = auditClient{apiClient: client}
+	client.InternalUsers = internalusersClient{apiClient: client}
+	client.NodesDN = nodesdnClient{apiClient: client}
+	client.Roles = rolesClient{apiClient: client}
+	client.RolesMapping = rolesmappingClient{apiClient: client}
+	client.SecurityConfig = securityconfigClient{apiClient: client}
+	client.SSL = sslClient{apiClient: client}
+	client.Tenants = tenantsClient{apiClient: client}
+	return client
+}
+
+// NewClient returns a security client
+func NewClient(config Config) (*Client, error) {
+	rootClient, err := opensearch.NewClient(config.Client)
+	if err != nil {
+		return nil, err
+	}
+
+	return clientInit(rootClient), nil
+}
+
+// do calls the opensearch.Client.Do() and checks the response for errors
+func (c *Client) do(ctx context.Context, req opensearch.Request, dataPointer any) (*opensearch.Response, error) {
+	resp, err := c.Client.Do(ctx, req, dataPointer)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.IsError() {
+		if dataPointer != nil {
+			return resp, opensearch.ParseError(resp)
+		} else {
+			return resp, fmt.Errorf("status: %s", resp.Status())
+		}
+	}
+
+	return resp, nil
+}

plugins/security/api_account-get.go

@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// The OpenSearch Contributors require contributions made to
+// this file be licensed under the Apache-2.0 license or a
+// compatible open source license.
+
+package security
+
+import (
+	"net/http"
+
+	"github.com/opensearch-project/opensearch-go/v3"
+)
+
+// AccountGetReq represents possible options for the account get request
+type AccountGetReq struct {
+	Header http.Header
+}
+
+// GetRequest returns the *http.Request that gets executed by the client
+func (r AccountGetReq) GetRequest() (*http.Request, error) {
+	return opensearch.BuildRequest(
+		"GET",
+		"/_plugins/_security/api/account",
+		nil,
+		make(map[string]string),
+		r.Header,
+	)
+}
+
+// AccountGetResp represents the returned struct of the account get response
+type AccountGetResp struct {
+	UserName            string          `json:"user_name"`
+	IsReserved          bool            `json:"is_reserved"`
+	IsHidden            bool            `json:"is_hidden"`
+	IsInternaluser      bool            `json:"is_internal_user"`
+	BackendRoles        []string        `json:"backend_roles"`
+	CustomAttributes    []string        `json:"custom_attribute_names"`
+	UserRequestedTenant *string         `json:"user_requested_tenant"`
+	Tennants            map[string]bool `json:"tenants"`
+	Roles               []string        `json:"roles"`
+	response            *opensearch.Response
+}
+
+// Inspect returns the Inspect type containing the raw *opensearch.Reponse
+func (r AccountGetResp) Inspect() Inspect {
+	return Inspect{Response: r.response}
+}

plugins/security/api_account-put.go

@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// The OpenSearch Contributors require contributions made to
+// this file be licensed under the Apache-2.0 license or a
+// compatible open source license.
+
+package security
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+
+	"github.com/opensearch-project/opensearch-go/v3"
+)
+
+// AccountPutReq represents possible options for the account put request
+type AccountPutReq struct {
+	Body AccountPutBody
+
+	Header http.Header
+}
+
+// GetRequest returns the *http.Request that gets executed by the client
+func (r AccountPutReq) GetRequest() (*http.Request, error) {
+	body, err := json.Marshal(r.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return opensearch.BuildRequest(
+		"PUT",
+		"/_plugins/_security/api/account",
+		bytes.NewReader(body),
+		make(map[string]string),
+		r.Header,
+	)
+}
+
+// AccountPutBody reperensts the request body for AccountPutReq
+type AccountPutBody struct {
+	CurrentPassword string `json:"current_password"`
+	Password        string `json:"password"`
+}
+
+// AccountPutResp represents the returned struct of the account put response
+type AccountPutResp struct {
+	Message  string `json:"message"`
+	Status   string `json:"status"`
+	response *opensearch.Response
+}
+
+// Inspect returns the Inspect type containing the raw *opensearch.Reponse
+func (r AccountPutResp) Inspect() Inspect {
+	return Inspect{Response: r.response}
+}

plugins/security/api_account.go

@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// The OpenSearch Contributors require contributions made to
+// this file be licensed under the Apache-2.0 license or a
+// compatible open source license.
+
+package security
+
+import (
+	"context"
+)
+
+type accountClient struct {
+	apiClient *Client
+}
+
+// Get executes a get account request with the optional AccountGetReq
+func (c accountClient) Get(ctx context.Context, req *AccountGetReq) (AccountGetResp, error) {
+	if req == nil {
+		req = &AccountGetReq{}
+	}
+
+	var (
+		data AccountGetResp
+		err  error
+	)
+	if data.response, err = c.apiClient.do(ctx, req, &data); err != nil {
+		return data, err
+	}
+
+	return data, nil
+}
+
+// Put executes a put account request with the required AccountPutReq
+func (c accountClient) Put(ctx context.Context, req AccountPutReq) (AccountPutResp, error) {
+	var (
+		data AccountPutResp
+		err  error
+	)
+	if data.response, err = c.apiClient.do(ctx, req, &data); err != nil {
+		return data, err
+	}
+
+	return data, nil
+}