Stop using the internal reset credentials in the serverless provider (#…

…2307)

Stop using the internal `_reset-internal-credentials` API to obtain the credentials used
for serverless stacks managed by elastic-package. This API is only available to Elastic
employees.

There were two reasons to obtain credentials this way:
- Some APIs were restricted in serverless to credentials obtained this way. From these
  internal APIs, the only one we use now is the one to check the cluster status (`_cluster/health`).
  In this case, if normal credentials are used, it returns HTTP status 410 (Gone) if the
  cluster is alive and the credentials are valid. We can consider this as an OK for this case.
- At some point during development the project creation API didn't return credentials with
  enough permissions, so we needed to request them using the reset credentials APIs.
  Serverless is GA now, and the project creation API returns user and password with admin
  privileges, what is enough for all elastic-package operations.

As these reasons are not strong enough anymore, it would be better to avoid these resets,
what will allow to use elastic-package more openly.

This is also needed for API keys support (#1633), because API keys neither have access to
these internal APIs.

internal/elasticsearch/client.go

@@ -145,6 +145,10 @@ func (client *Client) CheckHealth(ctx context.Context) error {
 	}
 	defer resp.Body.Close()
 
+	if resp.StatusCode == http.StatusGone {
+		// We are in a managed deployment, API not available, assume healthy.
+		return nil
+	}
 	if resp.StatusCode != http.StatusOK {
 		return fmt.Errorf("failed to check cluster health: %s", resp.String())
 	}

internal/elasticsearch/client_test.go

@@ -63,6 +63,11 @@ func TestClusterHealth(t *testing.T) {
 			// To reproduce the scenario, just start the stack with 8.5 version.
 			Record: "./testdata/elasticsearch-8-5-healthy",
 		},
+		{
+			// To reproduce the scenario, start a project in serverless, and
+			// replace the host in the urls with https://127.0.0.1:9200.
+			Record: "./testdata/elasticsearch-serverless-healthy",
+		},
 		{
 			// To reproduce the scenario, start the stack with 8.5 version and
 			// limited disk space. If difficult to reproduce, manually modify

internal/elasticsearch/testdata/elasticsearch-serverless-healthy.yaml

@@ -0,0 +1,122 @@
+---
+version: 2
+interactions:
+    - id: 0
+      request:
+        proto: HTTP/1.1
+        proto_major: 1
+        proto_minor: 1
+        content_length: 0
+        transfer_encoding: []
+        trailer: {}
+        host: ""
+        remote_addr: ""
+        request_uri: ""
+        body: ""
+        form: {}
+        headers:
+            Authorization:
+                - Basic YWRtaW46cWN5V21PNTM5WTBlOWw5T1lxM3Y1TjIx
+            User-Agent:
+                - go-elasticsearch/7.17.10 (linux amd64; Go 1.23.4)
+            X-Elastic-Client-Meta:
+                - es=7.17.10,go=1.23.4,t=7.17.10,hc=1.23.4
+        url: https://127.0.0.1:9200/
+        method: GET
+      response:
+        proto: HTTP/2.0
+        proto_major: 2
+        proto_minor: 0
+        transfer_encoding: []
+        trailer: {}
+        content_length: 508
+        uncompressed: false
+        body: |
+            {
+              "name" : "serverless",
+              "cluster_name" : "b28743a6426f4b58829f422195a77b1b",
+              "cluster_uuid" : "RzabjGAWQRCQ0vHoYTrRMA",
+              "version" : {
+                "number" : "8.11.0",
+                "build_flavor" : "serverless",
+                "build_type" : "docker",
+                "build_hash" : "00000000",
+                "build_date" : "2023-10-31",
+                "build_snapshot" : false,
+                "lucene_version" : "9.7.0",
+                "minimum_wire_compatibility_version" : "8.11.0",
+                "minimum_index_compatibility_version" : "8.11.0"
+              },
+              "tagline" : "You Know, for Search"
+            }
+        headers:
+            Content-Length:
+                - "508"
+            Content-Type:
+                - application/json
+            Date:
+                - Tue, 24 Dec 2024 16:55:35 GMT
+            Elastic-Api-Version:
+                - "2023-10-31"
+            X-Cloud-Request-Id:
+                - OcEBAr_LQMioTlvn_pbkIQ
+            X-Elastic-Product:
+                - Elasticsearch
+            X-Found-Handling-Cluster:
+                - b28743a6426f4b58829f422195a77b1b.es
+            X-Found-Handling-Instance:
+                - es-es-search-7b77d558d4-pv8xf
+        status: 200 OK
+        code: 200
+        duration: 5.561667976s
+    - id: 1
+      request:
+        proto: HTTP/1.1
+        proto_major: 1
+        proto_minor: 1
+        content_length: 0
+        transfer_encoding: []
+        trailer: {}
+        host: ""
+        remote_addr: ""
+        request_uri: ""
+        body: ""
+        form: {}
+        headers:
+            Authorization:
+                - Basic YWRtaW46cWN5V21PNTM5WTBlOWw5T1lxM3Y1TjIx
+            User-Agent:
+                - go-elasticsearch/7.17.10 (linux amd64; Go 1.23.4)
+            X-Elastic-Client-Meta:
+                - es=7.17.10,go=1.23.4,t=7.17.10,hc=1.23.4
+        url: https://127.0.0.1:9200/_cluster/health
+        method: GET
+      response:
+        proto: HTTP/2.0
+        proto_major: 2
+        proto_minor: 0
+        transfer_encoding: []
+        trailer: {}
+        content_length: 363
+        uncompressed: false
+        body: '{"error":{"root_cause":[{"type":"api_not_available_exception","reason":"Request for uri [/_cluster/health] with method [GET] exists but is not available when running in serverless mode"}],"type":"api_not_available_exception","reason":"Request for uri [/_cluster/health] with method [GET] exists but is not available when running in serverless mode"},"status":410}'
+        headers:
+            Content-Length:
+                - "363"
+            Content-Type:
+                - application/json
+            Date:
+                - Tue, 24 Dec 2024 16:55:35 GMT
+            Elastic-Api-Version:
+                - "2023-10-31"
+            X-Cloud-Request-Id:
+                - z0crFf48SH6oYcdTJWoptQ
+            X-Elastic-Product:
+                - Elasticsearch
+            X-Found-Handling-Cluster:
+                - b28743a6426f4b58829f422195a77b1b.es
+            X-Found-Handling-Instance:
+                - es-es-search-7b77d558d4-pv8xf
+        status: 410 Gone
+        code: 410
+        duration: 129.604829ms

internal/serverless/client.go

@@ -171,11 +171,6 @@ func (c *Client) CreateProject(ctx context.Context, name, region, projectType st
 		return nil, fmt.Errorf("error while decoding create project response: %w", err)
 	}
 
-	err = c.ResetCredentials(ctx, serverlessProject)
-	if err != nil {
-		return nil, fmt.Errorf("failed to reset credentials: %w", err)
-	}
-
 	return serverlessProject, nil
 }
 
@@ -230,34 +225,6 @@ func (c *Client) StatusProject(ctx context.Context, project *Project) (string, e
 	return status.Phase, nil
 }
 
-func (c *Client) ResetCredentials(ctx context.Context, project *Project) error {
-	resourcePath, err := url.JoinPath(c.host, projectsAPI, project.Type, project.ID, "_reset-internal-credentials")
-	if err != nil {
-		return fmt.Errorf("could not build the URL: %w", err)
-	}
-	statusCode, respBody, err := c.post(ctx, resourcePath, nil)
-	if err != nil {
-		return fmt.Errorf("error creating project: %w", err)
-	}
-
-	if statusCode != http.StatusOK {
-		return fmt.Errorf("unexpected status code %d", statusCode)
-	}
-
-	var credentials struct {
-		Username string `json:"username"`
-		Password string `json:"password"`
-	}
-	if err := json.Unmarshal(respBody, &credentials); err != nil {
-		return fmt.Errorf("unable to decode credentials: %w", err)
-	}
-
-	project.Credentials.Username = credentials.Username
-	project.Credentials.Password = credentials.Password
-
-	return nil
-}
-
 func (c *Client) DeleteProject(ctx context.Context, project *Project) error {
 	resourcePath, err := url.JoinPath(c.host, projectsAPI, project.Type, project.ID)
 	if err != nil {