[rubrik] Add sla_domains data stream (#12923)

packages/rubrik/_dev/build/docs/README.md

@@ -155,12 +155,24 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 
 ### Unmanaged Objects
 
-The `unmanaged_objects` dataset provides metrics related to the performance of the Rubrik cluster nodes.
+The `unmanaged_objects` dataset provides unmanaged object snapshot and storage metrics.
 
 **ECS Field Reference**
 
 Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
 
 {{fields "unmanaged_objects"}}
 
-{{event "unmanaged_objects"}}
+{{event "unmanaged_objects"}}
+
+### SLA Domains
+
+The `sla_domains` dataset captures key metrics and configurations of Service Level Agreement (SLA) policy domains in a Rubrik environment, including details on the number of protected objects, such as virtual machines, databases, filesets, and hosts.
+
+**ECS Field Reference**
+
+Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
+
+{{fields "sla_domains"}}
+
+{{event "sla_domains"}}

packages/rubrik/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.7.0"
+  changes:
+    - description: Add `sla_domains` data stream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12923
 - version: "0.5.0"
   changes:
     - description: Add `unmanaged_objects` data stream

packages/rubrik/data_stream/sla_domains/_dev/test/pipeline/test-sla-domains-metrics.log

@@ -0,0 +1 @@
+{"numNutanixVms":0,"numHypervVms":0,"numDbs":1,"numManagedVolumes":1,"numEc2Instances":0,"numVcdVapps":0,"numOracleDbs":0,"numFilesets":1,"numStorageArrayVolumeGroups":0,"numWindowsVolumeGroups":0,"numLinuxHosts":1,"numShares":0,"numWindowsHosts":0,"numVms":0,"numProtectedObjects":2,"id":"6fc7ca5c-5926-4a85-b236-a43a9935dff9","primaryClusterId":"5e81520e-a7e1-4ed4-adce-870804739532","name":"cluster-1","primaryClusterSlaVersion":20,"replicationTargetSlaVersion":4,"frequencies":{"daily":{"frequency":1,"retention":35},"weekly":{"frequency":1,"retention":53,"dayOfWeek":"Monday"},"monthly":{"frequency":1,"retention":3,"dayOfMonth":"FirstDay"}},"allowedBackupWindows":[{"startTimeAttributes":{"minutes":0,"hour":19},"durationInHours":13}],"firstFullAllowedBackupWindows":[],"localRetentionLimit":259200,"maxLocalRetentionLimit":32054400,"archivalSpecs":[{"locationId":"4b6ccad7-907b-41eb-88c7-3b271802d1ed","locationName":"S3:test-1","archivalThreshold":259200,"isPassthroughSupported":true}],"replicationSpecs":[{"locationId":"2d458abf-df57-4873-9075-63d7b674e307","locationName":"LOCAL","retentionLimit":259200,"replicationType":"REPLICATION_TO_CLUSTER"}],"isDefault":false,"uiColor":"#7f3340","showAdvancedUi":false,"advancedUiConfig":[],"isRetentionLocked":false,"isPaused":false}

packages/rubrik/data_stream/sla_domains/_dev/test/pipeline/test-sla-domains-metrics.log-expected.json

@@ -0,0 +1,34 @@
+{
+    "expected": [
+        {
+            "ecs": {
+                "version": "8.16.0"
+            },
+            "event": {
+                "kind": "metric",
+                "original": "{\"numNutanixVms\":0,\"numHypervVms\":0,\"numDbs\":1,\"numManagedVolumes\":1,\"numEc2Instances\":0,\"numVcdVapps\":0,\"numOracleDbs\":0,\"numFilesets\":1,\"numStorageArrayVolumeGroups\":0,\"numWindowsVolumeGroups\":0,\"numLinuxHosts\":1,\"numShares\":0,\"numWindowsHosts\":0,\"numVms\":0,\"numProtectedObjects\":2,\"id\":\"6fc7ca5c-5926-4a85-b236-a43a9935dff9\",\"primaryClusterId\":\"5e81520e-a7e1-4ed4-adce-870804739532\",\"name\":\"cluster-1\",\"primaryClusterSlaVersion\":20,\"replicationTargetSlaVersion\":4,\"frequencies\":{\"daily\":{\"frequency\":1,\"retention\":35},\"weekly\":{\"frequency\":1,\"retention\":53,\"dayOfWeek\":\"Monday\"},\"monthly\":{\"frequency\":1,\"retention\":3,\"dayOfMonth\":\"FirstDay\"}},\"allowedBackupWindows\":[{\"startTimeAttributes\":{\"minutes\":0,\"hour\":19},\"durationInHours\":13}],\"firstFullAllowedBackupWindows\":[],\"localRetentionLimit\":259200,\"maxLocalRetentionLimit\":32054400,\"archivalSpecs\":[{\"locationId\":\"4b6ccad7-907b-41eb-88c7-3b271802d1ed\",\"locationName\":\"S3:test-1\",\"archivalThreshold\":259200,\"isPassthroughSupported\":true}],\"replicationSpecs\":[{\"locationId\":\"2d458abf-df57-4873-9075-63d7b674e307\",\"locationName\":\"LOCAL\",\"retentionLimit\":259200,\"replicationType\":\"REPLICATION_TO_CLUSTER\"}],\"isDefault\":false,\"uiColor\":\"#7f3340\",\"showAdvancedUi\":false,\"advancedUiConfig\":[],\"isRetentionLocked\":false,\"isPaused\":false}"
+            },
+            "rubrik": {
+                "sla_domains": {
+                    "dbs": {
+                        "count": 1
+                    },
+                    "filesets": {
+                        "count": 1
+                    },
+                    "id": "6fc7ca5c-5926-4a85-b236-a43a9935dff9",
+                    "linux_hosts": {
+                        "count": 1
+                    },
+                    "name": "cluster-1",
+                    "vms": {
+                        "count": 0
+                    },
+                    "windows_hosts": {
+                        "count": 0
+                    }
+                }
+            }
+        }
+    ]
+}

packages/rubrik/data_stream/sla_domains/agent/stream/cel.yml.hbs

@@ -0,0 +1,107 @@
+config_version: 2
+interval: {{interval}}
+
+state:
+  client_id: {{client_id}}
+  client_secret: {{client_secret}}
+  cluster_uuid: {{cluster_uuid}}
+  token_url: https://{{hostname}}.my.rubrik.com/api/cdm_client_token
+
+resource.url: https://{{cluster_ip}}/api/v2/sla_domain
+
+{{#if ssl}}
+resource.ssl: 
+  {{ssl}}
+{{/if}}
+{{#if resource_timeout}}
+resource.timeout: {{resource_timeout}}
+{{/if}}
+
+{{#if enable_request_tracer}}
+resource.tracer.filename: "../../logs/cel/http-request-trace-*.ndjson"
+{{/if}}
+
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}
+
+program: |-
+  state.with(
+    request("POST", state.token_url, {
+        "client_id": state.client_id,
+        "client_secret": state.client_secret,
+        "cluster_uuid": state.cluster_uuid
+      }.encode_json()
+    ).with(
+      {
+        "Header": {
+          "Content-Type": ["application/json"],
+        },
+      }
+    ).do_request().as(token_resp, (token_resp.StatusCode == 200) ?
+      bytes(token_resp.Body).decode_json().as(token_body,
+        request("GET", state.url).with(
+          {
+            "Header": {
+              "Content-Type": ["application/json"],
+              "Authorization": ["Bearer " + token_body.session.token],
+            }
+          }
+        ).do_request().as(resp, (resp.StatusCode == 200) ?
+          bytes(resp.Body).decode_json().as(body,
+            {
+              "events": body.data.map(e,
+                {
+                  "message": e.encode_json(),
+                }
+              ),
+              "want_more": body.hasMore
+            }
+          )
+        :
+          {
+            "events": [
+              {
+                "error": {
+                  "code": string(resp.StatusCode),
+                  "id": string(resp.Status),
+                  "message": "GET request failed: " +
+                  (
+                    (size(resp.Body) != 0) ?
+                      string(resp.Body)
+                    :
+                      string(resp.Status) + " (" + string(resp.StatusCode) + ")"
+                  ),
+                }
+              }
+            ],
+            "want_more": false
+          }
+        )
+      )
+    :
+      {
+        "events": [
+          {
+            "error": {
+              "code": string(token_resp.StatusCode),
+              "id": string(token_resp.Status),
+              "message": "Token request failed: " + string(token_resp.Status),
+            }
+          }
+        ],
+        "want_more": false
+      }
+    )
+  )

packages/rubrik/data_stream/sla_domains/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,82 @@
+---
+description: Pipeline for parsing Rubrik SLA domains metrics
+processors:
+  - set:
+      field: ecs.version
+      value: 8.16.0
+  - set:
+      field: event.kind
+      value: metric
+  - rename:
+      field: message
+      target_field: event.original
+      ignore_missing: true
+      if: "ctx.event?.original == null"
+      description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.'
+  - remove:
+      field: message
+      ignore_missing: true
+      if: "ctx.event?.original != null"
+      description: 'The `message` field is no longer required if the document has an `event.original` field.'
+  - json:
+      field: event.original
+      target_field: json
+  - rename:
+      field: json.id
+      target_field: rubrik.sla_domains.id
+      ignore_missing: true
+  - rename:
+      field: json.name
+      target_field: rubrik.sla_domains.name
+      ignore_missing: true
+  - rename:
+      field: json.numVms
+      target_field: rubrik.sla_domains.vms.count
+      ignore_missing: true
+  - rename:
+      field: json.numDbs
+      target_field: rubrik.sla_domains.dbs.count
+      ignore_missing: true
+  - rename:
+      field: json.numFilesets
+      target_field: rubrik.sla_domains.filesets.count
+      ignore_missing: true
+  - rename:
+      field: json.numLinuxHosts
+      target_field: rubrik.sla_domains.linux_hosts.count
+      ignore_missing: true
+  - rename:
+      field: json.numWindowsHosts
+      target_field: rubrik.sla_domains.windows_hosts.count
+      ignore_missing: true
+  - script:
+      description: Drops null/empty values recursively
+      lang: painless
+      ignore_failure: true
+      source: |
+        boolean drop(Object o) {
+          if (o == null || o == "") {
+            return true;
+          } else if (o instanceof Map) {
+            ((Map) o).values().removeIf(v -> drop(v));
+            return (((Map) o).size() == 0);
+          } else if (o instanceof List) {
+            ((List) o).removeIf(v -> drop(v));
+            return (((List) o).length == 0);
+          }
+          return false;
+        }
+        drop(ctx);
+  - remove:
+      field: json
+      ignore_missing: true
+on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
+      field: error.message
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'

packages/rubrik/data_stream/sla_domains/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

packages/rubrik/data_stream/sla_domains/fields/fields.yml

@@ -0,0 +1,34 @@
+- name: rubrik
+  type: group
+  fields:
+    - name: sla_domains
+      type: group
+      fields:
+        - name: id
+          type: keyword
+          dimension: true
+          description: The ID of the SLA domain.
+        - name: name
+          type: keyword
+          dimension: true
+          description: The name of the SLA domain.
+        - name: vms.count
+          type: long
+          metric_type: gauge
+          description: Number of virtual machines.
+        - name: dbs.count
+          type: long
+          metric_type: gauge
+          description: Number of databases.
+        - name: filesets.count
+          type: long
+          metric_type: gauge
+          description: Number of filesets.
+        - name: linux_hosts.count
+          type: long
+          metric_type: gauge
+          description: Number of physical Linux hosts.
+        - name: windows_hosts.count
+          type: long
+          metric_type: gauge
+          description: Number of physical Winux hosts.

packages/rubrik/data_stream/sla_domains/manifest.yml

@@ -0,0 +1,63 @@
+type: metrics
+title: Rubrik SLA Domains Metrics
+streams:
+  - input: cel
+    title: Rubrik SLA Domains
+    description: Collect Rubrik SLA domains data via the RSC API
+    enabled: true
+    template_path: cel.yml.hbs
+    vars:
+      - name: cluster_ip
+        type: text
+        title: Cluster IP
+        description: IP of the cluster.
+        multi: false
+        required: true
+        show_user: true
+      - name: cluster_uuid
+        type: text
+        title: Cluster UUID
+        multi: false
+        required: true
+        show_user: true
+        description: The UUID of the Rubrik cluster
+      - name: resource_timeout
+        type: text
+        title: Resource Timeout
+        description: Duration before declaring that the HTTP client connection has timed out. Valid time units are ns, us, ms, s, m, h. Default is 60s
+        show_user: false
+        multi: false
+        required: false
+        default: 60s
+      - name: interval
+        type: text
+        title: Interval
+        description: The interval at which the API is polled, supported in seconds, minutes, and hours.
+        show_user: true
+        required: true
+        default: 5m
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details.
+
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        required: false
+        show_user: false
+        default:
+          - rubrik.sla_domains
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false