From 066edc45073de1b07b95838ca6f5d7208d38b444 Mon Sep 17 00:00:00 2001 From: flexitrev Date: Fri, 7 Feb 2025 12:49:02 -0500 Subject: [PATCH] third time is the charm --- packages/logstash/_dev/build/docs/README.md | 1222 +------ packages/logstash/changelog.yml | 5 + .../health_report/agent/stream/stream.yml.hbs | 63 + .../health_report/fields/base-fields.yml | 9 + .../data_stream/health_report/fields/ecs.yml | 24 + .../health_report/fields/fields.yml | 90 + .../data_stream/health_report/manifest.yml | 20 + .../logstash/data_stream/node/manifest.yml | 1 + .../data_stream/node_cel/manifest.yml | 1 - .../data_stream/node_stats/manifest.yml | 1 + .../data_stream/pipeline/manifest.yml | 2 - .../logstash/data_stream/plugins/manifest.yml | 2 - packages/logstash/docs/README.md | 2180 ++++++------ ...-838aac39-8edd-48b0-95b4-289e42b1e98a.json | 3090 +++++++++++++++++ ...-9a72208d-e446-48b9-8a63-c4256b9aa4e3.json | 1022 ++++++ packages/logstash/manifest.yml | 8 +- 16 files changed, 5536 insertions(+), 2204 deletions(-) create mode 100644 packages/logstash/data_stream/health_report/agent/stream/stream.yml.hbs create mode 100644 packages/logstash/data_stream/health_report/fields/base-fields.yml create mode 100644 packages/logstash/data_stream/health_report/fields/ecs.yml create mode 100644 packages/logstash/data_stream/health_report/fields/fields.yml create mode 100644 packages/logstash/data_stream/health_report/manifest.yml create mode 100644 packages/logstash/kibana/dashboard/logstash-838aac39-8edd-48b0-95b4-289e42b1e98a.json create mode 100644 packages/logstash/kibana/dashboard/logstash-9a72208d-e446-48b9-8a63-c4256b9aa4e3.json diff --git a/packages/logstash/_dev/build/docs/README.md b/packages/logstash/_dev/build/docs/README.md index 200c38117c6..ae6a16aca81 100644 --- a/packages/logstash/_dev/build/docs/README.md +++ b/packages/logstash/_dev/build/docs/README.md @@ -2,7 +2,7 @@ This integration collects logs and metrics from Logstash instances. -You can find additional information about monitoring Logstash with the Logstash integration in the **Logstash Reference**: [Monitoring Logstash with Elastic Agent](https://www.elastic.co/guide/en/logstash/current/monitoring-with-ea.html). +You can find additional information about monitoring Logstash with the Logstash integration in the **Logstash Reference**: {{ url "logstash-monitoring-ea" "Monitoring Logstash with Elastic Agent" }}. ## Compatibility @@ -16,6 +16,48 @@ By utilizing Elastic Agent we are able to query additional monitoring APIs and p Elastic Agent based metrics collection is not compatible with the Stack Monitoring UI inside Kibana, please only select Metrics (Elastic Agent). Users that prefer the Stack Monitoring UI should uncheck `Metrics (Elastic Agent)` and continue to use `Metrics (Stack Monitoring)`. +### Fields and Sample Events + +#### Health Report +The health report api is available starting with Logstash 8.16.0, which provides the `health_report` dataset for Node health and Pipeline health dashboards + +#### Example +An example event for 'health_report' looks as following: + +{{fields "health_report"}} + +{{event "health_report"}} + +#### Node + +This is the `node` dataset, which drives the Node dashboard pages. + +#### Example + +{{fields "node_cel"}} + +{{event "node_cel"}} + +#### Pipeline + +This is the `pipeline` dataset, which drives the Pipeline dashboard pages. + +#### Example + +{{fields "pipeline"}} + +{{event "pipeline"}} + +#### Plugin + +This is the `plugin` dataset, which drives the Pipeline detail dashboard pages. Note that this dataset may produce many documents for logstash instances using a large number of pipelines and/or plugins within those pipelines. For those instances, we recommend reviewing the +pipeline collection period, and setting it to an appropriate value. + +#### Example + +{{fields "plugins"}} + +{{event "plugins"}} ## Logs @@ -37,279 +79,7 @@ Logstash metric related data streams works with Logstash 7.3.0 and later. ### Node Stats -An example event for `node_stats` looks as following: - -```json -{ - "@timestamp": "2023-03-02T15:57:56.968Z", - "agent": { - "ephemeral_id": "16f2dd63-454b-4699-a8c8-2a748bd044b8", - "id": "3cc85092-54dc-4b58-8726-5e9458167f42", - "name": "docker-fleet-agent", - "type": "metricbeat", - "version": "8.5.0" - }, - "data_stream": { - "dataset": "logstash.stack_monitoring.node_stats", - "namespace": "ep", - "type": "metrics" - }, - "ecs": { - "version": "8.0.0" - }, - "elastic_agent": { - "id": "3cc85092-54dc-4b58-8726-5e9458167f42", - "snapshot": false, - "version": "8.5.0" - }, - "event": { - "agent_id_status": "verified", - "dataset": "logstash.stack_monitoring.node_stats", - "duration": 48419400, - "ingested": "2023-03-02T15:57:58Z", - "module": "logstash" - }, - "host": { - "architecture": "x86_64", - "containerized": true, - "hostname": "docker-fleet-agent", - "id": "66392b0697b84641af8006d87aeb89f1", - "ip": [ - "192.168.224.7" - ], - "mac": [ - "02-42-C0-A8-E0-07" - ], - "name": "docker-fleet-agent", - "os": { - "codename": "focal", - "family": "debian", - "kernel": "5.10.47-linuxkit", - "name": "Ubuntu", - "platform": "ubuntu", - "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)" - } - }, - "logstash": { - "cluster": { - "id": "0toa26-cTzmqx0WD40-4XQ" - }, - "elasticsearch": { - "cluster": { - "id": "0toa26-cTzmqx0WD40-4XQ" - } - }, - "node": { - "stats": { - "events": { - "duration_in_millis": 334, - "filtered": 138, - "in": 618, - "out": 138 - }, - "jvm": { - "gc": { - "collectors": { - "old": { - "collection_count": 0, - "collection_time_in_millis": 0 - }, - "young": { - "collection_count": 13, - "collection_time_in_millis": 177 - } - } - }, - "mem": { - "heap_max_in_bytes": 10527703038, - "heap_used_in_bytes": 234688352, - "heap_used_percent": 2 - }, - "uptime_in_millis": 21450 - }, - "logstash": { - "ephemeral_id": "17681d23-bd67-4c40-b6b1-63e97b560856", - "host": "170bc3698b89", - "http_address": "0.0.0.0:9600", - "name": "170bc3698b89", - "pipeline": { - "batch_size": 125, - "workers": 10 - }, - "snapshot": false, - "status": "green", - "uuid": "a4224a67-aae8-4bce-8660-079d068b2e72", - "version": "8.5.0" - }, - "os": { - "cgroup": { - "cpu": { - "cfs_quota_micros": -1, - "control_group": "/", - "stat": { - "number_of_elapsed_periods": 0, - "number_of_times_throttled": 0, - "time_throttled_nanos": 0 - } - }, - "cpuacct": { - "control_group": "/", - "usage_nanos": 55911664431 - } - }, - "cpu": { - "load_average": { - "15m": 2.28, - "1m": 2.85, - "5m": 2.62 - }, - "percent": 0 - } - }, - "pipelines": [ - { - "ephemeral_id": "453a2361-82d8-4d88-b7a4-063c3293cd4a", - "events": { - "duration_in_millis": 0, - "filtered": 0, - "in": 476, - "out": 0, - "queue_push_duration_in_millis": 59 - }, - "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa", - "id": "pipeline-with-persisted-queue", - "queue": { - "capacity": { - "max_queue_size_in_bytes": 1073741824, - "max_unread_events": 0, - "page_capacity_in_bytes": 67108864, - "queue_size_in_bytes": 132880 - }, - "data": { - "free_space_in_bytes": 51709984768, - "path": "/usr/share/logstash/data/queue/pipeline-with-persisted-queue", - "storage_type": "overlay" - }, - "events": 0, - "events_count": 0, - "max_queue_size_in_bytes": 1073741824, - "queue_size_in_bytes": 132880, - "type": "persisted" - }, - "reloads": { - "failures": 0, - "successes": 0 - }, - "vertices": [ - { - "events_out": 475, - "id": "dfc132c40b9f5dbc970604f191cf87ee04b102b6f4be5a235436973dc7ea6368", - "pipeline_ephemeral_id": "453a2361-82d8-4d88-b7a4-063c3293cd4a", - "queue_push_duration_in_millis": 59 - }, - { - "duration_in_millis": 0, - "events_in": 375, - "events_out": 0, - "id": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", - "pipeline_ephemeral_id": "453a2361-82d8-4d88-b7a4-063c3293cd4a" - }, - { - "cluster_uuid": "0toa26-cTzmqx0WD40-4XQ", - "duration_in_millis": 1, - "events_in": 0, - "events_out": 0, - "id": "9ba6577aa5c41a5ebcaae010b9a0ef44015ae68c624596ed924417d1701abc21", - "pipeline_ephemeral_id": "453a2361-82d8-4d88-b7a4-063c3293cd4a" - } - ] - }, - { - "ephemeral_id": "7114cd7d-8d91-4afc-a986-32487c3edcbe", - "events": { - "duration_in_millis": 191, - "filtered": 91, - "in": 95, - "out": 91, - "queue_push_duration_in_millis": 4 - }, - "hash": "0542fa70daa36dc3e858ea099f125cc8c9e451ebbfe8ea8867e52f9764da0a35", - "id": "pipeline-with-memory-queue", - "queue": { - "events_count": 0, - "max_queue_size_in_bytes": 0, - "queue_size_in_bytes": 0, - "type": "memory" - }, - "reloads": { - "failures": 0, - "successes": 0 - }, - "vertices": [ - { - "events_out": 95, - "id": "4c5941552cdaa72ebc285557c697a7150c359ee3eacf9b5664c4b1048e26153b", - "pipeline_ephemeral_id": "7114cd7d-8d91-4afc-a986-32487c3edcbe", - "queue_push_duration_in_millis": 4 - }, - { - "cluster_uuid": "0toa26-cTzmqx0WD40-4XQ", - "duration_in_millis": 193, - "events_in": 91, - "events_out": 91, - "id": "635a080aacc8700059852859da284a9cb92cb78a6d7112fbf55e441e51b6658a", - "long_counters": [ - { - "name": "bulk_requests.successes", - "value": 12 - }, - { - "name": "bulk_requests.responses.200", - "value": 12 - }, - { - "name": "documents.successes", - "value": 91 - } - ], - "pipeline_ephemeral_id": "7114cd7d-8d91-4afc-a986-32487c3edcbe" - } - ] - } - ], - "process": { - "cpu": { - "percent": 4 - }, - "max_file_descriptors": 1048576, - "open_file_descriptors": 89 - }, - "queue": { - "events_count": 0 - }, - "reloads": { - "failures": 0, - "successes": 0 - }, - "timestamp": "2023-03-02T15:57:57.016Z" - } - } - }, - "metricset": { - "name": "node_stats", - "period": 10000 - }, - "service": { - "address": "http://elastic-package-service_logstash_1:9600/_node/stats", - "hostname": "170bc3698b89", - "id": "", - "name": "logstash", - "type": "logstash", - "version": "8.5.0" - } -} -``` +{{event "node_stats"}} **Exported fields** @@ -361,908 +131,6 @@ An example event for `node_stats` looks as following: ### Node -An example event for `node` looks as following: - -```json -{ - "@timestamp": "2023-03-02T15:57:03.999Z", - "agent": { - "ephemeral_id": "16f2dd63-454b-4699-a8c8-2a748bd044b8", - "id": "3cc85092-54dc-4b58-8726-5e9458167f42", - "name": "docker-fleet-agent", - "type": "metricbeat", - "version": "8.5.0" - }, - "data_stream": { - "dataset": "logstash.stack_monitoring.node", - "namespace": "ep", - "type": "metrics" - }, - "ecs": { - "version": "8.0.0" - }, - "elastic_agent": { - "id": "3cc85092-54dc-4b58-8726-5e9458167f42", - "snapshot": false, - "version": "8.5.0" - }, - "event": { - "agent_id_status": "verified", - "dataset": "logstash.stack_monitoring.node", - "duration": 69490100, - "ingested": "2023-03-02T15:57:05Z", - "module": "logstash" - }, - "host": { - "architecture": "x86_64", - "containerized": true, - "hostname": "docker-fleet-agent", - "id": "66392b0697b84641af8006d87aeb89f1", - "ip": [ - "192.168.224.7" - ], - "mac": [ - "02-42-C0-A8-E0-07" - ], - "name": "docker-fleet-agent", - "os": { - "codename": "focal", - "family": "debian", - "kernel": "5.10.47-linuxkit", - "name": "Ubuntu", - "platform": "ubuntu", - "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)" - } - }, - "logstash": { - "cluster": { - "id": "0toa26-cTzmqx0WD40-4XQ" - }, - "elasticsearch": { - "cluster": { - "id": "0toa26-cTzmqx0WD40-4XQ" - } - }, - "node": { - "host": "45730b5f8c3d", - "id": "2e17cd45-ecb8-4358-a420-b867f2e32b7a", - "jvm": { - "version": "17.0.4" - }, - "state": { - "pipeline": { - "batch_size": 125, - "ephemeral_id": "472cf082-aa15-41ca-9ed1-62d03afbadd0", - "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa", - "id": "pipeline-with-persisted-queue", - "representation": { - "graph": { - "edges": [ - { - "from": "dfc132c40b9f5dbc970604f191cf87ee04b102b6f4be5a235436973dc7ea6368", - "id": "9ed824e4f189b461c111ae27c17644c3c5f6d7c3c2bb213cbc7cc067cbd68fe6", - "to": "__QUEUE__", - "type": "plain" - }, - { - "from": "__QUEUE__", - "id": "cb33f8fb7611e31a2c1751b74cdedf5b8cdb96ea46b812a2541e2db4f13dca10", - "to": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", - "type": "plain" - }, - { - "from": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", - "id": "63ef166c45b87a40f31e0a6def175f10460b6b0ed656e70968eb52b1c454ab16", - "to": "9ba6577aa5c41a5ebcaae010b9a0ef44015ae68c624596ed924417d1701abc21", - "type": "plain" - } - ], - "vertices": [ - { - "config_name": "java_generator", - "explicit_id": false, - "id": "dfc132c40b9f5dbc970604f191cf87ee04b102b6f4be5a235436973dc7ea6368", - "meta": { - "source": { - "column": 3, - "id": "/usr/share/logstash/pipeline/persisted-queue.conf", - "line": 2, - "protocol": "file" - } - }, - "plugin_type": "input", - "type": "plugin" - }, - { - "explicit_id": false, - "id": "__QUEUE__", - "meta": null, - "type": "queue" - }, - { - "config_name": "sleep", - "explicit_id": false, - "id": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", - "meta": { - "source": { - "column": 3, - "id": "/usr/share/logstash/pipeline/persisted-queue.conf", - "line": 8, - "protocol": "file" - } - }, - "plugin_type": "filter", - "type": "plugin" - }, - { - "config_name": "elasticsearch", - "explicit_id": false, - "id": "9ba6577aa5c41a5ebcaae010b9a0ef44015ae68c624596ed924417d1701abc21", - "meta": { - "source": { - "column": 3, - "id": "/usr/share/logstash/pipeline/persisted-queue.conf", - "line": 15, - "protocol": "file" - } - }, - "plugin_type": "output", - "type": "plugin" - } - ] - }, - "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa", - "type": "lir", - "version": "0.0.0" - }, - "workers": 10 - } - }, - "version": "8.5.0" - } - }, - "metricset": { - "name": "node", - "period": 10000 - }, - "process": { - "pid": 1 - }, - "service": { - "address": "http://elastic-package-service_logstash_1:9600/_node", - "hostname": "45730b5f8c3d", - "id": "2e17cd45-ecb8-4358-a420-b867f2e32b7a", - "name": "logstash", - "type": "logstash", - "version": "8.5.0" - } -} -``` - - -## Metrics (Technical Preview) - -This Logstash package also includes a technical preview of Logstash data collection and dashboards -native to elastic agent. The technical preview includes enhanced data collection, and a number of dashboards, which include additional insight into running pipelines. - -Note that this feature is not intended for use with the Stack Monitoring UI inside Kibana, -and is included as a technical preview. Existing implementations wishing to continue using the Stack Monitoring UI should uncheck the technical preview option, and continue to use `Metrics (Stack Monitoring)`. Those users who wish to use the technical preview should uncheck `Metrics (Stack Monitoring)` and check `Metrics (Technical Preview)` - -### Fields and Sample Event - -#### Node - -This is the `node` dataset, which drives the Node dashboard pages. - -#### Example - -**Exported fields** - -| Field | Description | Type | Metric Type | -|---|---|---|---| -| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | -| cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | -| cloud.region | Region in which this host, resource, or service is located. | keyword | | -| cluster_uuid | | alias | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | -| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | -| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | -| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | -| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | -| host.architecture | Operating system architecture. | keyword | | -| host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | -| host.os.build | OS build information. | keyword | | -| host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| input.type | | keyword | | -| logstash.elasticsearch.cluster.id | | keyword | | -| logstash.host.address | | alias | | -| logstash.host.name | | alias | | -| logstash.node.stats.events.duration_in_millis | | long | counter | -| logstash.node.stats.events.filtered | Filtered events counter | long | counter | -| logstash.node.stats.events.in | Incoming events counter | long | counter | -| logstash.node.stats.events.out | Outgoing events counter | long | counter | -| logstash.node.stats.events.queue_push_duration_in_millis | | long | counter | -| logstash.node.stats.jvm.gc.collectors.old.collection_count | | long | counter | -| logstash.node.stats.jvm.gc.collectors.old.collection_time_in_millis | | long | counter | -| logstash.node.stats.jvm.gc.collectors.young.collection_count | | long | counter | -| logstash.node.stats.jvm.gc.collectors.young.collection_time_in_millis | | long | counter | -| logstash.node.stats.jvm.mem.heap_committed_in_bytes | | long | gauge | -| logstash.node.stats.jvm.mem.heap_max_in_bytes | | long | counter | -| logstash.node.stats.jvm.mem.heap_used_in_bytes | | long | gauge | -| logstash.node.stats.jvm.mem.heap_used_percent | | long | gauge | -| logstash.node.stats.jvm.mem.non_heap_committed_in_bytes | | long | gauge | -| logstash.node.stats.jvm.mem.non_heap_used_in_bytes | | long | gauge | -| logstash.node.stats.jvm.threads.count | current number of threads | long | counter | -| logstash.node.stats.jvm.threads.peak_count | peak number of threads | long | counter | -| logstash.node.stats.jvm.uptime_in_millis | | long | counter | -| logstash.node.stats.logstash.ephemeral_id | | keyword | | -| logstash.node.stats.logstash.host | | keyword | | -| logstash.node.stats.logstash.http_address | | keyword | | -| logstash.node.stats.logstash.name | | keyword | | -| logstash.node.stats.logstash.pipeline.batch_delay | | long | gauge | -| logstash.node.stats.logstash.pipeline.batch_size | | long | gauge | -| logstash.node.stats.logstash.pipeline.workers | | long | gauge | -| logstash.node.stats.logstash.pipelines | | keyword | | -| logstash.node.stats.logstash.snapshot | | boolean | | -| logstash.node.stats.logstash.status | | keyword | | -| logstash.node.stats.logstash.uuid | | keyword | | -| logstash.node.stats.logstash.version | | keyword | | -| logstash.node.stats.os.cgroup.cpu.cfs_quota_micros | | long | gauge | -| logstash.node.stats.os.cgroup.cpu.control_group | | text | | -| logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods | | long | gauge | -| logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled | | long | counter | -| logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos | | long | counter | -| logstash.node.stats.os.cgroup.cpuacct.control_group | | text | | -| logstash.node.stats.os.cgroup.cpuacct.usage_nanos | | long | counter | -| logstash.node.stats.os.cpu.load_average.15m | | half_float | gauge | -| logstash.node.stats.os.cpu.load_average.1m | | half_float | gauge | -| logstash.node.stats.os.cpu.load_average.5m | | half_float | gauge | -| logstash.node.stats.os.cpu.percent | | double | gauge | -| logstash.node.stats.os.cpu.total_in_millis | | long | counter | -| logstash.node.stats.pipelines.ephemeral_id | | keyword | | -| logstash.node.stats.pipelines.events.duration_in_millis | | long | | -| logstash.node.stats.pipelines.events.filtered | | long | | -| logstash.node.stats.pipelines.events.in | | long | | -| logstash.node.stats.pipelines.events.out | | long | | -| logstash.node.stats.pipelines.events.queue_push_duration_in_millis | | long | | -| logstash.node.stats.pipelines.hash | | keyword | | -| logstash.node.stats.pipelines.id | | keyword | | -| logstash.node.stats.pipelines.queue.events_count | | long | | -| logstash.node.stats.pipelines.queue.max_queue_size_in_bytes | | long | | -| logstash.node.stats.pipelines.queue.queue_size_in_bytes | | long | | -| logstash.node.stats.pipelines.queue.type | | keyword | | -| logstash.node.stats.pipelines.reloads.failures | | long | | -| logstash.node.stats.pipelines.reloads.successes | | long | | -| logstash.node.stats.process.cpu.load_average.15m | | half_float | gauge | -| logstash.node.stats.process.cpu.load_average.1m | | half_float | gauge | -| logstash.node.stats.process.cpu.load_average.5m | | half_float | gauge | -| logstash.node.stats.process.cpu.percent | | double | gauge | -| logstash.node.stats.process.cpu.total_in_millis | | long | counter | -| logstash.node.stats.process.max_file_descriptors | | long | gauge | -| logstash.node.stats.process.mem.total_virtual_in_bytes | | long | gauge | -| logstash.node.stats.process.open_file_descriptors | | long | gauge | -| logstash.node.stats.process.peak_open_file_descriptors | | long | gauge | -| logstash.node.stats.queue.events_count | | long | counter | -| logstash.node.stats.reloads.failures | | long | counter | -| logstash.node.stats.reloads.successes | | long | counter | -| logstash.node.stats.timestamp | | date | | -| logstash.pipeline.name | | alias | | -| process.pid | Process id. | long | | -| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | -| service.hostname | Hostname of the service | keyword | | -| service.id | Unique identifier of the running service. If the service is comprised of many nodes, the `service.id` should be the same for all nodes. This id should uniquely identify the service. This makes it possible to correlate logs and metrics for one specific service, no matter which particular node emitted the event. Note that if you need to see the events from one specific host of the service, you should filter on that `host.name` or `host.id` instead. | keyword | | -| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | -| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | | - - -An example event for `node_cel` looks as following: - -```json -{ - "logstash": { - "node": { - "stats": { - "jvm": { - "mem": { - "heap_committed_in_bytes": 264241152, - "heap_used_percent": 2, - "heap_max_in_bytes": 5184159742, - "non_heap_committed_in_bytes": 191889408, - "heap_used_in_bytes": 143564464, - "non_heap_used_in_bytes": 180940656 - }, - "threads": { - "count": 83, - "peak_count": 85 - }, - "uptime_in_millis": 448206 - }, - "logstash": { - "pipeline": { - "batch_delay": 50, - "batch_size": 125, - "workers": 8 - }, - "pipelines": [ - "standalone-pipeline", - "pipeline-with-memory-queue", - "pipeline-with-persisted-queue" - ], - "http_address": "0.0.0.0:9600", - "name": "21d61ee7529e", - "host": "21d61ee7529e", - "ephemeral_id": "fa27552b-e31d-463d-a5db-f470e6c2f0ba", - "version": "8.6.0", - "uuid": "2566e68f-ea0e-4dd0-8b65-17bc7bd9f685", - "snapshot": false, - "status": "green" - }, - "process": { - "open_file_descriptors": 94, - "mem": { - "total_virtual_in_bytes": 11442712576 - }, - "max_file_descriptors": 1048576, - "cpu": { - "load_average": { - "5m": 1.49, - "15m": 1.23, - "1m": 0.74 - }, - "total_in_millis": 130690, - "percent": 2 - }, - "peak_open_file_descriptors": 95 - }, - "os": { - "cpu": { - "load_average": { - "5m": 1.49, - "15m": 1.23, - "1m": 0.74 - }, - "total_in_millis": 130690, - "percent": 2 - }, - "cgroup": {} - }, - "events": { - "filtered": 27752, - "in": 28442, - "queue_push_duration_in_millis": 597, - "duration_in_millis": 3202220, - "out": 27752 - }, - "queue": { - "events_count": 0 - }, - "reloads": { - "failures": 0, - "successes": 0 - } - } - } - }, - "input": { - "type": "cel" - }, - "agent": { - "name": "MacBook-Pro.local", - "id": "b88de78b-7bd7-49ae-99d7-f68ea18070c4", - "type": "filebeat", - "ephemeral_id": "e24a6e70-8e93-4d18-8535-319e63c81bc8", - "version": "8.10.1" - }, - "@timestamp": "2023-10-04T18:53:48.769Z", - "ecs": { - "version": "8.0.0" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "logstash.node" - }, - "elastic_agent": { - "id": "b88de78b-7bd7-49ae-99d7-f68ea18070c4", - "version": "8.10.1", - "snapshot": false - }, - "host": { - "hostname": "macbook-pro.local", - "os": { - "build": "22F82", - "kernel": "22.5.0", - "name": "macOS", - "family": "darwin", - "type": "macos", - "version": "13.4.1", - "platform": "darwin" - }, - "ip": [ - "192.168.1.184" - ], - "name": "macbook-pro.local", - "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", - "mac": [ - "AC-DE-48-00-11-22" - ], - "architecture": "x86_64" - }, - "event": { - "agent_id_status": "verified", - "ingested": "2023-10-04T18:53:49Z", - "dataset": "logstash.node" - } -} -``` - -#### Pipeline - -This is the `pipeline` dataset, which drives the Pipeline dashboard pages. - -#### Example - -**Exported fields** - -| Field | Description | Type | Unit | Metric Type | -|---|---|---|---|---| -| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | -| cloud.image.id | Image ID for the cloud instance. | keyword | | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | | -| cloud.instance.name | Instance name of the host machine. | keyword | | | -| cloud.machine.type | Machine type of the host machine. | keyword | | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | -| cloud.region | Region in which this host, resource, or service is located. | keyword | | | -| cluster_uuid | | alias | | | -| container.id | Unique container id. | keyword | | | -| container.image.name | Name of the image the container was built on. | keyword | | | -| container.labels | Image labels. | object | | | -| container.name | Container name. | keyword | | | -| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | | -| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | | -| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | -| error.message | Error message. | match_only_text | | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | -| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | | -| host.architecture | Operating system architecture. | keyword | | | -| host.containerized | If the host is a container. | boolean | | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | -| host.ip | Host ip addresses. | ip | | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | -| host.os.build | OS build information. | keyword | | | -| host.os.codename | OS codename, if any. | keyword | | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | | -| host.os.name | Operating system name, without the version. | keyword | | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | -| host.os.version | Operating system version as a raw string. | keyword | | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | -| input.type | | keyword | | | -| logstash.host.address | | alias | | | -| logstash.host.name | | alias | | | -| logstash.pipeline.elasticsearch.cluster.id | Elasticsearch clusters this Logstash pipeline is attached to | keyword | | | -| logstash.pipeline.host.address | address hosting this instance of logstash | keyword | | | -| logstash.pipeline.host.name | Host name of the node running logstash | keyword | | | -| logstash.pipeline.info.batch_delay | Batch delay for the running pipeline | long | | | -| logstash.pipeline.info.batch_size | Batch size for the running pipeline | long | | | -| logstash.pipeline.info.ephemeral_id | Ephemeral Id for the running pipeline | keyword | | | -| logstash.pipeline.info.workers | Number of workers for the running pipeline | long | | | -| logstash.pipeline.name | Logstash Pipeline id/name | keyword | | | -| logstash.pipeline.total.events.filtered | Number of events filtered by the pipeline | long | | counter | -| logstash.pipeline.total.events.in | Number of events received by the pipeline | long | | counter | -| logstash.pipeline.total.events.out | Number of events emitted by the pipeline | long | | counter | -| logstash.pipeline.total.flow.filter_throughput.current | current value of the filter throughput flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.filter_throughput.last_1_minute | current value of the filter throughput flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.input_throughput.current | current value of the input throughput flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.input_throughput.last_1_minute | current value of the throughput flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.output_throughput.current | current value of the output throughput flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.output_throughput.last_1_minute | current value of the output throughput flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.queue_backpressure.current | current value of the queue backpressure flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.queue_backpressure.last_1_minute | current value of the queue backpressure flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.queue_persisted_growth_bytes.current | current value of the queue persisted growth bytes flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.queue_persisted_growth_bytes.last_1_minute | current value of the queue persisted growth bytes flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.queue_persisted_growth_events.current | current value of the queue persisted growth events flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.queue_persisted_growth_events.last_1_minute | current value of the queue persisted growth events flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.worker_concurrency.current | last 1 minute value of the worker utilization flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.worker_concurrency.last_1_minute | current value of the worker concurrency flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.worker_utilization.current | last 1 minute value of the worker concurrency flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.worker_utilization.last_1_minute | current value of the worker concurrency flow metric | scaled_float | | gauge | -| logstash.pipeline.total.queues.current_size.bytes | Current size of the PQ | long | byte | gauge | -| logstash.pipeline.total.queues.events | Number of events in the PQ for this pipeline | long | | counter | -| logstash.pipeline.total.queues.max_size.bytes | Maximum possible size of the PQ | long | | gauge | -| logstash.pipeline.total.queues.type | Type of queue - persistent or memory | keyword | | | -| logstash.pipeline.total.reloads.failures | Number of failed reloads for this pipeline | long | | counter | -| logstash.pipeline.total.reloads.successes | Number of successful reloads for this pipeline | long | | counter | -| logstash.pipeline.total.time.duration.ms | Time spent processing events through the pipeline. | long | ms | counter | -| logstash.pipeline.total.time.queue_push_duration.ms | Time spent pushing events to the queue for this pipeline. | long | ms | counter | -| process.pid | Process id. | long | | | -| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | -| service.hostname | Hostname of the service | keyword | | | -| service.id | Unique identifier of the running service. If the service is comprised of many nodes, the `service.id` should be the same for all nodes. This id should uniquely identify the service. This makes it possible to correlate logs and metrics for one specific service, no matter which particular node emitted the event. Note that if you need to see the events from one specific host of the service, you should filter on that `host.name` or `host.id` instead. | keyword | | | -| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | -| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | | | - - -An example event for `pipeline` looks as following: - -```json -{ - "@timestamp": "2023-10-04T18:53:18.708Z", - "data_stream": { - "dataset": "logstash.pipeline", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "agent_id_status": "verified", - "dataset": "logstash.pipeline", - "ingested": "2023-10-04T18:53:19Z" - }, - "host": { - "architecture": "x86_64", - "hostname": "macbook-pro.local", - "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", - "ip": [ - "192.168.1.184" - ], - "mac": [ - "AC-DE-48-00-11-22" - ], - "name": "macbook-pro.local", - "os": { - "build": "22F82", - "family": "darwin", - "kernel": "22.5.0", - "name": "macOS", - "platform": "darwin", - "version": "13.4.1" - } - }, - "input": { - "type": "cel" - }, - "logstash": { - "pipeline": { - "host": { - "address": "0.0.0.0:9600", - "name": "21d61ee7529e" - }, - "name": "standalone-pipeline", - "total": { - "events": { - "filtered": 2038, - "in": 2038, - "out": 2038 - }, - "flow": { - "filter_throughput": { - "current": 5.02, - "last_1_minute": 5.003 - }, - "input_throughput": { - "current": 4.948, - "last_1_minute": 5.003 - }, - "output_throughput": { - "current": 5.02, - "last_1_minute": 5.003 - }, - "queue_backpressure": { - "current": 0, - "last_1_minute": 0 - }, - "worker_concurrency": { - "current": 0.001, - "last_1_minute": 0.001 - } - }, - "queues": { - "current_size": { - "bytes": 0 - }, - "events": 0, - "max_size": { - "bytes": 0 - }, - "type": "memory" - }, - "reloads": { - "failures": 0, - "successes": 0 - }, - "time": { - "duration": { - "ms": 1363 - }, - "queue_push_duration": { - "ms": 12 - } - } - } - } - } -} -``` - -#### Plugin - -This is the `plugin` dataset, which drives the Pipeline detail dashboard pages. Note that this dataset may produce many documents for logstash instances using a large number of pipelines and/or plugins within those pipelines. For those instances, we recommend reviewing the -pipeline collection period, and setting it to an appropriate value. - -#### Example - -**Exported fields** - -| Field | Description | Type | Unit | Metric Type | -|---|---|---|---|---| -| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | -| cloud.image.id | Image ID for the cloud instance. | keyword | | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | | -| cloud.instance.name | Instance name of the host machine. | keyword | | | -| cloud.machine.type | Machine type of the host machine. | keyword | | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | -| cloud.region | Region in which this host, resource, or service is located. | keyword | | | -| cluster_uuid | | alias | | | -| container.id | Unique container id. | keyword | | | -| container.image.name | Name of the image the container was built on. | keyword | | | -| container.labels | Image labels. | object | | | -| container.name | Container name. | keyword | | | -| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | | -| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | | -| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | -| error.message | Error message. | match_only_text | | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | -| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | | -| host.architecture | Operating system architecture. | keyword | | | -| host.containerized | If the host is a container. | boolean | | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | -| host.ip | Host ip addresses. | ip | | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | -| host.os.build | OS build information. | keyword | | | -| host.os.codename | OS codename, if any. | keyword | | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | | -| host.os.name | Operating system name, without the version. | keyword | | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | -| host.os.version | Operating system version as a raw string. | keyword | | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | -| input.type | | keyword | | | -| logstash.host.address | | alias | | | -| logstash.host.name | | alias | | | -| logstash.pipeline.elasticsearch.cluster.id | Elasticsearch clusters this Logstash pipeline is attached to | keyword | | | -| logstash.pipeline.host.address | address hosting this instance of logstash | keyword | | | -| logstash.pipeline.host.name | Host name of the node running logstash | keyword | | | -| logstash.pipeline.id | Logstash Pipeline hash | keyword | | | -| logstash.pipeline.name | Logstash Pipeline id/name | keyword | | | -| logstash.pipeline.plugin.codec.decode.duration.ms | amount of time spend decoding events | long | ms | counter | -| logstash.pipeline.plugin.codec.decode.in | number of events entering the decoder | long | | counter | -| logstash.pipeline.plugin.codec.decode.out | number of events exiting the decoder | long | | counter | -| logstash.pipeline.plugin.codec.encode.duration.ms | amount of time spend encoding events | long | ms | counter | -| logstash.pipeline.plugin.codec.encode.in | number of events encoded | long | | counter | -| logstash.pipeline.plugin.codec.id | Id of codec plugin | keyword | | | -| logstash.pipeline.plugin.codec.name | Name of codec plugin | keyword | | | -| logstash.pipeline.plugin.filter.elasticsearch.cluster.id | Elasticsearch clusters this Logstash plugin is attached to | keyword | | | -| logstash.pipeline.plugin.filter.events.in | number of events received by the filter | long | | counter | -| logstash.pipeline.plugin.filter.events.out | number of events emitted by the filter | long | | counter | -| logstash.pipeline.plugin.filter.flow.worker_millis_per_event.current | amount of time spent per event for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute | amount of time spent per event for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.filter.flow.worker_utilization.current | worker utilization for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.filter.flow.worker_utilization.last_1_minute | worker utilization for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.filter.id | Id of filter plugin | keyword | | | -| logstash.pipeline.plugin.filter.metrics.dissect.failures | number of dissect failures | long | | counter | -| logstash.pipeline.plugin.filter.metrics.dissect.matches | number of dissect matches | long | | counter | -| logstash.pipeline.plugin.filter.metrics.grok.failures | number of grok failures | long | | counter | -| logstash.pipeline.plugin.filter.metrics.grok.matches | number of grok matches | long | | counter | -| logstash.pipeline.plugin.filter.name | Name of filter plugin | keyword | | | -| logstash.pipeline.plugin.filter.source.column | | keyword | | | -| logstash.pipeline.plugin.filter.source.id | | keyword | | | -| logstash.pipeline.plugin.filter.source.line | | long | | | -| logstash.pipeline.plugin.filter.source.protocol | | keyword | | | -| logstash.pipeline.plugin.filter.time.duration.ms | amount of time working on events in this plugin | long | ms | counter | -| logstash.pipeline.plugin.input.elasticsearch.cluster.id | Elasticsearch clusters this Logstash plugin is attached to | keyword | | | -| logstash.pipeline.plugin.input.events.out | number of events emitted by the input | long | | counter | -| logstash.pipeline.plugin.input.flow.throughput.current | throughput of this input plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.input.flow.throughput.last_1_minute | throughput of this input plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.input.id | Id of input plugin | keyword | | | -| logstash.pipeline.plugin.input.name | Name of input plugin | keyword | | | -| logstash.pipeline.plugin.input.source.column | | keyword | | | -| logstash.pipeline.plugin.input.source.id | | keyword | | | -| logstash.pipeline.plugin.input.source.line | | long | | | -| logstash.pipeline.plugin.input.source.protocol | | keyword | | | -| logstash.pipeline.plugin.input.time.queue_push_duration.ms | amount of time spend pushing events to the queue | long | ms | counter | -| logstash.pipeline.plugin.output.elasticsearch.cluster.id | Elasticsearch clusters this Logstash plugin is attached to | keyword | | | -| logstash.pipeline.plugin.output.events.in | number of events received by the output | long | | counter | -| logstash.pipeline.plugin.output.events.out | number of events emitted by the output | long | | counter | -| logstash.pipeline.plugin.output.flow.worker_millis_per_event.current | amount of time spent per event for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_1_minute | amount of time spent per event for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.output.flow.worker_utilization.current | worker utilization for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute | worker utilization for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.output.id | Id of output plugin | keyword | | | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.200 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.201 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.400 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.401 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.403 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.404 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.409 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.413 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.429 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.500 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.successes | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.documents.non_retryable_failures | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.documents.successes | | long | | counter | -| logstash.pipeline.plugin.output.name | Name of output plugin | keyword | | | -| logstash.pipeline.plugin.output.source.column | | keyword | | | -| logstash.pipeline.plugin.output.source.id | | keyword | | | -| logstash.pipeline.plugin.output.source.line | | long | | | -| logstash.pipeline.plugin.output.source.protocol | | keyword | | | -| logstash.pipeline.plugin.output.time.duration.ms | amount of time working on events in this plugin | long | ms | counter | -| logstash.pipeline.plugin.type | Type of the plugin | keyword | | | -| process.pid | Process id. | long | | | -| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | -| service.hostname | Hostname of the service | keyword | | | -| service.id | Unique identifier of the running service. If the service is comprised of many nodes, the `service.id` should be the same for all nodes. This id should uniquely identify the service. This makes it possible to correlate logs and metrics for one specific service, no matter which particular node emitted the event. Note that if you need to see the events from one specific host of the service, you should filter on that `host.name` or `host.id` instead. | keyword | | | -| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | -| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | | | - +{{event "node"}} -An example event for `plugins` looks as following: -```json -{ - "@timestamp": "2023-10-24T17:56:40.316Z", - "data_stream": { - "dataset": "logstash.plugins", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "agent_id_status": "verified", - "dataset": "logstash.plugins", - "ingested": "2023-10-24T17:56:41Z" - }, - "host": { - "architecture": "x86_64", - "hostname": "macbook-pro.local", - "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", - "ip": [ - "192.168.4.26" - ], - "mac": [ - "AC-DE-48-00-11-22" - ], - "name": "macbook-pro.local", - "os": { - "build": "22G120", - "family": "darwin", - "kernel": "22.6.0", - "name": "macOS", - "platform": "darwin", - "version": "13.6" - } - }, - "input": { - "type": "cel" - }, - "logstash": { - "pipeline": { - "elasticsearch": { - "cluster": { - "id": "9MOGoKiESvaklNVmxLo3iA" - } - }, - "host": { - "address": "127.0.0.1:9602", - "name": "logstash9602" - }, - "id": "b18ff60bcd82055aab2bf5601a2bc170502f80b33ab5938f25fa95ec8b04cd4b", - "name": "work", - "plugin": { - "output": { - "elasticsearch": { - "cluster": { - "id": "9MOGoKiESvaklNVmxLo3iA" - } - }, - "events": { - "in": 798, - "out": 798 - }, - "flow": { - "worker_millis_per_event": { - "current": 54, - "last_1_minute": 54 - }, - "worker_utilization": { - "current": 0.023, - "last_1_minute": 0.01 - } - }, - "id": "out_to_elasticsearch", - "metrics": { - "elasticsearch": { - "bulk_requests": { - "responses": { - "200": 798 - }, - "successes": 798 - }, - "documents": { - "successes": 798 - } - } - }, - "name": "elasticsearch", - "source": { - "column": "3", - "id": "/Users/test/ingestdemo/logstash-8.8.2/remap.conf", - "line": 132, - "protocol": "file" - }, - "time": { - "duration": { - "ms": 198060 - } - } - }, - "type": "output" - } - } - } -} -``` diff --git a/packages/logstash/changelog.yml b/packages/logstash/changelog.yml index cd2e309d567..c021f459c97 100644 --- a/packages/logstash/changelog.yml +++ b/packages/logstash/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.5.0" + changes: + - description: Adding data and dashboards from Logstash Health Report + type: enhancement + link: https://github.com/elastic/integrations/pull/12464 - version: "2.4.12" changes: - description: Update documentation for GA of agent based monitoring diff --git a/packages/logstash/data_stream/health_report/agent/stream/stream.yml.hbs b/packages/logstash/data_stream/health_report/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..5227b1e1602 --- /dev/null +++ b/packages/logstash/data_stream/health_report/agent/stream/stream.yml.hbs @@ -0,0 +1,63 @@ +config_version: "1" +interval: {{period}} +resource.url: "{{url}}/_health_report" +{{#if resource_ssl}} +resource.ssl: + {{resource_ssl}} +{{/if}} + +{{#if username}} +auth.basic.user: {{escape_string username}} +{{/if}} +{{#if password}} +auth.basic.password: {{escape_string password}} +{{/if}} +{{#if condition}} +condition: {{ condition }} +{{/if}} + +redact: + fields: ~ + +program: | + get(state.url) + .as(resp, bytes(resp.Body) + .decode_json().as(body, + [ + [{ + "logstash.node": { + "name":body.name, + "address":body.http_address, + "uuid":body.id, + "version":body.version, + "status":body.status, + "symptom":body.symptom, + }, + }], + body.indicators.pipelines.indicators.as(pipelines, // pipelines = body.indicators.pipelines.indicators + pipelines.map(pipeline_name, pipelines[pipeline_name].as(pipeline, { // pipeline = pipelines[pipeline_name] + "logstash": { + "node": { + "name": body.name, + "version": body.version, + "address": body.http_address, + "uuid": body.id + }, + "pipeline": { + "id":pipeline_name, + "status":pipeline.status, + "symptom":pipeline.symptom, + "state":pipeline.details.status.state, + "flow": pipeline.details.flow, + "diagnosis": has(pipeline.diagnosis) ? pipeline.diagnosis[0] : {}, + "impacts": has(pipeline.impacts) ? pipeline.impacts[0] : {}, + } + } + } + )) + ) + ].as(entries, { + "events": entries.flatten() + }) + ) + ) \ No newline at end of file diff --git a/packages/logstash/data_stream/health_report/fields/base-fields.yml b/packages/logstash/data_stream/health_report/fields/base-fields.yml new file mode 100644 index 00000000000..1bfb71226cf --- /dev/null +++ b/packages/logstash/data_stream/health_report/fields/base-fields.yml @@ -0,0 +1,9 @@ +- name: data_stream.type + external: ecs +- name: data_stream.dataset + external: ecs +- name: data_stream.namespace + external: ecs +- name: service.hostname + type: keyword + description: Hostname of the service diff --git a/packages/logstash/data_stream/health_report/fields/ecs.yml b/packages/logstash/data_stream/health_report/fields/ecs.yml new file mode 100644 index 00000000000..29b7c409f04 --- /dev/null +++ b/packages/logstash/data_stream/health_report/fields/ecs.yml @@ -0,0 +1,24 @@ +- name: '@timestamp' + external: ecs +- name: service.id + external: ecs +- name: service.type + external: ecs +- name: service.version + external: ecs +- name: service.address + external: ecs +- name: service.name + external: ecs +- name: process.pid + external: ecs +- name: ecs.version + external: ecs +- name: event.dataset + external: ecs +- name: event.duration + external: ecs +- name: event.module + external: ecs +- name: error.message + external: ecs diff --git a/packages/logstash/data_stream/health_report/fields/fields.yml b/packages/logstash/data_stream/health_report/fields/fields.yml new file mode 100644 index 00000000000..558d6e8ce0d --- /dev/null +++ b/packages/logstash/data_stream/health_report/fields/fields.yml @@ -0,0 +1,90 @@ +- name: logstash + type: group + fields: + - name: node + type: group + fields: + - name: name + type: keyword + - name: version + type: keyword + - name: address + type: keyword + - name: symptom + type: keyword + multi_fields: + - name: text + type: match_only_text + - name: uuid + type: keyword + - name: status + type: keyword + - name: pipeline + type: group + fields: + - name: id + type: keyword + - name: status + type: keyword + - name: state + type: keyword + - name: symptom + type: keyword + multi_fields: + - name: text + type: match_only_text + - name: diagnosis + type: group + fields: + - name: id + type: keyword + - name: cause + type: keyword + multi_fields: + - name: text + type: match_only_text + - name: action + type: keyword + multi_fields: + - name: text + type: match_only_text + - name: help_url + type: keyword + format: url + multi_fields: + - name: text + type: match_only_text + - name: impacts + type: group + fields: + - name: id + type: keyword + - name: severity + type: short + - name: description + type: keyword + multi_fields: + - name: text + type: match_only_text + - name: impact_areas + type: keyword + - name: flow + type: group + fields: + - name: worker_utilization + type: group + fields: + - name: current + type: float + - name: last_1_hour + type: float + - name: last_5_minutes + type: float + - name: last_15_minutes + type: float + - name: lifetime + type: float + - name: last_1_minute + type: float + - name: last_24_hours + type: float diff --git a/packages/logstash/data_stream/health_report/manifest.yml b/packages/logstash/data_stream/health_report/manifest.yml new file mode 100644 index 00000000000..b9e80b79082 --- /dev/null +++ b/packages/logstash/data_stream/health_report/manifest.yml @@ -0,0 +1,20 @@ +title: "Health Report" +type: metrics +dataset: logstash.health_report +release: beta +elasticsearch: + index_template: + mappings: + dynamic: false +streams: + - input: cel + title: "Collect Health Report" + description: "Health Report for Logstash instance" + vars: + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 30s diff --git a/packages/logstash/data_stream/node/manifest.yml b/packages/logstash/data_stream/node/manifest.yml index 3a586c4af1c..97f506e171e 100644 --- a/packages/logstash/data_stream/node/manifest.yml +++ b/packages/logstash/data_stream/node/manifest.yml @@ -8,6 +8,7 @@ elasticsearch: streams: - input: logstash/metrics title: Logstash node metrics + enabled: false description: Collect Logstash node metrics vars: - name: period diff --git a/packages/logstash/data_stream/node_cel/manifest.yml b/packages/logstash/data_stream/node_cel/manifest.yml index 000f7f19f94..1c0d5c38a7f 100644 --- a/packages/logstash/data_stream/node_cel/manifest.yml +++ b/packages/logstash/data_stream/node_cel/manifest.yml @@ -7,7 +7,6 @@ elasticsearch: dynamic: false streams: - input: cel - enabled: false title: "Collect Node Metrics" description: "Collect Metrics related to Nodes running Logstash" template_path: cel.yml.hbs diff --git a/packages/logstash/data_stream/node_stats/manifest.yml b/packages/logstash/data_stream/node_stats/manifest.yml index 450f8f13bd7..823dea0e578 100644 --- a/packages/logstash/data_stream/node_stats/manifest.yml +++ b/packages/logstash/data_stream/node_stats/manifest.yml @@ -8,6 +8,7 @@ elasticsearch: streams: - input: logstash/metrics title: Logstash node stats metrics + enabled: false description: Collect Logstash node stats metrics vars: - name: period diff --git a/packages/logstash/data_stream/pipeline/manifest.yml b/packages/logstash/data_stream/pipeline/manifest.yml index ddbb9b41cd1..21e245259da 100644 --- a/packages/logstash/data_stream/pipeline/manifest.yml +++ b/packages/logstash/data_stream/pipeline/manifest.yml @@ -1,6 +1,5 @@ type: metrics title: Logstash pipeline -release: experimental elasticsearch: index_mode: "time_series" index_template: @@ -8,7 +7,6 @@ elasticsearch: dynamic: false streams: - input: cel - enabled: false title: "Collect Pipeline Metrics" description: "Collect Metrics related to Logstash Pipeline usage" template_path: cel.yml.hbs diff --git a/packages/logstash/data_stream/plugins/manifest.yml b/packages/logstash/data_stream/plugins/manifest.yml index 18d575c2a2e..037388e1d5f 100644 --- a/packages/logstash/data_stream/plugins/manifest.yml +++ b/packages/logstash/data_stream/plugins/manifest.yml @@ -1,6 +1,5 @@ type: metrics title: Logstash plugins -release: experimental elasticsearch: index_mode: "time_series" index_template: @@ -8,7 +7,6 @@ elasticsearch: dynamic: false streams: - input: cel - enabled: false title: "Collect Plugin Metrics" description: "Collect metrics for Logstash plugin use. Note that large pipelines will increase the volume of plugin metrics, and a slower rate of collection may be appropriate" template_path: cel.yml.hbs diff --git a/packages/logstash/docs/README.md b/packages/logstash/docs/README.md index 200c38117c6..e6b4d92ee51 100644 --- a/packages/logstash/docs/README.md +++ b/packages/logstash/docs/README.md @@ -16,274 +16,370 @@ By utilizing Elastic Agent we are able to query additional monitoring APIs and p Elastic Agent based metrics collection is not compatible with the Stack Monitoring UI inside Kibana, please only select Metrics (Elastic Agent). Users that prefer the Stack Monitoring UI should uncheck `Metrics (Elastic Agent)` and continue to use `Metrics (Stack Monitoring)`. +### Fields and Sample Events -## Logs - -Logstash package supports the plain text format and the JSON format. Also, two types of -logs can be activated with the Logstash package: - -* `log` collects and parses the logs that Logstash writes to disk. -* `slowlog` parses the logstash slowlog (make sure to configure the Logstash slowlog option). - -#### Known issues - -When using the `log` data stream to parse plaintext logs, if a multiline plaintext log contains an embedded JSON object such that -the JSON object starts on a new line, the fileset may not parse the multiline plaintext log event correctly. +#### Health Report +The health report api is available starting with Logstash 8.16.0, which provides the `health_report` dataset for Node health and Pipeline health dashboards +#### Example +An example event for 'health_report' looks as following: -## Metrics +**Exported fields** -Logstash metric related data streams works with Logstash 7.3.0 and later. +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | +| logstash.node.address | | keyword | +| logstash.node.name | | keyword | +| logstash.node.status | | keyword | +| logstash.node.symptom | | keyword | +| logstash.node.symptom.text | Multi-field of `logstash.node.symptom`. | match_only_text | +| logstash.node.uuid | | keyword | +| logstash.node.version | | keyword | +| logstash.pipeline.diagnosis.action | | keyword | +| logstash.pipeline.diagnosis.action.text | Multi-field of `logstash.pipeline.diagnosis.action`. | match_only_text | +| logstash.pipeline.diagnosis.cause | | keyword | +| logstash.pipeline.diagnosis.cause.text | Multi-field of `logstash.pipeline.diagnosis.cause`. | match_only_text | +| logstash.pipeline.diagnosis.help_url | | keyword | +| logstash.pipeline.diagnosis.help_url.text | Multi-field of `logstash.pipeline.diagnosis.help_url`. | match_only_text | +| logstash.pipeline.diagnosis.id | | keyword | +| logstash.pipeline.flow.worker_utilization.current | | float | +| logstash.pipeline.flow.worker_utilization.last_15_minutes | | float | +| logstash.pipeline.flow.worker_utilization.last_1_hour | | float | +| logstash.pipeline.flow.worker_utilization.last_1_minute | | float | +| logstash.pipeline.flow.worker_utilization.last_24_hours | | float | +| logstash.pipeline.flow.worker_utilization.last_5_minutes | | float | +| logstash.pipeline.flow.worker_utilization.lifetime | | float | +| logstash.pipeline.id | | keyword | +| logstash.pipeline.impacts.description | | keyword | +| logstash.pipeline.impacts.description.text | Multi-field of `logstash.pipeline.impacts.description`. | match_only_text | +| logstash.pipeline.impacts.id | | keyword | +| logstash.pipeline.impacts.impact_areas | | keyword | +| logstash.pipeline.impacts.severity | | short | +| logstash.pipeline.state | | keyword | +| logstash.pipeline.status | | keyword | +| logstash.pipeline.symptom | | keyword | +| logstash.pipeline.symptom.text | Multi-field of `logstash.pipeline.symptom`. | match_only_text | +| process.pid | Process id. | long | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | +| service.hostname | Hostname of the service | keyword | +| service.id | Unique identifier of the running service. If the service is comprised of many nodes, the `service.id` should be the same for all nodes. This id should uniquely identify the service. This makes it possible to correlate logs and metrics for one specific service, no matter which particular node emitted the event. Note that if you need to see the events from one specific host of the service, you should filter on that `host.name` or `host.id` instead. | keyword | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | -### Node Stats -An example event for `node_stats` looks as following: +An example event for `health_report` looks as following: ```json { - "@timestamp": "2023-03-02T15:57:56.968Z", - "agent": { - "ephemeral_id": "16f2dd63-454b-4699-a8c8-2a748bd044b8", - "id": "3cc85092-54dc-4b58-8726-5e9458167f42", - "name": "docker-fleet-agent", - "type": "metricbeat", - "version": "8.5.0" + "logstash": { + "pipeline": { + "symptom": "The pipeline is concerning; 1 area is impacted and 1 diagnosis is available", + "diagnosis": { + "help_url": "https://www.elastic.co/guide/en/logstash/8.17/health-report-pipeline-status.html#finished", + "cause": "pipeline has finished running because its inputs have been closed and events have been processed", + "action": "if you expect this pipeline to run indefinitely, you will need to configure its inputs to continue receiving or fetching events", + "id": "logstash:health:pipeline:status:diagnosis:finished" + }, + "id": "self-closer", + "state": "FINISHED", + "impacts": { + "severity": 10, + "impact_areas": [ + "pipeline_execution" + ], + "description": "pipeline has finished running", + "id": "logstash:health:pipeline:status:impact:not_processing" + }, + "flow": { + "worker_utilization": { + "current": 0.0009642, + "last_24_hours": 0.0009642, + "last_5_minutes": 0.0009642, + "last_1_hour": 0.0009642, + "last_15_minutes": 0.0009642, + "lifetime": 0.0009642, + "last_1_minute": 0.0009642 + } + }, + "status": "yellow" + }, + "node": { + "address": "0.0.0.0:9600", + "name": "87f8aa570fcb", + "uuid": "8c2afc7e-a64f-42f3-9ab9-5e16dc95c9bc", + "version": "8.17.1" + } }, - "data_stream": { - "dataset": "logstash.stack_monitoring.node_stats", - "namespace": "ep", - "type": "metrics" + "agent": { + "name": "a9f1b9c5936b", + "id": "af72217c-8c4d-427f-8c92-6b4566e9937f", + "type": "filebeat", + "ephemeral_id": "40ea6231-3856-4b90-8083-73b30558cfe7", + "version": "8.17.1" }, + "@timestamp": "2025-01-28T18:41:24.669Z", "ecs": { "version": "8.0.0" }, - "elastic_agent": { - "id": "3cc85092-54dc-4b58-8726-5e9458167f42", - "snapshot": false, - "version": "8.5.0" - }, - "event": { - "agent_id_status": "verified", - "dataset": "logstash.stack_monitoring.node_stats", - "duration": 48419400, - "ingested": "2023-03-02T15:57:58Z", - "module": "logstash" + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "logstash.health_report" }, "host": { - "architecture": "x86_64", - "containerized": true, - "hostname": "docker-fleet-agent", - "id": "66392b0697b84641af8006d87aeb89f1", - "ip": [ - "192.168.224.7" - ], - "mac": [ - "02-42-C0-A8-E0-07" - ], - "name": "docker-fleet-agent", + "hostname": "a9f1b9c5936b", "os": { + "kernel": "6.10.14-linuxkit", "codename": "focal", - "family": "debian", - "kernel": "5.10.47-linuxkit", "name": "Ubuntu", - "platform": "ubuntu", + "family": "debian", "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)" - } + "version": "20.04.6 LTS (Focal Fossa)", + "platform": "ubuntu" + }, + "containerized": false, + "ip": [ + "172.17.0.3" + ], + "name": "a9f1b9c5936b", + "mac": [ + "02-42-AC-11-00-03" + ], + "architecture": "aarch64" + }, + "elastic_agent": { + "id": "af72217c-8c4d-427f-8c92-6b4566e9937f", + "version": "8.17.1", + "snapshot": false }, + "event": { + "agent_id_status": "verified", + "ingested": "2025-01-28T18:41:28Z", + "dataset": "logstash.health_report" + } +} +``` + +#### Node + +This is the `node` dataset, which drives the Node dashboard pages. + +#### Example + +**Exported fields** + +| Field | Description | Type | Metric Type | +|---|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | +| cloud.image.id | Image ID for the cloud instance. | keyword | | +| cloud.instance.id | Instance ID of the host machine. | keyword | | +| cloud.instance.name | Instance name of the host machine. | keyword | | +| cloud.machine.type | Machine type of the host machine. | keyword | | +| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | +| cluster_uuid | | alias | | +| container.id | Unique container id. | keyword | | +| container.image.name | Name of the image the container was built on. | keyword | | +| container.labels | Image labels. | object | | +| container.name | Container name. | keyword | | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | +| error.message | Error message. | match_only_text | | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | +| host.architecture | Operating system architecture. | keyword | | +| host.containerized | If the host is a container. | boolean | | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | +| host.ip | Host ip addresses. | ip | | +| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | +| host.os.build | OS build information. | keyword | | +| host.os.codename | OS codename, if any. | keyword | | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | | +| host.os.name | Operating system name, without the version. | keyword | | +| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | +| host.os.version | Operating system version as a raw string. | keyword | | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | +| input.type | | keyword | | +| logstash.elasticsearch.cluster.id | | keyword | | +| logstash.host.address | | alias | | +| logstash.host.name | | alias | | +| logstash.node.stats.events.duration_in_millis | | long | counter | +| logstash.node.stats.events.filtered | Filtered events counter | long | counter | +| logstash.node.stats.events.in | Incoming events counter | long | counter | +| logstash.node.stats.events.out | Outgoing events counter | long | counter | +| logstash.node.stats.events.queue_push_duration_in_millis | | long | counter | +| logstash.node.stats.jvm.gc.collectors.old.collection_count | | long | counter | +| logstash.node.stats.jvm.gc.collectors.old.collection_time_in_millis | | long | counter | +| logstash.node.stats.jvm.gc.collectors.young.collection_count | | long | counter | +| logstash.node.stats.jvm.gc.collectors.young.collection_time_in_millis | | long | counter | +| logstash.node.stats.jvm.mem.heap_committed_in_bytes | | long | gauge | +| logstash.node.stats.jvm.mem.heap_max_in_bytes | | long | counter | +| logstash.node.stats.jvm.mem.heap_used_in_bytes | | long | gauge | +| logstash.node.stats.jvm.mem.heap_used_percent | | long | gauge | +| logstash.node.stats.jvm.mem.non_heap_committed_in_bytes | | long | gauge | +| logstash.node.stats.jvm.mem.non_heap_used_in_bytes | | long | gauge | +| logstash.node.stats.jvm.threads.count | current number of threads | long | counter | +| logstash.node.stats.jvm.threads.peak_count | peak number of threads | long | counter | +| logstash.node.stats.jvm.uptime_in_millis | | long | counter | +| logstash.node.stats.logstash.ephemeral_id | | keyword | | +| logstash.node.stats.logstash.host | | keyword | | +| logstash.node.stats.logstash.http_address | | keyword | | +| logstash.node.stats.logstash.name | | keyword | | +| logstash.node.stats.logstash.pipeline.batch_delay | | long | gauge | +| logstash.node.stats.logstash.pipeline.batch_size | | long | gauge | +| logstash.node.stats.logstash.pipeline.workers | | long | gauge | +| logstash.node.stats.logstash.pipelines | | keyword | | +| logstash.node.stats.logstash.snapshot | | boolean | | +| logstash.node.stats.logstash.status | | keyword | | +| logstash.node.stats.logstash.uuid | | keyword | | +| logstash.node.stats.logstash.version | | keyword | | +| logstash.node.stats.os.cgroup.cpu.cfs_quota_micros | | long | gauge | +| logstash.node.stats.os.cgroup.cpu.control_group | | text | | +| logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods | | long | gauge | +| logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled | | long | counter | +| logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos | | long | counter | +| logstash.node.stats.os.cgroup.cpuacct.control_group | | text | | +| logstash.node.stats.os.cgroup.cpuacct.usage_nanos | | long | counter | +| logstash.node.stats.os.cpu.load_average.15m | | half_float | gauge | +| logstash.node.stats.os.cpu.load_average.1m | | half_float | gauge | +| logstash.node.stats.os.cpu.load_average.5m | | half_float | gauge | +| logstash.node.stats.os.cpu.percent | | double | gauge | +| logstash.node.stats.os.cpu.total_in_millis | | long | counter | +| logstash.node.stats.pipelines.ephemeral_id | | keyword | | +| logstash.node.stats.pipelines.events.duration_in_millis | | long | | +| logstash.node.stats.pipelines.events.filtered | | long | | +| logstash.node.stats.pipelines.events.in | | long | | +| logstash.node.stats.pipelines.events.out | | long | | +| logstash.node.stats.pipelines.events.queue_push_duration_in_millis | | long | | +| logstash.node.stats.pipelines.hash | | keyword | | +| logstash.node.stats.pipelines.id | | keyword | | +| logstash.node.stats.pipelines.queue.events_count | | long | | +| logstash.node.stats.pipelines.queue.max_queue_size_in_bytes | | long | | +| logstash.node.stats.pipelines.queue.queue_size_in_bytes | | long | | +| logstash.node.stats.pipelines.queue.type | | keyword | | +| logstash.node.stats.pipelines.reloads.failures | | long | | +| logstash.node.stats.pipelines.reloads.successes | | long | | +| logstash.node.stats.process.cpu.load_average.15m | | half_float | gauge | +| logstash.node.stats.process.cpu.load_average.1m | | half_float | gauge | +| logstash.node.stats.process.cpu.load_average.5m | | half_float | gauge | +| logstash.node.stats.process.cpu.percent | | double | gauge | +| logstash.node.stats.process.cpu.total_in_millis | | long | counter | +| logstash.node.stats.process.max_file_descriptors | | long | gauge | +| logstash.node.stats.process.mem.total_virtual_in_bytes | | long | gauge | +| logstash.node.stats.process.open_file_descriptors | | long | gauge | +| logstash.node.stats.process.peak_open_file_descriptors | | long | gauge | +| logstash.node.stats.queue.events_count | | long | counter | +| logstash.node.stats.reloads.failures | | long | counter | +| logstash.node.stats.reloads.successes | | long | counter | +| logstash.node.stats.timestamp | | date | | +| logstash.pipeline.name | | alias | | +| process.pid | Process id. | long | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | +| service.hostname | Hostname of the service | keyword | | +| service.id | Unique identifier of the running service. If the service is comprised of many nodes, the `service.id` should be the same for all nodes. This id should uniquely identify the service. This makes it possible to correlate logs and metrics for one specific service, no matter which particular node emitted the event. Note that if you need to see the events from one specific host of the service, you should filter on that `host.name` or `host.id` instead. | keyword | | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | +| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | | + + +An example event for `node_cel` looks as following: + +```json +{ "logstash": { - "cluster": { - "id": "0toa26-cTzmqx0WD40-4XQ" - }, - "elasticsearch": { - "cluster": { - "id": "0toa26-cTzmqx0WD40-4XQ" - } - }, "node": { "stats": { - "events": { - "duration_in_millis": 334, - "filtered": 138, - "in": 618, - "out": 138 - }, "jvm": { - "gc": { - "collectors": { - "old": { - "collection_count": 0, - "collection_time_in_millis": 0 - }, - "young": { - "collection_count": 13, - "collection_time_in_millis": 177 - } - } - }, "mem": { - "heap_max_in_bytes": 10527703038, - "heap_used_in_bytes": 234688352, - "heap_used_percent": 2 + "heap_committed_in_bytes": 264241152, + "heap_used_percent": 2, + "heap_max_in_bytes": 5184159742, + "non_heap_committed_in_bytes": 191889408, + "heap_used_in_bytes": 143564464, + "non_heap_used_in_bytes": 180940656 }, - "uptime_in_millis": 21450 + "threads": { + "count": 83, + "peak_count": 85 + }, + "uptime_in_millis": 448206 }, "logstash": { - "ephemeral_id": "17681d23-bd67-4c40-b6b1-63e97b560856", - "host": "170bc3698b89", - "http_address": "0.0.0.0:9600", - "name": "170bc3698b89", "pipeline": { + "batch_delay": 50, "batch_size": 125, - "workers": 10 + "workers": 8 }, + "pipelines": [ + "standalone-pipeline", + "pipeline-with-memory-queue", + "pipeline-with-persisted-queue" + ], + "http_address": "0.0.0.0:9600", + "name": "21d61ee7529e", + "host": "21d61ee7529e", + "ephemeral_id": "fa27552b-e31d-463d-a5db-f470e6c2f0ba", + "version": "8.6.0", + "uuid": "2566e68f-ea0e-4dd0-8b65-17bc7bd9f685", "snapshot": false, - "status": "green", - "uuid": "a4224a67-aae8-4bce-8660-079d068b2e72", - "version": "8.5.0" + "status": "green" }, - "os": { - "cgroup": { - "cpu": { - "cfs_quota_micros": -1, - "control_group": "/", - "stat": { - "number_of_elapsed_periods": 0, - "number_of_times_throttled": 0, - "time_throttled_nanos": 0 - } - }, - "cpuacct": { - "control_group": "/", - "usage_nanos": 55911664431 - } + "process": { + "open_file_descriptors": 94, + "mem": { + "total_virtual_in_bytes": 11442712576 }, + "max_file_descriptors": 1048576, "cpu": { "load_average": { - "15m": 2.28, - "1m": 2.85, - "5m": 2.62 - }, - "percent": 0 - } - }, - "pipelines": [ - { - "ephemeral_id": "453a2361-82d8-4d88-b7a4-063c3293cd4a", - "events": { - "duration_in_millis": 0, - "filtered": 0, - "in": 476, - "out": 0, - "queue_push_duration_in_millis": 59 - }, - "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa", - "id": "pipeline-with-persisted-queue", - "queue": { - "capacity": { - "max_queue_size_in_bytes": 1073741824, - "max_unread_events": 0, - "page_capacity_in_bytes": 67108864, - "queue_size_in_bytes": 132880 - }, - "data": { - "free_space_in_bytes": 51709984768, - "path": "/usr/share/logstash/data/queue/pipeline-with-persisted-queue", - "storage_type": "overlay" - }, - "events": 0, - "events_count": 0, - "max_queue_size_in_bytes": 1073741824, - "queue_size_in_bytes": 132880, - "type": "persisted" - }, - "reloads": { - "failures": 0, - "successes": 0 + "5m": 1.49, + "15m": 1.23, + "1m": 0.74 }, - "vertices": [ - { - "events_out": 475, - "id": "dfc132c40b9f5dbc970604f191cf87ee04b102b6f4be5a235436973dc7ea6368", - "pipeline_ephemeral_id": "453a2361-82d8-4d88-b7a4-063c3293cd4a", - "queue_push_duration_in_millis": 59 - }, - { - "duration_in_millis": 0, - "events_in": 375, - "events_out": 0, - "id": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", - "pipeline_ephemeral_id": "453a2361-82d8-4d88-b7a4-063c3293cd4a" - }, - { - "cluster_uuid": "0toa26-cTzmqx0WD40-4XQ", - "duration_in_millis": 1, - "events_in": 0, - "events_out": 0, - "id": "9ba6577aa5c41a5ebcaae010b9a0ef44015ae68c624596ed924417d1701abc21", - "pipeline_ephemeral_id": "453a2361-82d8-4d88-b7a4-063c3293cd4a" - } - ] + "total_in_millis": 130690, + "percent": 2 }, - { - "ephemeral_id": "7114cd7d-8d91-4afc-a986-32487c3edcbe", - "events": { - "duration_in_millis": 191, - "filtered": 91, - "in": 95, - "out": 91, - "queue_push_duration_in_millis": 4 - }, - "hash": "0542fa70daa36dc3e858ea099f125cc8c9e451ebbfe8ea8867e52f9764da0a35", - "id": "pipeline-with-memory-queue", - "queue": { - "events_count": 0, - "max_queue_size_in_bytes": 0, - "queue_size_in_bytes": 0, - "type": "memory" - }, - "reloads": { - "failures": 0, - "successes": 0 - }, - "vertices": [ - { - "events_out": 95, - "id": "4c5941552cdaa72ebc285557c697a7150c359ee3eacf9b5664c4b1048e26153b", - "pipeline_ephemeral_id": "7114cd7d-8d91-4afc-a986-32487c3edcbe", - "queue_push_duration_in_millis": 4 - }, - { - "cluster_uuid": "0toa26-cTzmqx0WD40-4XQ", - "duration_in_millis": 193, - "events_in": 91, - "events_out": 91, - "id": "635a080aacc8700059852859da284a9cb92cb78a6d7112fbf55e441e51b6658a", - "long_counters": [ - { - "name": "bulk_requests.successes", - "value": 12 - }, - { - "name": "bulk_requests.responses.200", - "value": 12 - }, - { - "name": "documents.successes", - "value": 91 - } - ], - "pipeline_ephemeral_id": "7114cd7d-8d91-4afc-a986-32487c3edcbe" - } - ] - } - ], - "process": { + "peak_open_file_descriptors": 95 + }, + "os": { "cpu": { - "percent": 4 + "load_average": { + "5m": 1.49, + "15m": 1.23, + "1m": 0.74 + }, + "total_in_millis": 130690, + "percent": 2 }, - "max_file_descriptors": 1048576, - "open_file_descriptors": 89 + "cgroup": {} + }, + "events": { + "filtered": 27752, + "in": 28442, + "queue_push_duration_in_millis": 597, + "duration_in_millis": 3202220, + "out": 27752 }, "queue": { "events_count": 0 @@ -291,543 +387,268 @@ An example event for `node_stats` looks as following: "reloads": { "failures": 0, "successes": 0 - }, - "timestamp": "2023-03-02T15:57:57.016Z" + } } } }, - "metricset": { - "name": "node_stats", - "period": 10000 + "input": { + "type": "cel" }, - "service": { - "address": "http://elastic-package-service_logstash_1:9600/_node/stats", - "hostname": "170bc3698b89", - "id": "", - "name": "logstash", - "type": "logstash", - "version": "8.5.0" - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| logstash.node.jvm.version | Version | keyword | -| logstash.node.state.pipeline.hash | | keyword | -| logstash.node.state.pipeline.id | | keyword | -| logstash.node.stats.events.duration_in_millis | | long | -| logstash.node.stats.events.filtered | Filtered events counter. | long | -| logstash.node.stats.events.in | Incoming events counter. | long | -| logstash.node.stats.events.out | Outgoing events counter. | long | -| logstash.node.stats.jvm.mem.heap_max_in_bytes | | long | -| logstash.node.stats.jvm.mem.heap_used_in_bytes | | long | -| logstash.node.stats.jvm.uptime_in_millis | | long | -| logstash.node.stats.logstash.uuid | | keyword | -| logstash.node.stats.logstash.version | | keyword | -| logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods | | long | -| logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled | | long | -| logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos | | long | -| logstash.node.stats.os.cgroup.cpuacct.usage_nanos | | long | -| logstash.node.stats.os.cpu.load_average.15m | | long | -| logstash.node.stats.os.cpu.load_average.1m | | long | -| logstash.node.stats.os.cpu.load_average.5m | | long | -| logstash.node.stats.pipelines.events.duration_in_millis | | long | -| logstash.node.stats.pipelines.events.out | | long | -| logstash.node.stats.pipelines.hash | | keyword | -| logstash.node.stats.pipelines.id | | keyword | -| logstash.node.stats.pipelines.queue.events_count | | long | -| logstash.node.stats.pipelines.queue.max_queue_size_in_bytes | | long | -| logstash.node.stats.pipelines.queue.queue_size_in_bytes | | long | -| logstash.node.stats.pipelines.queue.type | | keyword | -| logstash.node.stats.pipelines.vertices.duration_in_millis | | long | -| logstash.node.stats.pipelines.vertices.events_in | | long | -| logstash.node.stats.pipelines.vertices.events_out | events_out | long | -| logstash.node.stats.pipelines.vertices.id | id | keyword | -| logstash.node.stats.pipelines.vertices.pipeline_ephemeral_id | pipeline_ephemeral_id | keyword | -| logstash.node.stats.pipelines.vertices.queue_push_duration_in_millis | queue_push_duration_in_millis | float | -| logstash.node.stats.process.cpu.percent | | double | -| logstash.node.stats.queue.events_count | | long | -| logstash_stats.pipelines | | nested | -| process.pid | Process id. | long | -| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | + "agent": { + "name": "MacBook-Pro.local", + "id": "b88de78b-7bd7-49ae-99d7-f68ea18070c4", + "type": "filebeat", + "ephemeral_id": "e24a6e70-8e93-4d18-8535-319e63c81bc8", + "version": "8.10.1" + }, + "@timestamp": "2023-10-04T18:53:48.769Z", + "ecs": { + "version": "8.0.0" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "logstash.node" + }, + "elastic_agent": { + "id": "b88de78b-7bd7-49ae-99d7-f68ea18070c4", + "version": "8.10.1", + "snapshot": false + }, + "host": { + "hostname": "macbook-pro.local", + "os": { + "build": "22F82", + "kernel": "22.5.0", + "name": "macOS", + "family": "darwin", + "type": "macos", + "version": "13.4.1", + "platform": "darwin" + }, + "ip": [ + "192.168.1.184" + ], + "name": "macbook-pro.local", + "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", + "mac": [ + "AC-DE-48-00-11-22" + ], + "architecture": "x86_64" + }, + "event": { + "agent_id_status": "verified", + "ingested": "2023-10-04T18:53:49Z", + "dataset": "logstash.node" + } +} +``` -### Node +#### Pipeline -An example event for `node` looks as following: +This is the `pipeline` dataset, which drives the Pipeline dashboard pages. + +#### Example + +**Exported fields** + +| Field | Description | Type | Unit | Metric Type | +|---|---|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | +| cloud.image.id | Image ID for the cloud instance. | keyword | | | +| cloud.instance.id | Instance ID of the host machine. | keyword | | | +| cloud.instance.name | Instance name of the host machine. | keyword | | | +| cloud.machine.type | Machine type of the host machine. | keyword | | | +| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | | +| cluster_uuid | | alias | | | +| container.id | Unique container id. | keyword | | | +| container.image.name | Name of the image the container was built on. | keyword | | | +| container.labels | Image labels. | object | | | +| container.name | Container name. | keyword | | | +| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | | +| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | | +| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | +| error.message | Error message. | match_only_text | | | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | | +| host.architecture | Operating system architecture. | keyword | | | +| host.containerized | If the host is a container. | boolean | | | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | +| host.ip | Host ip addresses. | ip | | | +| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | +| host.os.build | OS build information. | keyword | | | +| host.os.codename | OS codename, if any. | keyword | | | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | | | +| host.os.name | Operating system name, without the version. | keyword | | | +| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | +| host.os.version | Operating system version as a raw string. | keyword | | | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | +| input.type | | keyword | | | +| logstash.host.address | | alias | | | +| logstash.host.name | | alias | | | +| logstash.pipeline.elasticsearch.cluster.id | Elasticsearch clusters this Logstash pipeline is attached to | keyword | | | +| logstash.pipeline.host.address | address hosting this instance of logstash | keyword | | | +| logstash.pipeline.host.name | Host name of the node running logstash | keyword | | | +| logstash.pipeline.info.batch_delay | Batch delay for the running pipeline | long | | | +| logstash.pipeline.info.batch_size | Batch size for the running pipeline | long | | | +| logstash.pipeline.info.ephemeral_id | Ephemeral Id for the running pipeline | keyword | | | +| logstash.pipeline.info.workers | Number of workers for the running pipeline | long | | | +| logstash.pipeline.name | Logstash Pipeline id/name | keyword | | | +| logstash.pipeline.total.events.filtered | Number of events filtered by the pipeline | long | | counter | +| logstash.pipeline.total.events.in | Number of events received by the pipeline | long | | counter | +| logstash.pipeline.total.events.out | Number of events emitted by the pipeline | long | | counter | +| logstash.pipeline.total.flow.filter_throughput.current | current value of the filter throughput flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.filter_throughput.last_1_minute | current value of the filter throughput flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.input_throughput.current | current value of the input throughput flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.input_throughput.last_1_minute | current value of the throughput flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.output_throughput.current | current value of the output throughput flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.output_throughput.last_1_minute | current value of the output throughput flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.queue_backpressure.current | current value of the queue backpressure flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.queue_backpressure.last_1_minute | current value of the queue backpressure flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.queue_persisted_growth_bytes.current | current value of the queue persisted growth bytes flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.queue_persisted_growth_bytes.last_1_minute | current value of the queue persisted growth bytes flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.queue_persisted_growth_events.current | current value of the queue persisted growth events flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.queue_persisted_growth_events.last_1_minute | current value of the queue persisted growth events flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.worker_concurrency.current | last 1 minute value of the worker utilization flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.worker_concurrency.last_1_minute | current value of the worker concurrency flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.worker_utilization.current | last 1 minute value of the worker concurrency flow metric | scaled_float | | gauge | +| logstash.pipeline.total.flow.worker_utilization.last_1_minute | current value of the worker concurrency flow metric | scaled_float | | gauge | +| logstash.pipeline.total.queues.current_size.bytes | Current size of the PQ | long | byte | gauge | +| logstash.pipeline.total.queues.events | Number of events in the PQ for this pipeline | long | | counter | +| logstash.pipeline.total.queues.max_size.bytes | Maximum possible size of the PQ | long | | gauge | +| logstash.pipeline.total.queues.type | Type of queue - persistent or memory | keyword | | | +| logstash.pipeline.total.reloads.failures | Number of failed reloads for this pipeline | long | | counter | +| logstash.pipeline.total.reloads.successes | Number of successful reloads for this pipeline | long | | counter | +| logstash.pipeline.total.time.duration.ms | Time spent processing events through the pipeline. | long | ms | counter | +| logstash.pipeline.total.time.queue_push_duration.ms | Time spent pushing events to the queue for this pipeline. | long | ms | counter | +| process.pid | Process id. | long | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.hostname | Hostname of the service | keyword | | | +| service.id | Unique identifier of the running service. If the service is comprised of many nodes, the `service.id` should be the same for all nodes. This id should uniquely identify the service. This makes it possible to correlate logs and metrics for one specific service, no matter which particular node emitted the event. Note that if you need to see the events from one specific host of the service, you should filter on that `host.name` or `host.id` instead. | keyword | | | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | +| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | | | + + +An example event for `pipeline` looks as following: ```json { - "@timestamp": "2023-03-02T15:57:03.999Z", - "agent": { - "ephemeral_id": "16f2dd63-454b-4699-a8c8-2a748bd044b8", - "id": "3cc85092-54dc-4b58-8726-5e9458167f42", - "name": "docker-fleet-agent", - "type": "metricbeat", - "version": "8.5.0" - }, + "@timestamp": "2023-10-04T18:53:18.708Z", "data_stream": { - "dataset": "logstash.stack_monitoring.node", - "namespace": "ep", + "dataset": "logstash.pipeline", + "namespace": "default", "type": "metrics" }, "ecs": { "version": "8.0.0" }, - "elastic_agent": { - "id": "3cc85092-54dc-4b58-8726-5e9458167f42", - "snapshot": false, - "version": "8.5.0" - }, "event": { "agent_id_status": "verified", - "dataset": "logstash.stack_monitoring.node", - "duration": 69490100, - "ingested": "2023-03-02T15:57:05Z", - "module": "logstash" + "dataset": "logstash.pipeline", + "ingested": "2023-10-04T18:53:19Z" }, "host": { "architecture": "x86_64", - "containerized": true, - "hostname": "docker-fleet-agent", - "id": "66392b0697b84641af8006d87aeb89f1", + "hostname": "macbook-pro.local", + "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", "ip": [ - "192.168.224.7" + "192.168.1.184" ], "mac": [ - "02-42-C0-A8-E0-07" + "AC-DE-48-00-11-22" ], - "name": "docker-fleet-agent", + "name": "macbook-pro.local", "os": { - "codename": "focal", - "family": "debian", - "kernel": "5.10.47-linuxkit", - "name": "Ubuntu", - "platform": "ubuntu", - "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)" + "build": "22F82", + "family": "darwin", + "kernel": "22.5.0", + "name": "macOS", + "platform": "darwin", + "version": "13.4.1" } }, + "input": { + "type": "cel" + }, "logstash": { - "cluster": { - "id": "0toa26-cTzmqx0WD40-4XQ" - }, - "elasticsearch": { - "cluster": { - "id": "0toa26-cTzmqx0WD40-4XQ" - } - }, - "node": { - "host": "45730b5f8c3d", - "id": "2e17cd45-ecb8-4358-a420-b867f2e32b7a", - "jvm": { - "version": "17.0.4" + "pipeline": { + "host": { + "address": "0.0.0.0:9600", + "name": "21d61ee7529e" }, - "state": { - "pipeline": { - "batch_size": 125, - "ephemeral_id": "472cf082-aa15-41ca-9ed1-62d03afbadd0", - "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa", - "id": "pipeline-with-persisted-queue", - "representation": { - "graph": { - "edges": [ - { - "from": "dfc132c40b9f5dbc970604f191cf87ee04b102b6f4be5a235436973dc7ea6368", - "id": "9ed824e4f189b461c111ae27c17644c3c5f6d7c3c2bb213cbc7cc067cbd68fe6", - "to": "__QUEUE__", - "type": "plain" - }, - { - "from": "__QUEUE__", - "id": "cb33f8fb7611e31a2c1751b74cdedf5b8cdb96ea46b812a2541e2db4f13dca10", - "to": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", - "type": "plain" - }, - { - "from": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", - "id": "63ef166c45b87a40f31e0a6def175f10460b6b0ed656e70968eb52b1c454ab16", - "to": "9ba6577aa5c41a5ebcaae010b9a0ef44015ae68c624596ed924417d1701abc21", - "type": "plain" - } - ], - "vertices": [ - { - "config_name": "java_generator", - "explicit_id": false, - "id": "dfc132c40b9f5dbc970604f191cf87ee04b102b6f4be5a235436973dc7ea6368", - "meta": { - "source": { - "column": 3, - "id": "/usr/share/logstash/pipeline/persisted-queue.conf", - "line": 2, - "protocol": "file" - } - }, - "plugin_type": "input", - "type": "plugin" - }, - { - "explicit_id": false, - "id": "__QUEUE__", - "meta": null, - "type": "queue" - }, - { - "config_name": "sleep", - "explicit_id": false, - "id": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", - "meta": { - "source": { - "column": 3, - "id": "/usr/share/logstash/pipeline/persisted-queue.conf", - "line": 8, - "protocol": "file" - } - }, - "plugin_type": "filter", - "type": "plugin" - }, - { - "config_name": "elasticsearch", - "explicit_id": false, - "id": "9ba6577aa5c41a5ebcaae010b9a0ef44015ae68c624596ed924417d1701abc21", - "meta": { - "source": { - "column": 3, - "id": "/usr/share/logstash/pipeline/persisted-queue.conf", - "line": 15, - "protocol": "file" - } - }, - "plugin_type": "output", - "type": "plugin" - } - ] - }, - "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa", - "type": "lir", - "version": "0.0.0" - }, - "workers": 10 - } - }, - "version": "8.5.0" - } - }, - "metricset": { - "name": "node", - "period": 10000 - }, - "process": { - "pid": 1 - }, - "service": { - "address": "http://elastic-package-service_logstash_1:9600/_node", - "hostname": "45730b5f8c3d", - "id": "2e17cd45-ecb8-4358-a420-b867f2e32b7a", - "name": "logstash", - "type": "logstash", - "version": "8.5.0" - } -} -``` - - -## Metrics (Technical Preview) - -This Logstash package also includes a technical preview of Logstash data collection and dashboards -native to elastic agent. The technical preview includes enhanced data collection, and a number of dashboards, which include additional insight into running pipelines. - -Note that this feature is not intended for use with the Stack Monitoring UI inside Kibana, -and is included as a technical preview. Existing implementations wishing to continue using the Stack Monitoring UI should uncheck the technical preview option, and continue to use `Metrics (Stack Monitoring)`. Those users who wish to use the technical preview should uncheck `Metrics (Stack Monitoring)` and check `Metrics (Technical Preview)` - -### Fields and Sample Event - -#### Node - -This is the `node` dataset, which drives the Node dashboard pages. - -#### Example - -**Exported fields** - -| Field | Description | Type | Metric Type | -|---|---|---|---| -| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | -| cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | -| cloud.region | Region in which this host, resource, or service is located. | keyword | | -| cluster_uuid | | alias | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | -| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | -| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | -| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | -| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | -| host.architecture | Operating system architecture. | keyword | | -| host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | -| host.os.build | OS build information. | keyword | | -| host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| input.type | | keyword | | -| logstash.elasticsearch.cluster.id | | keyword | | -| logstash.host.address | | alias | | -| logstash.host.name | | alias | | -| logstash.node.stats.events.duration_in_millis | | long | counter | -| logstash.node.stats.events.filtered | Filtered events counter | long | counter | -| logstash.node.stats.events.in | Incoming events counter | long | counter | -| logstash.node.stats.events.out | Outgoing events counter | long | counter | -| logstash.node.stats.events.queue_push_duration_in_millis | | long | counter | -| logstash.node.stats.jvm.gc.collectors.old.collection_count | | long | counter | -| logstash.node.stats.jvm.gc.collectors.old.collection_time_in_millis | | long | counter | -| logstash.node.stats.jvm.gc.collectors.young.collection_count | | long | counter | -| logstash.node.stats.jvm.gc.collectors.young.collection_time_in_millis | | long | counter | -| logstash.node.stats.jvm.mem.heap_committed_in_bytes | | long | gauge | -| logstash.node.stats.jvm.mem.heap_max_in_bytes | | long | counter | -| logstash.node.stats.jvm.mem.heap_used_in_bytes | | long | gauge | -| logstash.node.stats.jvm.mem.heap_used_percent | | long | gauge | -| logstash.node.stats.jvm.mem.non_heap_committed_in_bytes | | long | gauge | -| logstash.node.stats.jvm.mem.non_heap_used_in_bytes | | long | gauge | -| logstash.node.stats.jvm.threads.count | current number of threads | long | counter | -| logstash.node.stats.jvm.threads.peak_count | peak number of threads | long | counter | -| logstash.node.stats.jvm.uptime_in_millis | | long | counter | -| logstash.node.stats.logstash.ephemeral_id | | keyword | | -| logstash.node.stats.logstash.host | | keyword | | -| logstash.node.stats.logstash.http_address | | keyword | | -| logstash.node.stats.logstash.name | | keyword | | -| logstash.node.stats.logstash.pipeline.batch_delay | | long | gauge | -| logstash.node.stats.logstash.pipeline.batch_size | | long | gauge | -| logstash.node.stats.logstash.pipeline.workers | | long | gauge | -| logstash.node.stats.logstash.pipelines | | keyword | | -| logstash.node.stats.logstash.snapshot | | boolean | | -| logstash.node.stats.logstash.status | | keyword | | -| logstash.node.stats.logstash.uuid | | keyword | | -| logstash.node.stats.logstash.version | | keyword | | -| logstash.node.stats.os.cgroup.cpu.cfs_quota_micros | | long | gauge | -| logstash.node.stats.os.cgroup.cpu.control_group | | text | | -| logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods | | long | gauge | -| logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled | | long | counter | -| logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos | | long | counter | -| logstash.node.stats.os.cgroup.cpuacct.control_group | | text | | -| logstash.node.stats.os.cgroup.cpuacct.usage_nanos | | long | counter | -| logstash.node.stats.os.cpu.load_average.15m | | half_float | gauge | -| logstash.node.stats.os.cpu.load_average.1m | | half_float | gauge | -| logstash.node.stats.os.cpu.load_average.5m | | half_float | gauge | -| logstash.node.stats.os.cpu.percent | | double | gauge | -| logstash.node.stats.os.cpu.total_in_millis | | long | counter | -| logstash.node.stats.pipelines.ephemeral_id | | keyword | | -| logstash.node.stats.pipelines.events.duration_in_millis | | long | | -| logstash.node.stats.pipelines.events.filtered | | long | | -| logstash.node.stats.pipelines.events.in | | long | | -| logstash.node.stats.pipelines.events.out | | long | | -| logstash.node.stats.pipelines.events.queue_push_duration_in_millis | | long | | -| logstash.node.stats.pipelines.hash | | keyword | | -| logstash.node.stats.pipelines.id | | keyword | | -| logstash.node.stats.pipelines.queue.events_count | | long | | -| logstash.node.stats.pipelines.queue.max_queue_size_in_bytes | | long | | -| logstash.node.stats.pipelines.queue.queue_size_in_bytes | | long | | -| logstash.node.stats.pipelines.queue.type | | keyword | | -| logstash.node.stats.pipelines.reloads.failures | | long | | -| logstash.node.stats.pipelines.reloads.successes | | long | | -| logstash.node.stats.process.cpu.load_average.15m | | half_float | gauge | -| logstash.node.stats.process.cpu.load_average.1m | | half_float | gauge | -| logstash.node.stats.process.cpu.load_average.5m | | half_float | gauge | -| logstash.node.stats.process.cpu.percent | | double | gauge | -| logstash.node.stats.process.cpu.total_in_millis | | long | counter | -| logstash.node.stats.process.max_file_descriptors | | long | gauge | -| logstash.node.stats.process.mem.total_virtual_in_bytes | | long | gauge | -| logstash.node.stats.process.open_file_descriptors | | long | gauge | -| logstash.node.stats.process.peak_open_file_descriptors | | long | gauge | -| logstash.node.stats.queue.events_count | | long | counter | -| logstash.node.stats.reloads.failures | | long | counter | -| logstash.node.stats.reloads.successes | | long | counter | -| logstash.node.stats.timestamp | | date | | -| logstash.pipeline.name | | alias | | -| process.pid | Process id. | long | | -| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | -| service.hostname | Hostname of the service | keyword | | -| service.id | Unique identifier of the running service. If the service is comprised of many nodes, the `service.id` should be the same for all nodes. This id should uniquely identify the service. This makes it possible to correlate logs and metrics for one specific service, no matter which particular node emitted the event. Note that if you need to see the events from one specific host of the service, you should filter on that `host.name` or `host.id` instead. | keyword | | -| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | -| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | | - - -An example event for `node_cel` looks as following: - -```json -{ - "logstash": { - "node": { - "stats": { - "jvm": { - "mem": { - "heap_committed_in_bytes": 264241152, - "heap_used_percent": 2, - "heap_max_in_bytes": 5184159742, - "non_heap_committed_in_bytes": 191889408, - "heap_used_in_bytes": 143564464, - "non_heap_used_in_bytes": 180940656 - }, - "threads": { - "count": 83, - "peak_count": 85 - }, - "uptime_in_millis": 448206 + "name": "standalone-pipeline", + "total": { + "events": { + "filtered": 2038, + "in": 2038, + "out": 2038 }, - "logstash": { - "pipeline": { - "batch_delay": 50, - "batch_size": 125, - "workers": 8 + "flow": { + "filter_throughput": { + "current": 5.02, + "last_1_minute": 5.003 }, - "pipelines": [ - "standalone-pipeline", - "pipeline-with-memory-queue", - "pipeline-with-persisted-queue" - ], - "http_address": "0.0.0.0:9600", - "name": "21d61ee7529e", - "host": "21d61ee7529e", - "ephemeral_id": "fa27552b-e31d-463d-a5db-f470e6c2f0ba", - "version": "8.6.0", - "uuid": "2566e68f-ea0e-4dd0-8b65-17bc7bd9f685", - "snapshot": false, - "status": "green" - }, - "process": { - "open_file_descriptors": 94, - "mem": { - "total_virtual_in_bytes": 11442712576 + "input_throughput": { + "current": 4.948, + "last_1_minute": 5.003 }, - "max_file_descriptors": 1048576, - "cpu": { - "load_average": { - "5m": 1.49, - "15m": 1.23, - "1m": 0.74 - }, - "total_in_millis": 130690, - "percent": 2 + "output_throughput": { + "current": 5.02, + "last_1_minute": 5.003 + }, + "queue_backpressure": { + "current": 0, + "last_1_minute": 0 }, - "peak_open_file_descriptors": 95 + "worker_concurrency": { + "current": 0.001, + "last_1_minute": 0.001 + } }, - "os": { - "cpu": { - "load_average": { - "5m": 1.49, - "15m": 1.23, - "1m": 0.74 - }, - "total_in_millis": 130690, - "percent": 2 + "queues": { + "current_size": { + "bytes": 0 }, - "cgroup": {} - }, - "events": { - "filtered": 27752, - "in": 28442, - "queue_push_duration_in_millis": 597, - "duration_in_millis": 3202220, - "out": 27752 - }, - "queue": { - "events_count": 0 + "events": 0, + "max_size": { + "bytes": 0 + }, + "type": "memory" }, "reloads": { "failures": 0, "successes": 0 + }, + "time": { + "duration": { + "ms": 1363 + }, + "queue_push_duration": { + "ms": 12 + } } } } - }, - "input": { - "type": "cel" - }, - "agent": { - "name": "MacBook-Pro.local", - "id": "b88de78b-7bd7-49ae-99d7-f68ea18070c4", - "type": "filebeat", - "ephemeral_id": "e24a6e70-8e93-4d18-8535-319e63c81bc8", - "version": "8.10.1" - }, - "@timestamp": "2023-10-04T18:53:48.769Z", - "ecs": { - "version": "8.0.0" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "logstash.node" - }, - "elastic_agent": { - "id": "b88de78b-7bd7-49ae-99d7-f68ea18070c4", - "version": "8.10.1", - "snapshot": false - }, - "host": { - "hostname": "macbook-pro.local", - "os": { - "build": "22F82", - "kernel": "22.5.0", - "name": "macOS", - "family": "darwin", - "type": "macos", - "version": "13.4.1", - "platform": "darwin" - }, - "ip": [ - "192.168.1.184" - ], - "name": "macbook-pro.local", - "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", - "mac": [ - "AC-DE-48-00-11-22" - ], - "architecture": "x86_64" - }, - "event": { - "agent_id_status": "verified", - "ingested": "2023-10-04T18:53:49Z", - "dataset": "logstash.node" } } ``` -#### Pipeline +#### Plugin -This is the `pipeline` dataset, which drives the Pipeline dashboard pages. +This is the `plugin` dataset, which drives the Pipeline detail dashboard pages. Note that this dataset may produce many documents for logstash instances using a large number of pipelines and/or plugins within those pipelines. For those instances, we recommend reviewing the +pipeline collection period, and setting it to an appropriate value. #### Example @@ -881,38 +702,72 @@ This is the `pipeline` dataset, which drives the Pipeline dashboard pages. | logstash.pipeline.elasticsearch.cluster.id | Elasticsearch clusters this Logstash pipeline is attached to | keyword | | | | logstash.pipeline.host.address | address hosting this instance of logstash | keyword | | | | logstash.pipeline.host.name | Host name of the node running logstash | keyword | | | -| logstash.pipeline.info.batch_delay | Batch delay for the running pipeline | long | | | -| logstash.pipeline.info.batch_size | Batch size for the running pipeline | long | | | -| logstash.pipeline.info.ephemeral_id | Ephemeral Id for the running pipeline | keyword | | | -| logstash.pipeline.info.workers | Number of workers for the running pipeline | long | | | +| logstash.pipeline.id | Logstash Pipeline hash | keyword | | | | logstash.pipeline.name | Logstash Pipeline id/name | keyword | | | -| logstash.pipeline.total.events.filtered | Number of events filtered by the pipeline | long | | counter | -| logstash.pipeline.total.events.in | Number of events received by the pipeline | long | | counter | -| logstash.pipeline.total.events.out | Number of events emitted by the pipeline | long | | counter | -| logstash.pipeline.total.flow.filter_throughput.current | current value of the filter throughput flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.filter_throughput.last_1_minute | current value of the filter throughput flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.input_throughput.current | current value of the input throughput flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.input_throughput.last_1_minute | current value of the throughput flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.output_throughput.current | current value of the output throughput flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.output_throughput.last_1_minute | current value of the output throughput flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.queue_backpressure.current | current value of the queue backpressure flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.queue_backpressure.last_1_minute | current value of the queue backpressure flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.queue_persisted_growth_bytes.current | current value of the queue persisted growth bytes flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.queue_persisted_growth_bytes.last_1_minute | current value of the queue persisted growth bytes flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.queue_persisted_growth_events.current | current value of the queue persisted growth events flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.queue_persisted_growth_events.last_1_minute | current value of the queue persisted growth events flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.worker_concurrency.current | last 1 minute value of the worker utilization flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.worker_concurrency.last_1_minute | current value of the worker concurrency flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.worker_utilization.current | last 1 minute value of the worker concurrency flow metric | scaled_float | | gauge | -| logstash.pipeline.total.flow.worker_utilization.last_1_minute | current value of the worker concurrency flow metric | scaled_float | | gauge | -| logstash.pipeline.total.queues.current_size.bytes | Current size of the PQ | long | byte | gauge | -| logstash.pipeline.total.queues.events | Number of events in the PQ for this pipeline | long | | counter | -| logstash.pipeline.total.queues.max_size.bytes | Maximum possible size of the PQ | long | | gauge | -| logstash.pipeline.total.queues.type | Type of queue - persistent or memory | keyword | | | -| logstash.pipeline.total.reloads.failures | Number of failed reloads for this pipeline | long | | counter | -| logstash.pipeline.total.reloads.successes | Number of successful reloads for this pipeline | long | | counter | -| logstash.pipeline.total.time.duration.ms | Time spent processing events through the pipeline. | long | ms | counter | -| logstash.pipeline.total.time.queue_push_duration.ms | Time spent pushing events to the queue for this pipeline. | long | ms | counter | +| logstash.pipeline.plugin.codec.decode.duration.ms | amount of time spend decoding events | long | ms | counter | +| logstash.pipeline.plugin.codec.decode.in | number of events entering the decoder | long | | counter | +| logstash.pipeline.plugin.codec.decode.out | number of events exiting the decoder | long | | counter | +| logstash.pipeline.plugin.codec.encode.duration.ms | amount of time spend encoding events | long | ms | counter | +| logstash.pipeline.plugin.codec.encode.in | number of events encoded | long | | counter | +| logstash.pipeline.plugin.codec.id | Id of codec plugin | keyword | | | +| logstash.pipeline.plugin.codec.name | Name of codec plugin | keyword | | | +| logstash.pipeline.plugin.filter.elasticsearch.cluster.id | Elasticsearch clusters this Logstash plugin is attached to | keyword | | | +| logstash.pipeline.plugin.filter.events.in | number of events received by the filter | long | | counter | +| logstash.pipeline.plugin.filter.events.out | number of events emitted by the filter | long | | counter | +| logstash.pipeline.plugin.filter.flow.worker_millis_per_event.current | amount of time spent per event for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute | amount of time spent per event for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.filter.flow.worker_utilization.current | worker utilization for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.filter.flow.worker_utilization.last_1_minute | worker utilization for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.filter.id | Id of filter plugin | keyword | | | +| logstash.pipeline.plugin.filter.metrics.dissect.failures | number of dissect failures | long | | counter | +| logstash.pipeline.plugin.filter.metrics.dissect.matches | number of dissect matches | long | | counter | +| logstash.pipeline.plugin.filter.metrics.grok.failures | number of grok failures | long | | counter | +| logstash.pipeline.plugin.filter.metrics.grok.matches | number of grok matches | long | | counter | +| logstash.pipeline.plugin.filter.name | Name of filter plugin | keyword | | | +| logstash.pipeline.plugin.filter.source.column | | keyword | | | +| logstash.pipeline.plugin.filter.source.id | | keyword | | | +| logstash.pipeline.plugin.filter.source.line | | long | | | +| logstash.pipeline.plugin.filter.source.protocol | | keyword | | | +| logstash.pipeline.plugin.filter.time.duration.ms | amount of time working on events in this plugin | long | ms | counter | +| logstash.pipeline.plugin.input.elasticsearch.cluster.id | Elasticsearch clusters this Logstash plugin is attached to | keyword | | | +| logstash.pipeline.plugin.input.events.out | number of events emitted by the input | long | | counter | +| logstash.pipeline.plugin.input.flow.throughput.current | throughput of this input plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.input.flow.throughput.last_1_minute | throughput of this input plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.input.id | Id of input plugin | keyword | | | +| logstash.pipeline.plugin.input.name | Name of input plugin | keyword | | | +| logstash.pipeline.plugin.input.source.column | | keyword | | | +| logstash.pipeline.plugin.input.source.id | | keyword | | | +| logstash.pipeline.plugin.input.source.line | | long | | | +| logstash.pipeline.plugin.input.source.protocol | | keyword | | | +| logstash.pipeline.plugin.input.time.queue_push_duration.ms | amount of time spend pushing events to the queue | long | ms | counter | +| logstash.pipeline.plugin.output.elasticsearch.cluster.id | Elasticsearch clusters this Logstash plugin is attached to | keyword | | | +| logstash.pipeline.plugin.output.events.in | number of events received by the output | long | | counter | +| logstash.pipeline.plugin.output.events.out | number of events emitted by the output | long | | counter | +| logstash.pipeline.plugin.output.flow.worker_millis_per_event.current | amount of time spent per event for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_1_minute | amount of time spent per event for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.output.flow.worker_utilization.current | worker utilization for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute | worker utilization for this plugin | scaled_float | | gauge | +| logstash.pipeline.plugin.output.id | Id of output plugin | keyword | | | +| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.200 | | long | | counter | +| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.201 | | long | | counter | +| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.400 | | long | | counter | +| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.401 | | long | | counter | +| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.403 | | long | | counter | +| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.404 | | long | | counter | +| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.409 | | long | | counter | +| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.413 | | long | | counter | +| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.429 | | long | | counter | +| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.500 | | long | | counter | +| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.successes | | long | | counter | +| logstash.pipeline.plugin.output.metrics.elasticsearch.documents.non_retryable_failures | | long | | counter | +| logstash.pipeline.plugin.output.metrics.elasticsearch.documents.successes | | long | | counter | +| logstash.pipeline.plugin.output.name | Name of output plugin | keyword | | | +| logstash.pipeline.plugin.output.source.column | | keyword | | | +| logstash.pipeline.plugin.output.source.id | | keyword | | | +| logstash.pipeline.plugin.output.source.line | | long | | | +| logstash.pipeline.plugin.output.source.protocol | | keyword | | | +| logstash.pipeline.plugin.output.time.duration.ms | amount of time working on events in this plugin | long | ms | counter | +| logstash.pipeline.plugin.type | Type of the plugin | keyword | | | | process.pid | Process id. | long | | | | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | | service.hostname | Hostname of the service | keyword | | | @@ -922,347 +777,636 @@ This is the `pipeline` dataset, which drives the Pipeline dashboard pages. | service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | | | -An example event for `pipeline` looks as following: +An example event for `plugins` looks as following: + +```json +{ + "@timestamp": "2023-10-24T17:56:40.316Z", + "data_stream": { + "dataset": "logstash.plugins", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "agent_id_status": "verified", + "dataset": "logstash.plugins", + "ingested": "2023-10-24T17:56:41Z" + }, + "host": { + "architecture": "x86_64", + "hostname": "macbook-pro.local", + "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", + "ip": [ + "192.168.4.26" + ], + "mac": [ + "AC-DE-48-00-11-22" + ], + "name": "macbook-pro.local", + "os": { + "build": "22G120", + "family": "darwin", + "kernel": "22.6.0", + "name": "macOS", + "platform": "darwin", + "version": "13.6" + } + }, + "input": { + "type": "cel" + }, + "logstash": { + "pipeline": { + "elasticsearch": { + "cluster": { + "id": "9MOGoKiESvaklNVmxLo3iA" + } + }, + "host": { + "address": "127.0.0.1:9602", + "name": "logstash9602" + }, + "id": "b18ff60bcd82055aab2bf5601a2bc170502f80b33ab5938f25fa95ec8b04cd4b", + "name": "work", + "plugin": { + "output": { + "elasticsearch": { + "cluster": { + "id": "9MOGoKiESvaklNVmxLo3iA" + } + }, + "events": { + "in": 798, + "out": 798 + }, + "flow": { + "worker_millis_per_event": { + "current": 54, + "last_1_minute": 54 + }, + "worker_utilization": { + "current": 0.023, + "last_1_minute": 0.01 + } + }, + "id": "out_to_elasticsearch", + "metrics": { + "elasticsearch": { + "bulk_requests": { + "responses": { + "200": 798 + }, + "successes": 798 + }, + "documents": { + "successes": 798 + } + } + }, + "name": "elasticsearch", + "source": { + "column": "3", + "id": "/Users/test/ingestdemo/logstash-8.8.2/remap.conf", + "line": 132, + "protocol": "file" + }, + "time": { + "duration": { + "ms": 198060 + } + } + }, + "type": "output" + } + } + } +} +``` + +## Logs + +Logstash package supports the plain text format and the JSON format. Also, two types of +logs can be activated with the Logstash package: + +* `log` collects and parses the logs that Logstash writes to disk. +* `slowlog` parses the logstash slowlog (make sure to configure the Logstash slowlog option). + +#### Known issues + +When using the `log` data stream to parse plaintext logs, if a multiline plaintext log contains an embedded JSON object such that +the JSON object starts on a new line, the fileset may not parse the multiline plaintext log event correctly. + + +## Metrics + +Logstash metric related data streams works with Logstash 7.3.0 and later. + +### Node Stats + +An example event for `node_stats` looks as following: ```json { - "@timestamp": "2023-10-04T18:53:18.708Z", + "@timestamp": "2023-03-02T15:57:56.968Z", + "agent": { + "ephemeral_id": "16f2dd63-454b-4699-a8c8-2a748bd044b8", + "id": "3cc85092-54dc-4b58-8726-5e9458167f42", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.5.0" + }, "data_stream": { - "dataset": "logstash.pipeline", - "namespace": "default", + "dataset": "logstash.stack_monitoring.node_stats", + "namespace": "ep", "type": "metrics" }, "ecs": { "version": "8.0.0" }, + "elastic_agent": { + "id": "3cc85092-54dc-4b58-8726-5e9458167f42", + "snapshot": false, + "version": "8.5.0" + }, "event": { "agent_id_status": "verified", - "dataset": "logstash.pipeline", - "ingested": "2023-10-04T18:53:19Z" + "dataset": "logstash.stack_monitoring.node_stats", + "duration": 48419400, + "ingested": "2023-03-02T15:57:58Z", + "module": "logstash" }, "host": { "architecture": "x86_64", - "hostname": "macbook-pro.local", - "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", + "containerized": true, + "hostname": "docker-fleet-agent", + "id": "66392b0697b84641af8006d87aeb89f1", "ip": [ - "192.168.1.184" + "192.168.224.7" ], "mac": [ - "AC-DE-48-00-11-22" + "02-42-C0-A8-E0-07" ], - "name": "macbook-pro.local", + "name": "docker-fleet-agent", "os": { - "build": "22F82", - "family": "darwin", - "kernel": "22.5.0", - "name": "macOS", - "platform": "darwin", - "version": "13.4.1" + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" } }, - "input": { - "type": "cel" - }, "logstash": { - "pipeline": { - "host": { - "address": "0.0.0.0:9600", - "name": "21d61ee7529e" - }, - "name": "standalone-pipeline", - "total": { + "cluster": { + "id": "0toa26-cTzmqx0WD40-4XQ" + }, + "elasticsearch": { + "cluster": { + "id": "0toa26-cTzmqx0WD40-4XQ" + } + }, + "node": { + "stats": { "events": { - "filtered": 2038, - "in": 2038, - "out": 2038 + "duration_in_millis": 334, + "filtered": 138, + "in": 618, + "out": 138 }, - "flow": { - "filter_throughput": { - "current": 5.02, - "last_1_minute": 5.003 + "jvm": { + "gc": { + "collectors": { + "old": { + "collection_count": 0, + "collection_time_in_millis": 0 + }, + "young": { + "collection_count": 13, + "collection_time_in_millis": 177 + } + } }, - "input_throughput": { - "current": 4.948, - "last_1_minute": 5.003 + "mem": { + "heap_max_in_bytes": 10527703038, + "heap_used_in_bytes": 234688352, + "heap_used_percent": 2 }, - "output_throughput": { - "current": 5.02, - "last_1_minute": 5.003 + "uptime_in_millis": 21450 + }, + "logstash": { + "ephemeral_id": "17681d23-bd67-4c40-b6b1-63e97b560856", + "host": "170bc3698b89", + "http_address": "0.0.0.0:9600", + "name": "170bc3698b89", + "pipeline": { + "batch_size": 125, + "workers": 10 }, - "queue_backpressure": { - "current": 0, - "last_1_minute": 0 + "snapshot": false, + "status": "green", + "uuid": "a4224a67-aae8-4bce-8660-079d068b2e72", + "version": "8.5.0" + }, + "os": { + "cgroup": { + "cpu": { + "cfs_quota_micros": -1, + "control_group": "/", + "stat": { + "number_of_elapsed_periods": 0, + "number_of_times_throttled": 0, + "time_throttled_nanos": 0 + } + }, + "cpuacct": { + "control_group": "/", + "usage_nanos": 55911664431 + } + }, + "cpu": { + "load_average": { + "15m": 2.28, + "1m": 2.85, + "5m": 2.62 + }, + "percent": 0 + } + }, + "pipelines": [ + { + "ephemeral_id": "453a2361-82d8-4d88-b7a4-063c3293cd4a", + "events": { + "duration_in_millis": 0, + "filtered": 0, + "in": 476, + "out": 0, + "queue_push_duration_in_millis": 59 + }, + "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa", + "id": "pipeline-with-persisted-queue", + "queue": { + "capacity": { + "max_queue_size_in_bytes": 1073741824, + "max_unread_events": 0, + "page_capacity_in_bytes": 67108864, + "queue_size_in_bytes": 132880 + }, + "data": { + "free_space_in_bytes": 51709984768, + "path": "/usr/share/logstash/data/queue/pipeline-with-persisted-queue", + "storage_type": "overlay" + }, + "events": 0, + "events_count": 0, + "max_queue_size_in_bytes": 1073741824, + "queue_size_in_bytes": 132880, + "type": "persisted" + }, + "reloads": { + "failures": 0, + "successes": 0 + }, + "vertices": [ + { + "events_out": 475, + "id": "dfc132c40b9f5dbc970604f191cf87ee04b102b6f4be5a235436973dc7ea6368", + "pipeline_ephemeral_id": "453a2361-82d8-4d88-b7a4-063c3293cd4a", + "queue_push_duration_in_millis": 59 + }, + { + "duration_in_millis": 0, + "events_in": 375, + "events_out": 0, + "id": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", + "pipeline_ephemeral_id": "453a2361-82d8-4d88-b7a4-063c3293cd4a" + }, + { + "cluster_uuid": "0toa26-cTzmqx0WD40-4XQ", + "duration_in_millis": 1, + "events_in": 0, + "events_out": 0, + "id": "9ba6577aa5c41a5ebcaae010b9a0ef44015ae68c624596ed924417d1701abc21", + "pipeline_ephemeral_id": "453a2361-82d8-4d88-b7a4-063c3293cd4a" + } + ] }, - "worker_concurrency": { - "current": 0.001, - "last_1_minute": 0.001 + { + "ephemeral_id": "7114cd7d-8d91-4afc-a986-32487c3edcbe", + "events": { + "duration_in_millis": 191, + "filtered": 91, + "in": 95, + "out": 91, + "queue_push_duration_in_millis": 4 + }, + "hash": "0542fa70daa36dc3e858ea099f125cc8c9e451ebbfe8ea8867e52f9764da0a35", + "id": "pipeline-with-memory-queue", + "queue": { + "events_count": 0, + "max_queue_size_in_bytes": 0, + "queue_size_in_bytes": 0, + "type": "memory" + }, + "reloads": { + "failures": 0, + "successes": 0 + }, + "vertices": [ + { + "events_out": 95, + "id": "4c5941552cdaa72ebc285557c697a7150c359ee3eacf9b5664c4b1048e26153b", + "pipeline_ephemeral_id": "7114cd7d-8d91-4afc-a986-32487c3edcbe", + "queue_push_duration_in_millis": 4 + }, + { + "cluster_uuid": "0toa26-cTzmqx0WD40-4XQ", + "duration_in_millis": 193, + "events_in": 91, + "events_out": 91, + "id": "635a080aacc8700059852859da284a9cb92cb78a6d7112fbf55e441e51b6658a", + "long_counters": [ + { + "name": "bulk_requests.successes", + "value": 12 + }, + { + "name": "bulk_requests.responses.200", + "value": 12 + }, + { + "name": "documents.successes", + "value": 91 + } + ], + "pipeline_ephemeral_id": "7114cd7d-8d91-4afc-a986-32487c3edcbe" + } + ] } - }, - "queues": { - "current_size": { - "bytes": 0 - }, - "events": 0, - "max_size": { - "bytes": 0 + ], + "process": { + "cpu": { + "percent": 4 }, - "type": "memory" + "max_file_descriptors": 1048576, + "open_file_descriptors": 89 + }, + "queue": { + "events_count": 0 }, "reloads": { "failures": 0, "successes": 0 }, - "time": { - "duration": { - "ms": 1363 - }, - "queue_push_duration": { - "ms": 12 - } - } + "timestamp": "2023-03-02T15:57:57.016Z" } } + }, + "metricset": { + "name": "node_stats", + "period": 10000 + }, + "service": { + "address": "http://elastic-package-service_logstash_1:9600/_node/stats", + "hostname": "170bc3698b89", + "id": "", + "name": "logstash", + "type": "logstash", + "version": "8.5.0" } } ``` -#### Plugin - -This is the `plugin` dataset, which drives the Pipeline detail dashboard pages. Note that this dataset may produce many documents for logstash instances using a large number of pipelines and/or plugins within those pipelines. For those instances, we recommend reviewing the -pipeline collection period, and setting it to an appropriate value. - -#### Example - **Exported fields** -| Field | Description | Type | Unit | Metric Type | -|---|---|---|---|---| -| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | -| cloud.image.id | Image ID for the cloud instance. | keyword | | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | | -| cloud.instance.name | Instance name of the host machine. | keyword | | | -| cloud.machine.type | Machine type of the host machine. | keyword | | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | -| cloud.region | Region in which this host, resource, or service is located. | keyword | | | -| cluster_uuid | | alias | | | -| container.id | Unique container id. | keyword | | | -| container.image.name | Name of the image the container was built on. | keyword | | | -| container.labels | Image labels. | object | | | -| container.name | Container name. | keyword | | | -| data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | | -| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | | -| data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | -| error.message | Error message. | match_only_text | | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | -| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | | -| host.architecture | Operating system architecture. | keyword | | | -| host.containerized | If the host is a container. | boolean | | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | -| host.ip | Host ip addresses. | ip | | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | -| host.os.build | OS build information. | keyword | | | -| host.os.codename | OS codename, if any. | keyword | | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | | -| host.os.name | Operating system name, without the version. | keyword | | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | -| host.os.version | Operating system version as a raw string. | keyword | | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | -| input.type | | keyword | | | -| logstash.host.address | | alias | | | -| logstash.host.name | | alias | | | -| logstash.pipeline.elasticsearch.cluster.id | Elasticsearch clusters this Logstash pipeline is attached to | keyword | | | -| logstash.pipeline.host.address | address hosting this instance of logstash | keyword | | | -| logstash.pipeline.host.name | Host name of the node running logstash | keyword | | | -| logstash.pipeline.id | Logstash Pipeline hash | keyword | | | -| logstash.pipeline.name | Logstash Pipeline id/name | keyword | | | -| logstash.pipeline.plugin.codec.decode.duration.ms | amount of time spend decoding events | long | ms | counter | -| logstash.pipeline.plugin.codec.decode.in | number of events entering the decoder | long | | counter | -| logstash.pipeline.plugin.codec.decode.out | number of events exiting the decoder | long | | counter | -| logstash.pipeline.plugin.codec.encode.duration.ms | amount of time spend encoding events | long | ms | counter | -| logstash.pipeline.plugin.codec.encode.in | number of events encoded | long | | counter | -| logstash.pipeline.plugin.codec.id | Id of codec plugin | keyword | | | -| logstash.pipeline.plugin.codec.name | Name of codec plugin | keyword | | | -| logstash.pipeline.plugin.filter.elasticsearch.cluster.id | Elasticsearch clusters this Logstash plugin is attached to | keyword | | | -| logstash.pipeline.plugin.filter.events.in | number of events received by the filter | long | | counter | -| logstash.pipeline.plugin.filter.events.out | number of events emitted by the filter | long | | counter | -| logstash.pipeline.plugin.filter.flow.worker_millis_per_event.current | amount of time spent per event for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.filter.flow.worker_millis_per_event.last_1_minute | amount of time spent per event for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.filter.flow.worker_utilization.current | worker utilization for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.filter.flow.worker_utilization.last_1_minute | worker utilization for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.filter.id | Id of filter plugin | keyword | | | -| logstash.pipeline.plugin.filter.metrics.dissect.failures | number of dissect failures | long | | counter | -| logstash.pipeline.plugin.filter.metrics.dissect.matches | number of dissect matches | long | | counter | -| logstash.pipeline.plugin.filter.metrics.grok.failures | number of grok failures | long | | counter | -| logstash.pipeline.plugin.filter.metrics.grok.matches | number of grok matches | long | | counter | -| logstash.pipeline.plugin.filter.name | Name of filter plugin | keyword | | | -| logstash.pipeline.plugin.filter.source.column | | keyword | | | -| logstash.pipeline.plugin.filter.source.id | | keyword | | | -| logstash.pipeline.plugin.filter.source.line | | long | | | -| logstash.pipeline.plugin.filter.source.protocol | | keyword | | | -| logstash.pipeline.plugin.filter.time.duration.ms | amount of time working on events in this plugin | long | ms | counter | -| logstash.pipeline.plugin.input.elasticsearch.cluster.id | Elasticsearch clusters this Logstash plugin is attached to | keyword | | | -| logstash.pipeline.plugin.input.events.out | number of events emitted by the input | long | | counter | -| logstash.pipeline.plugin.input.flow.throughput.current | throughput of this input plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.input.flow.throughput.last_1_minute | throughput of this input plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.input.id | Id of input plugin | keyword | | | -| logstash.pipeline.plugin.input.name | Name of input plugin | keyword | | | -| logstash.pipeline.plugin.input.source.column | | keyword | | | -| logstash.pipeline.plugin.input.source.id | | keyword | | | -| logstash.pipeline.plugin.input.source.line | | long | | | -| logstash.pipeline.plugin.input.source.protocol | | keyword | | | -| logstash.pipeline.plugin.input.time.queue_push_duration.ms | amount of time spend pushing events to the queue | long | ms | counter | -| logstash.pipeline.plugin.output.elasticsearch.cluster.id | Elasticsearch clusters this Logstash plugin is attached to | keyword | | | -| logstash.pipeline.plugin.output.events.in | number of events received by the output | long | | counter | -| logstash.pipeline.plugin.output.events.out | number of events emitted by the output | long | | counter | -| logstash.pipeline.plugin.output.flow.worker_millis_per_event.current | amount of time spent per event for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.output.flow.worker_millis_per_event.last_1_minute | amount of time spent per event for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.output.flow.worker_utilization.current | worker utilization for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.output.flow.worker_utilization.last_1_minute | worker utilization for this plugin | scaled_float | | gauge | -| logstash.pipeline.plugin.output.id | Id of output plugin | keyword | | | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.200 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.201 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.400 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.401 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.403 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.404 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.409 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.413 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.429 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.responses.500 | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.bulk_requests.successes | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.documents.non_retryable_failures | | long | | counter | -| logstash.pipeline.plugin.output.metrics.elasticsearch.documents.successes | | long | | counter | -| logstash.pipeline.plugin.output.name | Name of output plugin | keyword | | | -| logstash.pipeline.plugin.output.source.column | | keyword | | | -| logstash.pipeline.plugin.output.source.id | | keyword | | | -| logstash.pipeline.plugin.output.source.line | | long | | | -| logstash.pipeline.plugin.output.source.protocol | | keyword | | | -| logstash.pipeline.plugin.output.time.duration.ms | amount of time working on events in this plugin | long | ms | counter | -| logstash.pipeline.plugin.type | Type of the plugin | keyword | | | -| process.pid | Process id. | long | | | -| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | -| service.hostname | Hostname of the service | keyword | | | -| service.id | Unique identifier of the running service. If the service is comprised of many nodes, the `service.id` should be the same for all nodes. This id should uniquely identify the service. This makes it possible to correlate logs and metrics for one specific service, no matter which particular node emitted the event. Note that if you need to see the events from one specific host of the service, you should filter on that `host.name` or `host.id` instead. | keyword | | | -| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | -| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | | | +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| logstash.node.jvm.version | Version | keyword | +| logstash.node.state.pipeline.hash | | keyword | +| logstash.node.state.pipeline.id | | keyword | +| logstash.node.stats.events.duration_in_millis | | long | +| logstash.node.stats.events.filtered | Filtered events counter. | long | +| logstash.node.stats.events.in | Incoming events counter. | long | +| logstash.node.stats.events.out | Outgoing events counter. | long | +| logstash.node.stats.jvm.mem.heap_max_in_bytes | | long | +| logstash.node.stats.jvm.mem.heap_used_in_bytes | | long | +| logstash.node.stats.jvm.uptime_in_millis | | long | +| logstash.node.stats.logstash.uuid | | keyword | +| logstash.node.stats.logstash.version | | keyword | +| logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods | | long | +| logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled | | long | +| logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos | | long | +| logstash.node.stats.os.cgroup.cpuacct.usage_nanos | | long | +| logstash.node.stats.os.cpu.load_average.15m | | long | +| logstash.node.stats.os.cpu.load_average.1m | | long | +| logstash.node.stats.os.cpu.load_average.5m | | long | +| logstash.node.stats.pipelines.events.duration_in_millis | | long | +| logstash.node.stats.pipelines.events.out | | long | +| logstash.node.stats.pipelines.hash | | keyword | +| logstash.node.stats.pipelines.id | | keyword | +| logstash.node.stats.pipelines.queue.events_count | | long | +| logstash.node.stats.pipelines.queue.max_queue_size_in_bytes | | long | +| logstash.node.stats.pipelines.queue.queue_size_in_bytes | | long | +| logstash.node.stats.pipelines.queue.type | | keyword | +| logstash.node.stats.pipelines.vertices.duration_in_millis | | long | +| logstash.node.stats.pipelines.vertices.events_in | | long | +| logstash.node.stats.pipelines.vertices.events_out | events_out | long | +| logstash.node.stats.pipelines.vertices.id | id | keyword | +| logstash.node.stats.pipelines.vertices.pipeline_ephemeral_id | pipeline_ephemeral_id | keyword | +| logstash.node.stats.pipelines.vertices.queue_push_duration_in_millis | queue_push_duration_in_millis | float | +| logstash.node.stats.process.cpu.percent | | double | +| logstash.node.stats.queue.events_count | | long | +| logstash_stats.pipelines | | nested | +| process.pid | Process id. | long | +| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | +### Node -An example event for `plugins` looks as following: +An example event for `node` looks as following: ```json { - "@timestamp": "2023-10-24T17:56:40.316Z", + "@timestamp": "2023-03-02T15:57:03.999Z", + "agent": { + "ephemeral_id": "16f2dd63-454b-4699-a8c8-2a748bd044b8", + "id": "3cc85092-54dc-4b58-8726-5e9458167f42", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.5.0" + }, "data_stream": { - "dataset": "logstash.plugins", - "namespace": "default", + "dataset": "logstash.stack_monitoring.node", + "namespace": "ep", "type": "metrics" }, "ecs": { "version": "8.0.0" }, + "elastic_agent": { + "id": "3cc85092-54dc-4b58-8726-5e9458167f42", + "snapshot": false, + "version": "8.5.0" + }, "event": { "agent_id_status": "verified", - "dataset": "logstash.plugins", - "ingested": "2023-10-24T17:56:41Z" + "dataset": "logstash.stack_monitoring.node", + "duration": 69490100, + "ingested": "2023-03-02T15:57:05Z", + "module": "logstash" }, "host": { "architecture": "x86_64", - "hostname": "macbook-pro.local", - "id": "AA4215F6-994F-5CCE-B6F2-B6AED75AE125", + "containerized": true, + "hostname": "docker-fleet-agent", + "id": "66392b0697b84641af8006d87aeb89f1", "ip": [ - "192.168.4.26" + "192.168.224.7" ], "mac": [ - "AC-DE-48-00-11-22" + "02-42-C0-A8-E0-07" ], - "name": "macbook-pro.local", + "name": "docker-fleet-agent", "os": { - "build": "22G120", - "family": "darwin", - "kernel": "22.6.0", - "name": "macOS", - "platform": "darwin", - "version": "13.6" + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" } }, - "input": { - "type": "cel" - }, "logstash": { - "pipeline": { - "elasticsearch": { - "cluster": { - "id": "9MOGoKiESvaklNVmxLo3iA" - } - }, - "host": { - "address": "127.0.0.1:9602", - "name": "logstash9602" + "cluster": { + "id": "0toa26-cTzmqx0WD40-4XQ" + }, + "elasticsearch": { + "cluster": { + "id": "0toa26-cTzmqx0WD40-4XQ" + } + }, + "node": { + "host": "45730b5f8c3d", + "id": "2e17cd45-ecb8-4358-a420-b867f2e32b7a", + "jvm": { + "version": "17.0.4" }, - "id": "b18ff60bcd82055aab2bf5601a2bc170502f80b33ab5938f25fa95ec8b04cd4b", - "name": "work", - "plugin": { - "output": { - "elasticsearch": { - "cluster": { - "id": "9MOGoKiESvaklNVmxLo3iA" - } - }, - "events": { - "in": 798, - "out": 798 - }, - "flow": { - "worker_millis_per_event": { - "current": 54, - "last_1_minute": 54 - }, - "worker_utilization": { - "current": 0.023, - "last_1_minute": 0.01 - } - }, - "id": "out_to_elasticsearch", - "metrics": { - "elasticsearch": { - "bulk_requests": { - "responses": { - "200": 798 + "state": { + "pipeline": { + "batch_size": 125, + "ephemeral_id": "472cf082-aa15-41ca-9ed1-62d03afbadd0", + "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa", + "id": "pipeline-with-persisted-queue", + "representation": { + "graph": { + "edges": [ + { + "from": "dfc132c40b9f5dbc970604f191cf87ee04b102b6f4be5a235436973dc7ea6368", + "id": "9ed824e4f189b461c111ae27c17644c3c5f6d7c3c2bb213cbc7cc067cbd68fe6", + "to": "__QUEUE__", + "type": "plain" }, - "successes": 798 - }, - "documents": { - "successes": 798 - } - } - }, - "name": "elasticsearch", - "source": { - "column": "3", - "id": "/Users/test/ingestdemo/logstash-8.8.2/remap.conf", - "line": 132, - "protocol": "file" + { + "from": "__QUEUE__", + "id": "cb33f8fb7611e31a2c1751b74cdedf5b8cdb96ea46b812a2541e2db4f13dca10", + "to": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", + "type": "plain" + }, + { + "from": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", + "id": "63ef166c45b87a40f31e0a6def175f10460b6b0ed656e70968eb52b1c454ab16", + "to": "9ba6577aa5c41a5ebcaae010b9a0ef44015ae68c624596ed924417d1701abc21", + "type": "plain" + } + ], + "vertices": [ + { + "config_name": "java_generator", + "explicit_id": false, + "id": "dfc132c40b9f5dbc970604f191cf87ee04b102b6f4be5a235436973dc7ea6368", + "meta": { + "source": { + "column": 3, + "id": "/usr/share/logstash/pipeline/persisted-queue.conf", + "line": 2, + "protocol": "file" + } + }, + "plugin_type": "input", + "type": "plugin" + }, + { + "explicit_id": false, + "id": "__QUEUE__", + "meta": null, + "type": "queue" + }, + { + "config_name": "sleep", + "explicit_id": false, + "id": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", + "meta": { + "source": { + "column": 3, + "id": "/usr/share/logstash/pipeline/persisted-queue.conf", + "line": 8, + "protocol": "file" + } + }, + "plugin_type": "filter", + "type": "plugin" + }, + { + "config_name": "elasticsearch", + "explicit_id": false, + "id": "9ba6577aa5c41a5ebcaae010b9a0ef44015ae68c624596ed924417d1701abc21", + "meta": { + "source": { + "column": 3, + "id": "/usr/share/logstash/pipeline/persisted-queue.conf", + "line": 15, + "protocol": "file" + } + }, + "plugin_type": "output", + "type": "plugin" + } + ] + }, + "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa", + "type": "lir", + "version": "0.0.0" }, - "time": { - "duration": { - "ms": 198060 - } - } - }, - "type": "output" - } + "workers": 10 + } + }, + "version": "8.5.0" } + }, + "metricset": { + "name": "node", + "period": 10000 + }, + "process": { + "pid": 1 + }, + "service": { + "address": "http://elastic-package-service_logstash_1:9600/_node", + "hostname": "45730b5f8c3d", + "id": "2e17cd45-ecb8-4358-a420-b867f2e32b7a", + "name": "logstash", + "type": "logstash", + "version": "8.5.0" } } ``` + + diff --git a/packages/logstash/kibana/dashboard/logstash-838aac39-8edd-48b0-95b4-289e42b1e98a.json b/packages/logstash/kibana/dashboard/logstash-838aac39-8edd-48b0-95b4-289e42b1e98a.json new file mode 100644 index 00000000000..7572ba3c6ad --- /dev/null +++ b/packages/logstash/kibana/dashboard/logstash-838aac39-8edd-48b0-95b4-289e42b1e98a.json @@ -0,0 +1,3090 @@ +{ + "attributes": { + "controlGroupInput": { + "chainingSystem": "HIERARCHICAL", + "controlStyle": "oneLine", + "ignoreParentSettingsJSON": { + "ignoreFilters": false, + "ignoreQuery": false, + "ignoreTimerange": false, + "ignoreValidations": false + }, + "panelsJSON": { + "c010ef1f-9346-41c1-a252-bcb3d2ca8cb0": { + "explicitInput": { + "dataViewId": "metrics-*", + "exclude": null, + "existsSelected": null, + "fieldName": "logstash.node.name", + "hideActionBar": null, + "hideExclude": null, + "hideExists": null, + "hideSort": null, + "id": "c010ef1f-9346-41c1-a252-bcb3d2ca8cb0", + "placeholder": null, + "runPastTimeout": null, + "searchTechnique": "prefix", + "selectedOptions": [], + "singleSelect": null, + "sort": { + "by": "_count", + "direction": "desc" + }, + "title": "Logstash Node" + }, + "grow": true, + "order": 0, + "type": "optionsListControl", + "width": "medium" + }, + "fe3c3404-df75-4ce5-9358-b3467a7755f0": { + "explicitInput": { + "dataViewId": "metrics-*", + "exclude": null, + "existsSelected": false, + "fieldName": "logstash.pipeline.id", + "hideActionBar": null, + "hideExclude": null, + "hideExists": null, + "hideSort": null, + "id": "fe3c3404-df75-4ce5-9358-b3467a7755f0", + "placeholder": null, + "runPastTimeout": null, + "searchTechnique": "prefix", + "selectedOptions": [], + "singleSelect": null, + "sort": { + "by": "_count", + "direction": "desc" + }, + "title": "Pipeline Id" + }, + "grow": true, + "order": 1, + "type": "optionsListControl", + "width": "medium" + } + }, + "showApplySelections": false + }, + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "layout": "horizontal", + "links": [ + { + "destinationRefName": "link_097a2e6e-6ccf-4aeb-ad60-48bb1dfc37e9_dashboard", + "id": "097a2e6e-6ccf-4aeb-ad60-48bb1dfc37e9", + "label": "Node Health", + "order": 0, + "type": "dashboardLink" + }, + { + "destinationRefName": "link_e8e52960-2726-45bc-8a1d-bd8d4e73cffb_dashboard", + "id": "e8e52960-2726-45bc-8a1d-bd8d4e73cffb", + "label": "Pipeline Health", + "order": 1, + "type": "dashboardLink" + } + ] + }, + "enhancements": {} + }, + "gridData": { + "h": 2, + "i": "794f0447-82c3-4747-9176-d78b7dd1eb12", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "794f0447-82c3-4747-9176-d78b7dd1eb12", + "type": "links" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "metrics-*", + "layers": { + "dce344e8-e3dd-42fc-b474-35a87078a4d2": { + "columnOrder": [ + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8" + ], + "columns": { + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total Nodes", + "operationType": "unique_count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.node.uuid" + } + }, + "incompleteColumns": {}, + "indexPatternId": "metrics-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "data_stream.dataset", + "index": "0d266976-a94d-4f05-b70f-f09b368a5426", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "logstash.health_report" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "logstash.health_report" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layerId": "dce344e8-e3dd-42fc-b474-35a87078a4d2", + "layerType": "data", + "metricAccessor": "f6c066c9-58b1-43cc-a4da-2192ccaf30e8" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "829075a9-1736-4515-ba87-cf99c519a09b", + "w": 7, + "x": 0, + "y": 2 + }, + "panelIndex": "829075a9-1736-4515-ba87-cf99c519a09b", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "metrics-*", + "layers": { + "dce344e8-e3dd-42fc-b474-35a87078a4d2": { + "columnOrder": [ + "c71287ed-1e77-4d38-8eef-cd4ee7e3768a", + "205777e2-953b-4b28-aa30-3aa56c30460e", + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8" + ], + "columns": { + "205777e2-953b-4b28-aa30-3aa56c30460e": { + "customLabel": true, + "dataType": "date", + "isBucketed": true, + "label": "Status", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "c71287ed-1e77-4d38-8eef-cd4ee7e3768a": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Pipeline Status", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f6c066c9-58b1-43cc-a4da-2192ccaf30e8", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 100 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.status" + }, + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Percentage by status", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "___records___" + } + }, + "incompleteColumns": {}, + "indexPatternId": "metrics-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "data_stream.dataset", + "index": "0d266976-a94d-4f05-b70f-f09b368a5426", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "logstash.health_report" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "logstash.health_report" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8" + ], + "colorMapping": { + "assignments": [ + { + "color": { + "colorIndex": 0, + "paletteId": "eui_amsterdam_color_blind", + "type": "categorical" + }, + "rule": { + "type": "matchExactly", + "values": [ + "green" + ] + }, + "touched": true + }, + { + "color": { + "colorIndex": 9, + "paletteId": "eui_amsterdam_color_blind", + "type": "categorical" + }, + "rule": { + "type": "matchExactly", + "values": [ + "red" + ] + }, + "touched": true + }, + { + "color": { + "colorIndex": 5, + "paletteId": "eui_amsterdam_color_blind", + "type": "categorical" + }, + "rule": { + "type": "matchExactly", + "values": [ + "yellow" + ] + }, + "touched": true + }, + { + "color": { + "colorIndex": 1, + "paletteId": "neutral", + "type": "categorical" + }, + "rule": { + "type": "matchExactly", + "values": [ + "unknown" + ] + }, + "touched": true + } + ], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "dce344e8-e3dd-42fc-b474-35a87078a4d2", + "layerType": "data", + "seriesType": "bar_percentage_stacked", + "splitAccessor": "c71287ed-1e77-4d38-8eef-cd4ee7e3768a", + "xAccessor": "205777e2-953b-4b28-aa30-3aa56c30460e" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {} + }, + "gridData": { + "h": 10, + "i": "cb1751b1-7889-4c9f-84ec-8e4332e47d89", + "w": 41, + "x": 7, + "y": 2 + }, + "panelIndex": "cb1751b1-7889-4c9f-84ec-8e4332e47d89", + "title": "Pipeline status over time", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "metrics-*", + "layers": { + "dce344e8-e3dd-42fc-b474-35a87078a4d2": { + "columnOrder": [ + "cb775650-62c5-41d0-a847-f1feca23e409", + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8" + ], + "columns": { + "cb775650-62c5-41d0-a847-f1feca23e409": { + "dataType": "string", + "isBucketed": true, + "label": "Top 1000 values of logstash.node.uuid", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f6c066c9-58b1-43cc-a4da-2192ccaf30e8", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.node.uuid" + }, + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total Pipelines", + "operationType": "unique_count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.id" + } + }, + "incompleteColumns": {}, + "indexPatternId": "metrics-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "data_stream.dataset", + "index": "0d266976-a94d-4f05-b70f-f09b368a5426", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "logstash.health_report" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "logstash.health_report" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "breakdownByAccessor": "cb775650-62c5-41d0-a847-f1feca23e409", + "collapseFn": "sum", + "layerId": "dce344e8-e3dd-42fc-b474-35a87078a4d2", + "layerType": "data", + "maxCols": 1, + "metricAccessor": "f6c066c9-58b1-43cc-a4da-2192ccaf30e8" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "80a338b2-c98a-4878-a065-1a340c954d69", + "w": 7, + "x": 0, + "y": 7 + }, + "panelIndex": "80a338b2-c98a-4878-a065-1a340c954d69", + "type": "lens" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "## Failed Pipelines", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 3, + "i": "148b36e6-4586-46d7-93c1-b19c9528807e", + "w": 48, + "x": 0, + "y": 12 + }, + "panelIndex": "148b36e6-4586-46d7-93c1-b19c9528807e", + "title": "", + "type": "visualization" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "metrics-*", + "layers": { + "dce344e8-e3dd-42fc-b474-35a87078a4d2": { + "columnOrder": [ + "73cd68c0-d7a0-4815-98ff-d4189665a84e", + "0319cc89-38ed-4476-a279-3214a4308f89", + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8", + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X0", + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X1", + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X2" + ], + "columns": { + "0319cc89-38ed-4476-a279-3214a4308f89": { + "customLabel": true, + "dataType": "date", + "isBucketed": true, + "label": "Pipelines with a reported failure", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "73cd68c0-d7a0-4815-98ff-d4189665a84e": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Pipelines on nodes", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.node.name" + ], + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.id" + }, + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Instances", + "operationType": "formula", + "params": { + "format": { + "id": "custom", + "params": { + "decimals": 0, + "pattern": "." + } + }, + "formula": "count()/count()", + "isFormulaBroken": false + }, + "references": [ + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X2" + ], + "scale": "ratio" + }, + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Count", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "___records___" + }, + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Count", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "___records___" + }, + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Count", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X0", + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X1" + ], + "location": { + "max": 15, + "min": 0 + }, + "name": "divide", + "text": "count()/count()", + "type": "function" + } + }, + "references": [ + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X0", + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X1" + ], + "scale": "ratio" + } + }, + "incompleteColumns": {}, + "indexPatternId": "metrics-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "logstash.pipeline.status", + "index": "a38d1f37-fc81-42cc-baf4-865bcefe3cb1", + "key": "logstash.pipeline.status", + "negate": false, + "params": { + "query": "red" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "logstash.pipeline.status": "red" + } + } + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "data_stream.dataset", + "index": "0d266976-a94d-4f05-b70f-f09b368a5426", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "logstash.health_report" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "logstash.health_report" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8" + ], + "layerId": "dce344e8-e3dd-42fc-b474-35a87078a4d2", + "layerType": "data", + "palette": { + "name": "negative", + "type": "palette" + }, + "seriesType": "bar_stacked", + "splitAccessor": "73cd68c0-d7a0-4815-98ff-d4189665a84e", + "xAccessor": "0319cc89-38ed-4476-a279-3214a4308f89" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "description": "Pipelines reporting any failures within the time period", + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "f10b370b-8cb4-42b1-9a61-63c0366bc9bb", + "w": 48, + "x": 0, + "y": 15 + }, + "panelIndex": "f10b370b-8cb4-42b1-9a61-63c0366bc9bb", + "title": "Pipelines with failures over time", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "metrics-*", + "layers": { + "dce344e8-e3dd-42fc-b474-35a87078a4d2": { + "columnOrder": [ + "464fd4fa-3499-4309-8d0a-ccab324315d8", + "e7871c70-0c9b-4585-ad2d-6ed2de96150d", + "709f9b5c-828b-444d-a46c-bf9cb7d963bd", + "2b988057-642e-493b-b22d-e1c3e79e0448", + "25e0f6f0-5965-415b-a2ae-c6b818c9ac27" + ], + "columns": { + "25e0f6f0-5965-415b-a2ae-c6b818c9ac27": { + "customLabel": true, + "dataType": "string", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.diagnosis.help_url\": *" + }, + "isBucketed": false, + "label": "Help", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.diagnosis.help_url" + }, + "2b988057-642e-493b-b22d-e1c3e79e0448": { + "customLabel": true, + "dataType": "string", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.diagnosis.action\": *" + }, + "isBucketed": false, + "label": "Action", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.diagnosis.action" + }, + "464fd4fa-3499-4309-8d0a-ccab324315d8": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Node and Pipeline", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.id" + ], + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.node.name" + }, + "709f9b5c-828b-444d-a46c-bf9cb7d963bd": { + "customLabel": true, + "dataType": "string", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.diagnosis.cause\": *" + }, + "isBucketed": false, + "label": "Cause", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.diagnosis.cause" + }, + "e7871c70-0c9b-4585-ad2d-6ed2de96150d": { + "customLabel": true, + "dataType": "string", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.state\": *" + }, + "isBucketed": false, + "label": "State", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.state" + } + }, + "incompleteColumns": {}, + "indexPatternId": "metrics-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "logstash.pipeline.status", + "index": "850108e6-1654-46d4-ba22-618ea8d132f6", + "key": "logstash.pipeline.status", + "negate": false, + "params": { + "query": "red" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "logstash.pipeline.status": "red" + } + } + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "data_stream.dataset", + "index": "0d266976-a94d-4f05-b70f-f09b368a5426", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "logstash.health_report" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "logstash.health_report" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "464fd4fa-3499-4309-8d0a-ccab324315d8", + "isTransposed": false + }, + { + "columnId": "e7871c70-0c9b-4585-ad2d-6ed2de96150d", + "isMetric": true, + "isTransposed": false, + "width": 111.85714285714285 + }, + { + "columnId": "709f9b5c-828b-444d-a46c-bf9cb7d963bd", + "isMetric": true, + "isTransposed": false + }, + { + "columnId": "2b988057-642e-493b-b22d-e1c3e79e0448", + "isMetric": true, + "isTransposed": false + }, + { + "columnId": "25e0f6f0-5965-415b-a2ae-c6b818c9ac27", + "isMetric": true, + "isTransposed": false + } + ], + "layerId": "dce344e8-e3dd-42fc-b474-35a87078a4d2", + "layerType": "data", + "rowHeight": "auto" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 12, + "i": "6ff38919-fafa-4fe1-b28e-2fd42e6d3783", + "w": 48, + "x": 0, + "y": 23 + }, + "panelIndex": "6ff38919-fafa-4fe1-b28e-2fd42e6d3783", + "title": "Failed Pipeline Details", + "type": "lens" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "id": "", + "params": { + "fontSize": 12, + "markdown": "## Concerning Pipelines", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 3, + "i": "7ff3e6ed-aec2-4b79-b05b-897f0c222471", + "w": 48, + "x": 0, + "y": 35 + }, + "panelIndex": "7ff3e6ed-aec2-4b79-b05b-897f0c222471", + "type": "visualization" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "metrics-*", + "layers": { + "dce344e8-e3dd-42fc-b474-35a87078a4d2": { + "columnOrder": [ + "73cd68c0-d7a0-4815-98ff-d4189665a84e", + "0319cc89-38ed-4476-a279-3214a4308f89", + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8", + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X0", + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X1", + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X2" + ], + "columns": { + "0319cc89-38ed-4476-a279-3214a4308f89": { + "customLabel": true, + "dataType": "date", + "isBucketed": true, + "label": "Pipelines with a concerning report", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "73cd68c0-d7a0-4815-98ff-d4189665a84e": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "pipeline -\u003e node", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.node.name" + ], + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.id" + }, + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Instances", + "operationType": "formula", + "params": { + "format": { + "id": "custom", + "params": { + "decimals": 0, + "pattern": "." + } + }, + "formula": "count()/count()", + "isFormulaBroken": false + }, + "references": [ + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X2" + ], + "scale": "ratio" + }, + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Count", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "___records___" + }, + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Count", + "operationType": "count", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "___records___" + }, + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Count", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X0", + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X1" + ], + "location": { + "max": 15, + "min": 0 + }, + "name": "divide", + "text": "count()/count()", + "type": "function" + } + }, + "references": [ + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X0", + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8X1" + ], + "scale": "ratio" + } + }, + "incompleteColumns": {}, + "indexPatternId": "metrics-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "logstash.pipeline.status", + "index": "f95628df-9210-4b34-882a-06933fc16961", + "key": "logstash.pipeline.status", + "negate": false, + "params": { + "query": "yellow" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "logstash.pipeline.status": "yellow" + } + } + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "data_stream.dataset", + "index": "0d266976-a94d-4f05-b70f-f09b368a5426", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "logstash.health_report" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "logstash.health_report" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f6c066c9-58b1-43cc-a4da-2192ccaf30e8" + ], + "layerId": "dce344e8-e3dd-42fc-b474-35a87078a4d2", + "layerType": "data", + "palette": { + "name": "warm", + "type": "palette" + }, + "seriesType": "bar_stacked", + "splitAccessor": "73cd68c0-d7a0-4815-98ff-d4189665a84e", + "xAccessor": "0319cc89-38ed-4476-a279-3214a4308f89" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "description": "Indicates whether a pipeline on a node reported a yellow or unknown status within a given time period. A pipeline may have reported a different status within the same time window.", + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "cffb0a85-9bbc-4cd7-9cfc-d55246a8e1a6", + "w": 48, + "x": 0, + "y": 38 + }, + "panelIndex": "cffb0a85-9bbc-4cd7-9cfc-d55246a8e1a6", + "title": "Pipelines with concerning status over time", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "metrics-*", + "layers": { + "dce344e8-e3dd-42fc-b474-35a87078a4d2": { + "columnOrder": [ + "464fd4fa-3499-4309-8d0a-ccab324315d8", + "24fd7ec9-9b86-4e0d-936a-da9b0679272b", + "e7871c70-0c9b-4585-ad2d-6ed2de96150d", + "709f9b5c-828b-444d-a46c-bf9cb7d963bd", + "2b988057-642e-493b-b22d-e1c3e79e0448", + "25e0f6f0-5965-415b-a2ae-c6b818c9ac27" + ], + "columns": { + "24fd7ec9-9b86-4e0d-936a-da9b0679272b": { + "customLabel": true, + "dataType": "string", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.status\": *" + }, + "isBucketed": false, + "label": "Status", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.status" + }, + "25e0f6f0-5965-415b-a2ae-c6b818c9ac27": { + "customLabel": true, + "dataType": "string", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.diagnosis.help_url\": *" + }, + "isBucketed": false, + "label": "Help", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.diagnosis.help_url" + }, + "2b988057-642e-493b-b22d-e1c3e79e0448": { + "customLabel": true, + "dataType": "string", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.diagnosis.action\": *" + }, + "isBucketed": false, + "label": "Action", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.diagnosis.action" + }, + "464fd4fa-3499-4309-8d0a-ccab324315d8": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Node and pipeline", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.id" + ], + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.node.name" + }, + "709f9b5c-828b-444d-a46c-bf9cb7d963bd": { + "customLabel": true, + "dataType": "string", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.diagnosis.cause\": *" + }, + "isBucketed": false, + "label": "Cause", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.diagnosis.cause" + }, + "e7871c70-0c9b-4585-ad2d-6ed2de96150d": { + "customLabel": true, + "dataType": "string", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.state\": *" + }, + "isBucketed": false, + "label": "State", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.state" + } + }, + "incompleteColumns": {}, + "indexPatternId": "metrics-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "657bcd10-7d07-41fd-ba7c-0ffd9e26053a", + "negate": false, + "params": [ + { + "meta": { + "alias": null, + "disabled": false, + "field": "logstash.pipeline.status", + "index": "metrics-*", + "key": "logstash.pipeline.status", + "negate": true, + "params": { + "query": "green" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "logstash.pipeline.status": "green" + } + } + }, + { + "meta": { + "alias": null, + "disabled": false, + "field": "logstash.pipeline.status", + "index": "metrics-*", + "key": "logstash.pipeline.status", + "negate": true, + "params": { + "query": "red" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "logstash.pipeline.status": "red" + } + } + } + ], + "relation": "AND", + "type": "combined" + }, + "query": {} + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "data_stream.dataset", + "index": "0d266976-a94d-4f05-b70f-f09b368a5426", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "logstash.health_report" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "logstash.health_report" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "464fd4fa-3499-4309-8d0a-ccab324315d8", + "isTransposed": false + }, + { + "columnId": "24fd7ec9-9b86-4e0d-936a-da9b0679272b", + "isMetric": true, + "isTransposed": false, + "width": 70.85714285714286 + }, + { + "columnId": "e7871c70-0c9b-4585-ad2d-6ed2de96150d", + "isMetric": true, + "isTransposed": false, + "width": 83.85714285714285 + }, + { + "columnId": "709f9b5c-828b-444d-a46c-bf9cb7d963bd", + "isMetric": true, + "isTransposed": false + }, + { + "columnId": "2b988057-642e-493b-b22d-e1c3e79e0448", + "isMetric": true, + "isTransposed": false + }, + { + "columnId": "25e0f6f0-5965-415b-a2ae-c6b818c9ac27", + "isMetric": true, + "isTransposed": false + } + ], + "layerId": "dce344e8-e3dd-42fc-b474-35a87078a4d2", + "layerType": "data", + "rowHeight": "auto" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "description": "Pipelines that have reported a concerning health status at least once during the interval selected ", + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 12, + "i": "8f81966e-36f2-4c0e-8844-afc094f8c346", + "w": 48, + "x": 0, + "y": 46 + }, + "panelIndex": "8f81966e-36f2-4c0e-8844-afc094f8c346", + "title": "Pipelines reporting a concerning health status ", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "metrics-*", + "layers": { + "dce344e8-e3dd-42fc-b474-35a87078a4d2": { + "columnOrder": [ + "464fd4fa-3499-4309-8d0a-ccab324315d8", + "4d81dbc6-18a0-404c-a8a0-3a56d69596ef" + ], + "columns": { + "464fd4fa-3499-4309-8d0a-ccab324315d8": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Node and pipeline", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "4d81dbc6-18a0-404c-a8a0-3a56d69596ef", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.id" + ], + "size": 100 + }, + "scale": "ordinal", + "sourceField": "logstash.node.name" + }, + "4d81dbc6-18a0-404c-a8a0-3a56d69596ef": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.flow.worker_utilization.last_1_minute\": *" + }, + "isBucketed": false, + "label": "Utilization (1 min)", + "operationType": "last_value", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 0, + "suffix": "%" + } + }, + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.flow.worker_utilization.last_1_minute" + } + }, + "incompleteColumns": {}, + "indexPatternId": "metrics-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "logstash.pipeline.flow.worker_utilization.last_1_minute", + "index": "2a34254e-8c52-4fe8-bb1f-1bf24b62a106", + "key": "logstash.pipeline.flow.worker_utilization.last_1_minute", + "negate": false, + "params": { + "gte": "50" + }, + "type": "range" + }, + "query": { + "range": { + "logstash.pipeline.flow.worker_utilization.last_1_minute": { + "gte": "50" + } + } + } + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "data_stream.dataset", + "index": "0d266976-a94d-4f05-b70f-f09b368a5426", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "logstash.health_report" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "logstash.health_report" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "464fd4fa-3499-4309-8d0a-ccab324315d8", + "isTransposed": false, + "width": 301.6666666666667 + }, + { + "columnId": "4d81dbc6-18a0-404c-a8a0-3a56d69596ef", + "isTransposed": false + } + ], + "headerRowHeightLines": 2, + "layerId": "dce344e8-e3dd-42fc-b474-35a87078a4d2", + "layerType": "data", + "rowHeight": "single", + "rowHeightLines": 1, + "sorting": { + "columnId": "4d81dbc6-18a0-404c-a8a0-3a56d69596ef", + "direction": "desc" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 11, + "i": "015c903c-0087-4d87-bd21-5b6ef9ab70e2", + "w": 15, + "x": 0, + "y": 58 + }, + "panelIndex": "015c903c-0087-4d87-bd21-5b6ef9ab70e2", + "title": "Pipelines with high utilization", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-aa9f3aa1-a8c4-496e-942e-b19ada17869b", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "metrics-*", + "layers": { + "aa9f3aa1-a8c4-496e-942e-b19ada17869b": { + "columnOrder": [ + "b16dd086-965c-4857-ada6-126a5925a865" + ], + "columns": { + "b16dd086-965c-4857-ada6-126a5925a865": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "isStaticValue": true, + "label": "Static value: 50", + "operationType": "static_value", + "params": { + "value": "50" + }, + "references": [], + "scale": "ratio" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "indexPatternId": "metrics-*", + "linkToLayers": [], + "sampling": 1 + }, + "dce344e8-e3dd-42fc-b474-35a87078a4d2": { + "columnOrder": [ + "464fd4fa-3499-4309-8d0a-ccab324315d8", + "9538ba82-3742-4452-98b6-b0ba7309800d", + "4d81dbc6-18a0-404c-a8a0-3a56d69596ef" + ], + "columns": { + "464fd4fa-3499-4309-8d0a-ccab324315d8": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Node and pipeline", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "4d81dbc6-18a0-404c-a8a0-3a56d69596ef", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.node.name" + ], + "size": 1000 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.id" + }, + "4d81dbc6-18a0-404c-a8a0-3a56d69596ef": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.flow.worker_utilization.last_1_minute\": *" + }, + "isBucketed": false, + "label": "Load (1 min)", + "operationType": "last_value", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 0, + "suffix": "%" + } + }, + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.flow.worker_utilization.last_1_minute" + }, + "9538ba82-3742-4452-98b6-b0ba7309800d": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {}, + "indexPatternId": "metrics-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "logstash.pipeline.flow.worker_utilization.last_1_minute", + "index": "aeeb7538-bb36-4fbd-ab65-4bdb1956a76a", + "key": "logstash.pipeline.flow.worker_utilization.last_1_minute", + "negate": false, + "params": { + "gte": "50" + }, + "type": "range", + "value": { + "gte": "50" + } + }, + "query": { + "range": { + "logstash.pipeline.flow.worker_utilization.last_1_minute": { + "gte": "50" + } + } + } + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "data_stream.dataset", + "index": "0d266976-a94d-4f05-b70f-f09b368a5426", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "logstash.health_report" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "logstash.health_report" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "4d81dbc6-18a0-404c-a8a0-3a56d69596ef" + ], + "layerId": "dce344e8-e3dd-42fc-b474-35a87078a4d2", + "layerType": "data", + "seriesType": "line", + "splitAccessor": "464fd4fa-3499-4309-8d0a-ccab324315d8", + "xAccessor": "9538ba82-3742-4452-98b6-b0ba7309800d" + }, + { + "accessors": [ + "b16dd086-965c-4857-ada6-126a5925a865" + ], + "layerId": "aa9f3aa1-a8c4-496e-942e-b19ada17869b", + "layerType": "referenceLine", + "yConfig": [ + { + "axisMode": "left", + "color": "#e7664c", + "fill": "above", + "forAccessor": "b16dd086-965c-4857-ada6-126a5925a865" + } + ] + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "lowerBound": 50, + "mode": "custom", + "upperBound": 100 + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {} + }, + "gridData": { + "h": 11, + "i": "21f65654-2212-4332-a6d0-32559b89166b", + "w": 33, + "x": 15, + "y": 58 + }, + "panelIndex": "21f65654-2212-4332-a6d0-32559b89166b", + "title": "High utilization over time", + "type": "lens" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "## General Pipeline Information", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 3, + "i": "b3be9f8e-fd15-42f4-b6e8-d2f15ce5963f", + "w": 48, + "x": 0, + "y": 69 + }, + "panelIndex": "b3be9f8e-fd15-42f4-b6e8-d2f15ce5963f", + "title": "", + "type": "visualization" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "metrics-*", + "layers": { + "dce344e8-e3dd-42fc-b474-35a87078a4d2": { + "columnOrder": [ + "7c2a72b2-4a23-4cc8-bf49-977a4598ba22", + "f74ebe13-0432-48be-b11e-0166d1157f2e", + "b533b1f6-4554-4e65-b170-cad58455d590", + "ce459291-9963-4d78-ae70-13c1ba2e9e4b", + "25c305b4-4be5-41e6-8500-01ab4a6cbab7", + "24fd7ec9-9b86-4e0d-936a-da9b0679272b", + "15eb8f95-50aa-4eb4-baef-31da9f64a429", + "b533b1f6-4554-4e65-b170-cad58455d590X0", + "b533b1f6-4554-4e65-b170-cad58455d590X1", + "b533b1f6-4554-4e65-b170-cad58455d590X2" + ], + "columns": { + "15eb8f95-50aa-4eb4-baef-31da9f64a429": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.flow.worker_utilization.last_1_hour\": *" + }, + "isBucketed": false, + "label": "Utilization (1 Hour)", + "operationType": "last_value", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 0, + "suffix": "%" + } + }, + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.flow.worker_utilization.last_1_hour" + }, + "24fd7ec9-9b86-4e0d-936a-da9b0679272b": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.flow.worker_utilization.current\": *" + }, + "isBucketed": false, + "label": "Current utilization", + "operationType": "last_value", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 0, + "suffix": "%" + } + }, + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.flow.worker_utilization.current" + }, + "25c305b4-4be5-41e6-8500-01ab4a6cbab7": { + "customLabel": true, + "dataType": "string", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.state\": *" + }, + "isBucketed": false, + "label": "State", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.state" + }, + "7c2a72b2-4a23-4cc8-bf49-977a4598ba22": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Node and Pipeline", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "24fd7ec9-9b86-4e0d-936a-da9b0679272b", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.pipeline.id" + ], + "size": 100 + }, + "scale": "ordinal", + "sourceField": "logstash.node.name" + }, + "b533b1f6-4554-4e65-b170-cad58455d590": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Last Report", + "operationType": "formula", + "params": { + "format": { + "id": "duration", + "params": { + "decimals": 0, + "fromUnit": "milliseconds", + "suffix": " Ago" + } + }, + "formula": "now()-last_value(@timestamp, kql='\"@timestamp\": *')", + "isFormulaBroken": false + }, + "references": [ + "b533b1f6-4554-4e65-b170-cad58455d590X2" + ], + "scale": "ratio" + }, + "b533b1f6-4554-4e65-b170-cad58455d590X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of now()-last_value(@timestamp, kql='\"@timestamp\": *')", + "operationType": "now", + "references": [], + "scale": "ratio" + }, + "b533b1f6-4554-4e65-b170-cad58455d590X1": { + "customLabel": true, + "dataType": "date", + "filter": { + "language": "kuery", + "query": "\"@timestamp\": *" + }, + "isBucketed": false, + "label": "Part of now()-last_value(@timestamp, kql='\"@timestamp\": *')", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "@timestamp" + }, + "b533b1f6-4554-4e65-b170-cad58455d590X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of now()-last_value(@timestamp, kql='\"@timestamp\": *')", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "b533b1f6-4554-4e65-b170-cad58455d590X0", + "b533b1f6-4554-4e65-b170-cad58455d590X1" + ], + "location": { + "max": 51, + "min": 0 + }, + "name": "subtract", + "text": "now()-last_value(@timestamp, kql='\"@timestamp\": *')", + "type": "function" + } + }, + "references": [ + "b533b1f6-4554-4e65-b170-cad58455d590X0", + "b533b1f6-4554-4e65-b170-cad58455d590X1" + ], + "scale": "ratio" + }, + "ce459291-9963-4d78-ae70-13c1ba2e9e4b": { + "customLabel": true, + "dataType": "string", + "filter": { + "language": "kuery", + "query": "\"logstash.pipeline.status\": *" + }, + "isBucketed": false, + "label": "Status", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.status" + }, + "f74ebe13-0432-48be-b11e-0166d1157f2e": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Version", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "24fd7ec9-9b86-4e0d-936a-da9b0679272b", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 100 + }, + "scale": "ordinal", + "sourceField": "logstash.node.version" + } + }, + "incompleteColumns": {}, + "indexPatternId": "metrics-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "logstash.pipeline.state", + "index": "c0df066d-5d8b-4d09-84a0-5d309931fef5", + "key": "logstash.pipeline.state", + "negate": true, + "params": { + "query": "TERMINATED" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "logstash.pipeline.state": "TERMINATED" + } + } + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "data_stream.dataset", + "index": "0d266976-a94d-4f05-b70f-f09b368a5426", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "logstash.health_report" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "logstash.health_report" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "7c2a72b2-4a23-4cc8-bf49-977a4598ba22", + "width": 255 + }, + { + "alignment": "left", + "colorMode": "cell", + "columnId": "24fd7ec9-9b86-4e0d-936a-da9b0679272b", + "isMetric": true, + "isTransposed": false, + "palette": { + "name": "status", + "params": { + "continuity": "above", + "name": "status", + "rangeMax": null, + "rangeMin": 0, + "rangeType": "number", + "reverse": false, + "steps": 5, + "stops": [ + { + "color": "#209280", + "stop": 0 + }, + { + "color": "#54b399", + "stop": 20 + }, + { + "color": "#d6bf57", + "stop": 40 + }, + { + "color": "#e7664c", + "stop": 60 + }, + { + "color": "#cc5642", + "stop": 80 + } + ] + }, + "type": "palette" + } + }, + { + "colorMode": "cell", + "columnId": "15eb8f95-50aa-4eb4-baef-31da9f64a429", + "isMetric": true, + "isTransposed": false, + "palette": { + "name": "custom", + "params": { + "colorStops": [ + { + "color": "#209280", + "stop": 0 + }, + { + "color": "#54b399", + "stop": 25 + }, + { + "color": "#d6bf57", + "stop": 50 + }, + { + "color": "#e7664c", + "stop": 75 + }, + { + "color": "#cc5642", + "stop": 90 + } + ], + "continuity": "above", + "name": "custom", + "rangeMax": null, + "rangeMin": 0, + "rangeType": "number", + "reverse": false, + "steps": 5, + "stops": [ + { + "color": "#209280", + "stop": 25 + }, + { + "color": "#54b399", + "stop": 50 + }, + { + "color": "#d6bf57", + "stop": 75 + }, + { + "color": "#e7664c", + "stop": 90 + }, + { + "color": "#cc5642", + "stop": 91 + } + ] + }, + "type": "palette" + } + }, + { + "columnId": "f74ebe13-0432-48be-b11e-0166d1157f2e", + "isMetric": false, + "isTransposed": false, + "width": 74.58333333333326 + }, + { + "columnId": "b533b1f6-4554-4e65-b170-cad58455d590", + "isMetric": true, + "isTransposed": false + }, + { + "columnId": "25c305b4-4be5-41e6-8500-01ab4a6cbab7", + "isMetric": true, + "isTransposed": false + }, + { + "colorMapping": { + "assignments": [ + { + "color": { + "colorIndex": 0, + "paletteId": "eui_amsterdam_color_blind", + "type": "categorical" + }, + "rule": { + "type": "matchExactly", + "values": [ + "green" + ] + }, + "touched": false + }, + { + "color": { + "colorIndex": 9, + "paletteId": "eui_amsterdam_color_blind", + "type": "categorical" + }, + "rule": { + "type": "matchExactly", + "values": [ + "red" + ] + }, + "touched": true + }, + { + "color": { + "colorIndex": 5, + "paletteId": "eui_amsterdam_color_blind", + "type": "categorical" + }, + "rule": { + "type": "matchExactly", + "values": [ + "yellow" + ] + }, + "touched": true + } + ], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "colorMode": "cell", + "columnId": "ce459291-9963-4d78-ae70-13c1ba2e9e4b", + "isMetric": true, + "isTransposed": false, + "palette": { + "name": "default", + "params": { + "stops": [ + { + "color": "#54B399", + "stop": 20 + }, + { + "color": "#6092C0", + "stop": 30 + }, + { + "color": "#D36086", + "stop": 40 + }, + { + "color": "#9170B8", + "stop": 50 + }, + { + "color": "#CA8EAE", + "stop": 60 + }, + { + "color": "#D6BF57", + "stop": 70 + }, + { + "color": "#B9A888", + "stop": 80 + }, + { + "color": "#DA8B45", + "stop": 90 + }, + { + "color": "#AA6556", + "stop": 100 + }, + { + "color": "#E7664C", + "stop": 110 + } + ] + }, + "type": "palette" + } + } + ], + "headerRowHeightLines": 2, + "layerId": "dce344e8-e3dd-42fc-b474-35a87078a4d2", + "layerType": "data", + "sorting": { + "columnId": "15eb8f95-50aa-4eb4-baef-31da9f64a429", + "direction": "desc" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 19, + "i": "b30ba597-df32-43a3-9f10-9d76f0666b80", + "w": 48, + "x": 0, + "y": 72 + }, + "panelIndex": "b30ba597-df32-43a3-9f10-9d76f0666b80", + "title": "Worker utilization by pipeline", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "metrics-*", + "layers": { + "dce344e8-e3dd-42fc-b474-35a87078a4d2": { + "columnOrder": [ + "7c2a72b2-4a23-4cc8-bf49-977a4598ba22", + "20113481-00ae-4076-b570-1efd35615557", + "24fd7ec9-9b86-4e0d-936a-da9b0679272b" + ], + "columns": { + "20113481-00ae-4076-b570-1efd35615557": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "24fd7ec9-9b86-4e0d-936a-da9b0679272b": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Worker Utilization (1 min)", + "operationType": "median", + "params": { + "emptyAsNull": true, + "format": { + "id": "number", + "params": { + "decimals": 0, + "suffix": "%" + } + } + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.flow.worker_utilization.last_1_minute" + }, + "7c2a72b2-4a23-4cc8-bf49-977a4598ba22": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of logstash.pipeline.id + 1 other", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "24fd7ec9-9b86-4e0d-936a-da9b0679272b", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "logstash.node.name" + ], + "size": 100 + }, + "scale": "ordinal", + "sourceField": "logstash.pipeline.id" + } + }, + "incompleteColumns": {}, + "indexPatternId": "metrics-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "data_stream.dataset", + "index": "0d266976-a94d-4f05-b70f-f09b368a5426", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "logstash.health_report" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "logstash.health_report" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "24fd7ec9-9b86-4e0d-936a-da9b0679272b" + ], + "layerId": "dce344e8-e3dd-42fc-b474-35a87078a4d2", + "layerType": "data", + "seriesType": "line", + "splitAccessor": "7c2a72b2-4a23-4cc8-bf49-977a4598ba22", + "xAccessor": "20113481-00ae-4076-b570-1efd35615557" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "lowerBound": 0, + "mode": "custom", + "upperBound": 100 + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "0e0170db-cfe9-483c-b076-d1850b1663ed", + "w": 48, + "x": 0, + "y": 91 + }, + "panelIndex": "0e0170db-cfe9-483c-b076-d1850b1663ed", + "title": "Worker utilization by pipeline over time", + "type": "lens" + } + ], + "timeRestore": false, + "title": "[Metrics Logstash] Pipeline Health Report", + "version": 3 + }, + "coreMigrationVersion": "8.8.0", + "created_at": "2025-01-23T18:34:55.073Z", + "created_by": "u_2424114829_cloud", + "id": "logstash-838aac39-8edd-48b0-95b4-289e42b1e98a", + "managed": false, + "references": [ + { + "id": "metrics-*", + "name": "829075a9-1736-4515-ba87-cf99c519a09b:indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "cb1751b1-7889-4c9f-84ec-8e4332e47d89:indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "80a338b2-c98a-4878-a065-1a340c954d69:indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "f10b370b-8cb4-42b1-9a61-63c0366bc9bb:indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "6ff38919-fafa-4fe1-b28e-2fd42e6d3783:indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "cffb0a85-9bbc-4cd7-9cfc-d55246a8e1a6:indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "8f81966e-36f2-4c0e-8844-afc094f8c346:indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "015c903c-0087-4d87-bd21-5b6ef9ab70e2:indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "21f65654-2212-4332-a6d0-32559b89166b:indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "21f65654-2212-4332-a6d0-32559b89166b:indexpattern-datasource-layer-aa9f3aa1-a8c4-496e-942e-b19ada17869b", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "b30ba597-df32-43a3-9f10-9d76f0666b80:indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "0e0170db-cfe9-483c-b076-d1850b1663ed:indexpattern-datasource-layer-dce344e8-e3dd-42fc-b474-35a87078a4d2", + "type": "index-pattern" + }, + { + "id": "logstash-9a72208d-e446-48b9-8a63-c4256b9aa4e3", + "name": "794f0447-82c3-4747-9176-d78b7dd1eb12:link_097a2e6e-6ccf-4aeb-ad60-48bb1dfc37e9_dashboard", + "type": "dashboard" + }, + { + "id": "logstash-838aac39-8edd-48b0-95b4-289e42b1e98a", + "name": "794f0447-82c3-4747-9176-d78b7dd1eb12:link_e8e52960-2726-45bc-8a1d-bd8d4e73cffb_dashboard", + "type": "dashboard" + }, + { + "id": "metrics-*", + "name": "controlGroup_b37a605f-f738-49e1-a654-1b2b597c059c:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "controlGroup_55051a1d-57ba-4ebf-963f-86cfe2dba3d2:optionsListDataView", + "type": "index-pattern" + } + ], + "type": "dashboard", + "typeMigrationVersion": "10.2.0", + "updated_by": "u_2424114829_cloud" +} \ No newline at end of file diff --git a/packages/logstash/kibana/dashboard/logstash-9a72208d-e446-48b9-8a63-c4256b9aa4e3.json b/packages/logstash/kibana/dashboard/logstash-9a72208d-e446-48b9-8a63-c4256b9aa4e3.json new file mode 100644 index 00000000000..b4798b78e46 --- /dev/null +++ b/packages/logstash/kibana/dashboard/logstash-9a72208d-e446-48b9-8a63-c4256b9aa4e3.json @@ -0,0 +1,1022 @@ +{ + "attributes": { + "controlGroupInput": { + "chainingSystem": "HIERARCHICAL", + "controlStyle": "oneLine", + "ignoreParentSettingsJSON": { + "ignoreFilters": false, + "ignoreQuery": false, + "ignoreTimerange": false, + "ignoreValidations": false + }, + "panelsJSON": { + "835e5891-04b0-4bd7-a3cc-b41a01d2fa23": { + "explicitInput": { + "dataViewId": "metrics-*", + "fieldName": "logstash.node.name", + "id": "835e5891-04b0-4bd7-a3cc-b41a01d2fa23", + "searchTechnique": "prefix", + "selectedOptions": [], + "sort": { + "by": "_count", + "direction": "desc" + }, + "title": "Logstash node" + }, + "grow": true, + "order": 0, + "type": "optionsListControl", + "width": "medium" + }, + "c5d8db13-609b-4e19-b3da-a8934358226d": { + "explicitInput": { + "dataViewId": "metrics-*", + "fieldName": "logstash.node.version", + "id": "c5d8db13-609b-4e19-b3da-a8934358226d", + "searchTechnique": "prefix", + "selectedOptions": [], + "sort": { + "by": "_count", + "direction": "desc" + }, + "title": "Logstash version" + }, + "grow": true, + "order": 1, + "type": "optionsListControl", + "width": "medium" + } + }, + "showApplySelections": false + }, + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "layout": "horizontal", + "links": [ + { + "destinationRefName": "link_097a2e6e-6ccf-4aeb-ad60-48bb1dfc37e9_dashboard", + "id": "097a2e6e-6ccf-4aeb-ad60-48bb1dfc37e9", + "label": "Node Health", + "order": 0, + "type": "dashboardLink" + }, + { + "destinationRefName": "link_e8e52960-2726-45bc-8a1d-bd8d4e73cffb_dashboard", + "id": "e8e52960-2726-45bc-8a1d-bd8d4e73cffb", + "label": "Pipeline Health", + "order": 1, + "type": "dashboardLink" + } + ] + }, + "enhancements": {} + }, + "gridData": { + "h": 2, + "i": "1355f5c4-3311-4d0b-a026-0b74ed0ebdc9", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "1355f5c4-3311-4d0b-a026-0b74ed0ebdc9", + "type": "links" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-11c73da6-29cd-4c76-8f67-d5dbd8fa9e7e", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "metrics-*", + "layers": { + "11c73da6-29cd-4c76-8f67-d5dbd8fa9e7e": { + "columnOrder": [ + "5d5fd7fa-ffcf-43de-8e65-d70e88700521" + ], + "columns": { + "5d5fd7fa-ffcf-43de-8e65-d70e88700521": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total Nodes", + "operationType": "unique_count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.node.uuid" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "indexPatternId": "metrics-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layerId": "11c73da6-29cd-4c76-8f67-d5dbd8fa9e7e", + "layerType": "data", + "metricAccessor": "5d5fd7fa-ffcf-43de-8e65-d70e88700521" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "2f07455e-9094-49fa-be1d-e3d759e0f5ad", + "w": 7, + "x": 0, + "y": 2 + }, + "panelIndex": "2f07455e-9094-49fa-be1d-e3d759e0f5ad", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-60345afe-2ea9-4c34-a1dd-ae7e651fab76", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "metrics-*", + "layers": { + "60345afe-2ea9-4c34-a1dd-ae7e651fab76": { + "columnOrder": [ + "cdee386d-20cd-494d-9ba2-1e6044cfa380", + "305591cf-62a3-4903-b314-6e64436c3dfb", + "1f79714d-7bb0-4fee-9fa9-dcbd8086fcf2" + ], + "columns": { + "1f79714d-7bb0-4fee-9fa9-dcbd8086fcf2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Count of nodes by status", + "operationType": "unique_count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.node.uuid" + }, + "305591cf-62a3-4903-b314-6e64436c3dfb": { + "customLabel": true, + "dataType": "date", + "isBucketed": true, + "label": "Nodes reporting", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "cdee386d-20cd-494d-9ba2-1e6044cfa380": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Node Status", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "1f79714d-7bb0-4fee-9fa9-dcbd8086fcf2", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "logstash.node.status" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "indexPatternId": "metrics-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "accessors": [ + "1f79714d-7bb0-4fee-9fa9-dcbd8086fcf2" + ], + "colorMapping": { + "assignments": [ + { + "color": { + "colorIndex": 9, + "paletteId": "eui_amsterdam_color_blind", + "type": "categorical" + }, + "rule": { + "type": "matchExactly", + "values": [ + "red" + ] + }, + "touched": true + }, + { + "color": { + "colorIndex": 0, + "paletteId": "eui_amsterdam_color_blind", + "type": "categorical" + }, + "rule": { + "type": "matchExactly", + "values": [ + "green" + ] + }, + "touched": false + }, + { + "color": { + "colorIndex": 5, + "paletteId": "eui_amsterdam_color_blind", + "type": "categorical" + }, + "rule": { + "type": "matchExactly", + "values": [ + "yellow" + ] + }, + "touched": true + }, + { + "color": { + "colorIndex": 2, + "paletteId": "neutral", + "type": "categorical" + }, + "rule": { + "type": "matchExactly", + "values": [ + "unknown" + ] + }, + "touched": true + } + ], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "60345afe-2ea9-4c34-a1dd-ae7e651fab76", + "layerType": "data", + "position": "top", + "seriesType": "bar_stacked", + "showGridlines": false, + "splitAccessor": "cdee386d-20cd-494d-9ba2-1e6044cfa380", + "xAccessor": "305591cf-62a3-4903-b314-6e64436c3dfb" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_stacked", + "title": "Empty XY chart", + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {} + }, + "gridData": { + "h": 10, + "i": "972e1561-9e0f-463e-a5f1-337b13aac240", + "w": 41, + "x": 7, + "y": 2 + }, + "panelIndex": "972e1561-9e0f-463e-a5f1-337b13aac240", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-11c73da6-29cd-4c76-8f67-d5dbd8fa9e7e", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "currentIndexPatternId": "metrics-*", + "layers": { + "11c73da6-29cd-4c76-8f67-d5dbd8fa9e7e": { + "columnOrder": [ + "cb87b705-73d1-4bd1-b5a1-0d2cf07bf42d", + "5d5fd7fa-ffcf-43de-8e65-d70e88700521" + ], + "columns": { + "5d5fd7fa-ffcf-43de-8e65-d70e88700521": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total Pipelines", + "operationType": "unique_count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "logstash.pipeline.id" + }, + "cb87b705-73d1-4bd1-b5a1-0d2cf07bf42d": { + "dataType": "string", + "isBucketed": true, + "label": "Top 5 values of logstash.node.uuid", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "5d5fd7fa-ffcf-43de-8e65-d70e88700521", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "logstash.node.uuid" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "indexPatternId": "metrics-*", + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "breakdownByAccessor": "cb87b705-73d1-4bd1-b5a1-0d2cf07bf42d", + "collapseFn": "sum", + "layerId": "11c73da6-29cd-4c76-8f67-d5dbd8fa9e7e", + "layerType": "data", + "metricAccessor": "5d5fd7fa-ffcf-43de-8e65-d70e88700521" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "eddd1759-6099-43a1-bc90-fee1cf3c5497", + "w": 7, + "x": 0, + "y": 7 + }, + "panelIndex": "eddd1759-6099-43a1-bc90-fee1cf3c5497", + "type": "lens" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "id": "", + "params": { + "fontSize": 12, + "markdown": "## Problem Nodes", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 3, + "i": "8b9cc0d5-983e-4f66-9f6c-d91cc14e8eab", + "w": 48, + "x": 0, + "y": 12 + }, + "panelIndex": "8b9cc0d5-983e-4f66-9f6c-d91cc14e8eab", + "type": "visualization" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-40151082-fb35-4d4a-aa1f-0647f3de0166", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "c1fda930-53ed-4ee7-961c-4a84131b3cbb", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "40151082-fb35-4d4a-aa1f-0647f3de0166": { + "columnOrder": [ + "7ebbad52-cb5b-4b73-b5bd-92324cade2a1", + "414e1840-28ca-46c3-9dd6-2bfa328656f9", + "6af954fa-beb6-4817-a6d2-859b8800b6df", + "c66a6e06-4fab-4c8c-942e-7a50679d9c5d", + "2722f3c2-756c-4cfe-b871-e09b4e00b399", + "6af954fa-beb6-4817-a6d2-859b8800b6dfX0", + "6af954fa-beb6-4817-a6d2-859b8800b6dfX1", + "6af954fa-beb6-4817-a6d2-859b8800b6dfX2" + ], + "columns": { + "2722f3c2-756c-4cfe-b871-e09b4e00b399": { + "customLabel": true, + "dataType": "string", + "filter": { + "language": "kuery", + "query": "\"logstash.node.symptom\": *" + }, + "isBucketed": false, + "label": "Problem", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ordinal", + "sourceField": "logstash.node.symptom" + }, + "414e1840-28ca-46c3-9dd6-2bfa328656f9": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Version", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 50 + }, + "scale": "ordinal", + "sourceField": "logstash.node.version" + }, + "6af954fa-beb6-4817-a6d2-859b8800b6df": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Last report", + "operationType": "formula", + "params": { + "format": { + "id": "duration", + "params": { + "decimals": 0, + "fromUnit": "milliseconds", + "suffix": " ago" + } + }, + "formula": "now()-last_value(@timestamp, kql='\"@timestamp\": *')", + "isFormulaBroken": false + }, + "references": [ + "6af954fa-beb6-4817-a6d2-859b8800b6dfX2" + ], + "scale": "ratio" + }, + "6af954fa-beb6-4817-a6d2-859b8800b6dfX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of now()-last_value(@timestamp, kql='\"@timestamp\": *')", + "operationType": "now", + "references": [], + "scale": "ratio" + }, + "6af954fa-beb6-4817-a6d2-859b8800b6dfX1": { + "customLabel": true, + "dataType": "date", + "filter": { + "language": "kuery", + "query": "\"@timestamp\": *" + }, + "isBucketed": false, + "label": "Part of now()-last_value(@timestamp, kql='\"@timestamp\": *')", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ratio", + "sourceField": "@timestamp" + }, + "6af954fa-beb6-4817-a6d2-859b8800b6dfX2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of now()-last_value(@timestamp, kql='\"@timestamp\": *')", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "6af954fa-beb6-4817-a6d2-859b8800b6dfX0", + "6af954fa-beb6-4817-a6d2-859b8800b6dfX1" + ], + "location": { + "max": 51, + "min": 0 + }, + "name": "subtract", + "text": "now()-last_value(@timestamp, kql='\"@timestamp\": *')", + "type": "function" + } + }, + "references": [ + "6af954fa-beb6-4817-a6d2-859b8800b6dfX0", + "6af954fa-beb6-4817-a6d2-859b8800b6dfX1" + ], + "scale": "ratio" + }, + "7ebbad52-cb5b-4b73-b5bd-92324cade2a1": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Node", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 100 + }, + "scale": "ordinal", + "sourceField": "logstash.node.name" + }, + "c66a6e06-4fab-4c8c-942e-7a50679d9c5d": { + "customLabel": true, + "dataType": "string", + "filter": { + "language": "kuery", + "query": "\"logstash.node.status\": *" + }, + "isBucketed": false, + "label": "Status", + "operationType": "last_value", + "params": { + "sortField": "@timestamp" + }, + "scale": "ordinal", + "sourceField": "logstash.node.status" + } + }, + "ignoreGlobalFilters": false, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "c1fda930-53ed-4ee7-961c-4a84131b3cbb", + "negate": false, + "params": [ + { + "meta": { + "alias": null, + "disabled": false, + "field": "logstash.node.status", + "index": "metrics-*", + "key": "logstash.node.status", + "negate": false, + "type": "exists", + "value": "exists" + }, + "query": { + "exists": { + "field": "logstash.node.status" + } + } + }, + { + "meta": { + "alias": null, + "disabled": false, + "field": "logstash.node.status", + "index": "metrics-*", + "key": "logstash.node.status", + "negate": true, + "params": { + "query": "green" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "logstash.node.status": "green" + } + } + } + ], + "relation": "AND", + "type": "combined" + }, + "query": {} + }, + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "field": "data_stream.dataset", + "index": "e3d7fcc4-2dc6-4b30-8aa2-b4fb1f8c13a2", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "logstash.health_report" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "logstash.health_report" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "7ebbad52-cb5b-4b73-b5bd-92324cade2a1", + "isMetric": false, + "isTransposed": false + }, + { + "colorMapping": { + "assignments": [ + { + "color": { + "colorIndex": 9, + "paletteId": "eui_amsterdam_color_blind", + "type": "categorical" + }, + "rule": { + "type": "matchExactly", + "values": [ + "red" + ] + }, + "touched": true + }, + { + "color": { + "colorIndex": 5, + "paletteId": "eui_amsterdam_color_blind", + "type": "categorical" + }, + "rule": { + "type": "matchExactly", + "values": [ + "yellow" + ] + }, + "touched": true + }, + { + "color": { + "colorIndex": 2, + "paletteId": "neutral", + "type": "categorical" + }, + "rule": { + "type": "matchExactly", + "values": [ + "unknown" + ] + }, + "touched": true + } + ], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "colorMode": "cell", + "columnId": "c66a6e06-4fab-4c8c-942e-7a50679d9c5d", + "isMetric": true, + "isTransposed": false, + "palette": { + "name": "status", + "type": "palette" + }, + "width": 139.75 + }, + { + "columnId": "414e1840-28ca-46c3-9dd6-2bfa328656f9", + "isMetric": false, + "isTransposed": false, + "width": 101.08333333333331 + }, + { + "columnId": "2722f3c2-756c-4cfe-b871-e09b4e00b399", + "isMetric": true, + "isTransposed": false + }, + { + "columnId": "6af954fa-beb6-4817-a6d2-859b8800b6df", + "isMetric": true, + "isTransposed": false, + "width": 173 + } + ], + "layerId": "40151082-fb35-4d4a-aa1f-0647f3de0166", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [ + { + "action": { + "config": { + "openInNewTab": false, + "useCurrentDateRange": true, + "useCurrentFilters": true + }, + "factoryId": "DASHBOARD_TO_DASHBOARD_DRILLDOWN", + "name": "View Node Health" + }, + "eventId": "0582350f-2442-4e40-be4a-f48687e858f7", + "triggers": [ + "FILTER_TRIGGER" + ] + } + ] + } + } + }, + "gridData": { + "h": 8, + "i": "187c603e-4013-460b-95a5-bf4d39d8fcec", + "w": 48, + "x": 0, + "y": 15 + }, + "panelIndex": "187c603e-4013-460b-95a5-bf4d39d8fcec", + "type": "lens" + } + ], + "timeRestore": false, + "title": "[Metrics Logstash] Node Health Report", + "version": 3 + }, + "coreMigrationVersion": "8.8.0", + "created_at": "2025-01-24T21:51:29.381Z", + "created_by": "u_2424114829_cloud", + "id": "logstash-9a72208d-e446-48b9-8a63-c4256b9aa4e3", + "managed": false, + "references": [ + { + "id": "metrics-*", + "name": "2f07455e-9094-49fa-be1d-e3d759e0f5ad:indexpattern-datasource-layer-11c73da6-29cd-4c76-8f67-d5dbd8fa9e7e", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "972e1561-9e0f-463e-a5f1-337b13aac240:indexpattern-datasource-layer-60345afe-2ea9-4c34-a1dd-ae7e651fab76", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "eddd1759-6099-43a1-bc90-fee1cf3c5497:indexpattern-datasource-layer-11c73da6-29cd-4c76-8f67-d5dbd8fa9e7e", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "187c603e-4013-460b-95a5-bf4d39d8fcec:indexpattern-datasource-layer-40151082-fb35-4d4a-aa1f-0647f3de0166", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "187c603e-4013-460b-95a5-bf4d39d8fcec:c1fda930-53ed-4ee7-961c-4a84131b3cbb", + "type": "index-pattern" + }, + { + "id": "logstash-838aac39-8edd-48b0-95b4-289e42b1e98a", + "name": "187c603e-4013-460b-95a5-bf4d39d8fcec:drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:0582350f-2442-4e40-be4a-f48687e858f7:dashboardId", + "type": "dashboard" + }, + { + "id": "logstash-9a72208d-e446-48b9-8a63-c4256b9aa4e3", + "name": "1355f5c4-3311-4d0b-a026-0b74ed0ebdc9:link_097a2e6e-6ccf-4aeb-ad60-48bb1dfc37e9_dashboard", + "type": "dashboard" + }, + { + "id": "logstash-838aac39-8edd-48b0-95b4-289e42b1e98a", + "name": "1355f5c4-3311-4d0b-a026-0b74ed0ebdc9:link_e8e52960-2726-45bc-8a1d-bd8d4e73cffb_dashboard", + "type": "dashboard" + }, + { + "id": "metrics-*", + "name": "controlGroup_550fa741-ece7-4199-a4e0-e9bd6d744ca8:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "controlGroup_2eadbd3f-11a3-4469-9942-decfc7fff885:optionsListDataView", + "type": "index-pattern" + } + ], + "type": "dashboard", + "typeMigrationVersion": "10.2.0", + "updated_by": "u_2424114829_cloud" +} \ No newline at end of file diff --git a/packages/logstash/manifest.yml b/packages/logstash/manifest.yml index b596b8f1980..5dd268bdd6e 100644 --- a/packages/logstash/manifest.yml +++ b/packages/logstash/manifest.yml @@ -1,6 +1,6 @@ name: logstash title: Logstash -version: 2.4.12 +version: 2.5.0 description: Collect logs and metrics from Logstash with Elastic Agent. type: integration icons: @@ -14,11 +14,11 @@ categories: - elastic_stack conditions: kibana: - version: ^8.10.1 + version: ^8.17.0 elastic: subscription: basic owner: - github: elastic/logstash + github: elastic/stack-monitoring type: elastic screenshots: - src: /img/kibana-logstash-log.png @@ -89,7 +89,7 @@ policy_templates: show_user: false - type: logstash/metrics title: "Metrics (Stack Monitoring)" - description: "Collect node metrics and stats from Logstash instances to power the Stack Monitoring application in Kibana.\n Disable if using Elastic Agent Based Monitoring." + description: "Collect node metrics and stats from Logstash instances to power the Stack Monitoring application in Kibana.\n Disable if using Metrics (Elastic Agent)." vars: - name: hosts type: text