Skip to content

Commit e15affc

Browse files
puckpuckpuckpuck
authored
[opensearch] - Use opensearch exporter & remove dataprepper (#1356)
* use opensearch native exporter Signed-off-by: Pierre Tessier <[email protected]> * cleanup opensearch config Signed-off-by: Pierre Tessier <[email protected]> * fix opensearch datasource Signed-off-by: Pierre Tessier <[email protected]> * fix opensearch datasource Signed-off-by: Pierre Tessier <[email protected]> * use opensearch exporter Signed-off-by: Pierre Tessier <[email protected]> * remove dataprepper config files Signed-off-by: Pierre Tessier <[email protected]> * remove opensearch auth details Signed-off-by: Pierre Tessier <[email protected]> * opensearch memory limits Signed-off-by: Pierre Tessier <[email protected]> * opensearch memory limits Signed-off-by: Pierre Tessier <[email protected]> * remove dataprepper Signed-off-by: Pierre Tessier <[email protected]> --------- Signed-off-by: Pierre Tessier <[email protected]>
1 parent 26b1710 commit e15affc

File tree

9 files changed

+41
-60
lines changed

9 files changed

+41
-60
lines changed

‎.env

+1-2
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,9 @@ IMAGE_NAME=ghcr.io/open-telemetry/demo
66

77
# Dependent images
88
COLLECTOR_CONTRIB_IMAGE=otel/opentelemetry-collector-contrib:0.93.0
9-
DATAPREPPER_IMAGE=opensearchproject/data-prepper:latest
109
GRAFANA_IMAGE=grafana/grafana:10.3.1
1110
JAEGERTRACING_IMAGE=jaegertracing/all-in-one:1.53
12-
OPENSEARCH_IMAGE=opensearchproject/opensearch:latest
11+
OPENSEARCH_IMAGE=opensearchproject/opensearch:2.11.1
1312
POSTGRES_IMAGE=postgres:16.1
1413
PROMETHEUS_IMAGE=quay.io/prometheus/prometheus:v2.49.1
1514
REDIS_IMAGE=redis:7.2-alpine

‎CHANGELOG.md

+2
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ the release.
4242
([#1353](https://github.com/open-telemetry/opentelemetry-demo/pull/1353))
4343
* Update dependent services - Collector, Grafana, Jaeger, Prometheus, etc.
4444
([#1354](https://github.com/open-telemetry/opentelemetry-demo/pull/1354))
45+
* [OpenSearch] Use native OpenSearch exporter from Collector
46+
([#1356](https://github.com/open-telemetry/opentelemetry-demo/pull/1356))
4547

4648
## 1.7.2
4749

‎docker-compose.minimal.yml

+12-13
Original file line numberDiff line numberDiff line change
@@ -492,6 +492,7 @@ services:
492492
resources:
493493
limits:
494494
memory: 100M
495+
restart: unless-stopped
495496
environment:
496497
- "GF_INSTALL_PLUGINS=grafana-opensearch-datasource"
497498
volumes:
@@ -543,21 +544,28 @@ services:
543544
resources:
544545
limits:
545546
memory: 300M
547+
restart: unless-stopped
546548
ports:
547549
- "${PROMETHEUS_SERVICE_PORT}:${PROMETHEUS_SERVICE_PORT}"
548550
logging: *logging
549551

552+
# OpenSearch
550553
opensearch:
551554
image: ${OPENSEARCH_IMAGE}
552555
container_name: opensearch
556+
deploy:
557+
resources:
558+
limits:
559+
memory: 1G
560+
restart: unless-stopped
553561
environment:
554562
- cluster.name=demo-cluster
555563
- node.name=demo-node
556564
- bootstrap.memory_lock=true
557565
- discovery.type=single-node
558-
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
559-
- "DISABLE_INSTALL_DEMO_CONFIG=true"
560-
- "DISABLE_SECURITY_PLUGIN=true"
566+
- OPENSEARCH_JAVA_OPTS=-Xms300m -Xmx300m
567+
- DISABLE_INSTALL_DEMO_CONFIG=true
568+
- DISABLE_SECURITY_PLUGIN=true
561569
ulimits:
562570
memlock:
563571
soft: -1
@@ -566,14 +574,5 @@ services:
566574
soft: 65536
567575
hard: 65536
568576
ports:
569-
- "9200:9200"
570-
logging: *logging
571-
572-
dataprepper:
573-
image: ${DATAPREPPER_IMAGE}
574-
volumes:
575-
- ./src/opensearch/pipelines.yaml:/usr/share/data-prepper/pipelines/pipelines.yaml
576-
- ./src/opensearch/data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml
577-
ports:
578-
- "21892"
577+
- "9200"
579578
logging: *logging

‎docker-compose.yml

+12-13
Original file line numberDiff line numberDiff line change
@@ -645,6 +645,7 @@ services:
645645
resources:
646646
limits:
647647
memory: 100M
648+
restart: unless-stopped
648649
environment:
649650
- "GF_INSTALL_PLUGINS=grafana-opensearch-datasource"
650651
volumes:
@@ -696,21 +697,28 @@ services:
696697
resources:
697698
limits:
698699
memory: 300M
700+
restart: unless-stopped
699701
ports:
700702
- "${PROMETHEUS_SERVICE_PORT}:${PROMETHEUS_SERVICE_PORT}"
701703
logging: *logging
702704

705+
# OpenSearch
703706
opensearch:
704707
image: ${OPENSEARCH_IMAGE}
705708
container_name: opensearch
709+
deploy:
710+
resources:
711+
limits:
712+
memory: 1G
713+
restart: unless-stopped
706714
environment:
707715
- cluster.name=demo-cluster
708716
- node.name=demo-node
709717
- bootstrap.memory_lock=true
710718
- discovery.type=single-node
711-
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
712-
- "DISABLE_INSTALL_DEMO_CONFIG=true"
713-
- "DISABLE_SECURITY_PLUGIN=true"
719+
- OPENSEARCH_JAVA_OPTS=-Xms300m -Xmx300m
720+
- DISABLE_INSTALL_DEMO_CONFIG=true
721+
- DISABLE_SECURITY_PLUGIN=true
714722
ulimits:
715723
memlock:
716724
soft: -1
@@ -719,16 +727,7 @@ services:
719727
soft: 65536
720728
hard: 65536
721729
ports:
722-
- "9200:9200"
723-
logging: *logging
724-
725-
dataprepper:
726-
image: ${DATAPREPPER_IMAGE}
727-
volumes:
728-
- ./src/opensearch/pipelines.yaml:/usr/share/data-prepper/pipelines/pipelines.yaml
729-
- ./src/opensearch/data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml
730-
ports:
731-
- "21892"
730+
- "9200"
732731
logging: *logging
733732

734733
# *****

‎src/grafana/provisioning/dashboards/demo/demo-dashboard.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -456,7 +456,7 @@
456456
"type": "count"
457457
}
458458
],
459-
"query": "search source=otel\n| where serviceName=\"${service}\"\n| stats count() by severityText",
459+
"query": "search source=otel\n| where resource.service.name=\"${service}\"\n| stats count() by severity.text",
460460
"queryType": "PPL",
461461
"refId": "A",
462462
"timeField": "time"
@@ -544,7 +544,7 @@
544544
"type": "count"
545545
}
546546
],
547-
"query": "search source=otel\n| where serviceName=\"${service}\"",
547+
"query": "search source=otel\n| where resource.service.name=\"${service}\"",
548548
"queryType": "PPL",
549549
"refId": "A",
550550
"timeField": "time"

‎src/grafana/provisioning/datasources/opensearch.yaml

+5-6
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,9 @@ datasources:
1111
editable: true
1212
isDefault: false
1313
jsonData:
14-
database: 'otel'
15-
interval: Daily
16-
timeField: 'time'
17-
version: '2.10.0'
18-
flavor: 'opensearch'
14+
database: otel
15+
flavor: opensearch
16+
logLevelField: severity
1917
logMessageField: body
20-
logLevelField: severitynumber
18+
pplEnabled: true
19+
timeField: "@timestamp"

‎src/opensearch/data-prepper-config.yaml

-4
This file was deleted.

‎src/opensearch/pipelines.yaml

-14
This file was deleted.

‎src/otelcollector/otelcol-config.yml

+7-6
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
# Copyright The OpenTelemetry Authors
22
# SPDX-License-Identifier: Apache-2.0
33

4-
54
receivers:
65
otlp:
76
protocols:
@@ -21,14 +20,16 @@ exporters:
2120
endpoint: "jaeger:4317"
2221
tls:
2322
insecure: true
24-
otlp/logs:
25-
endpoint: "dataprepper:21892"
26-
tls:
27-
insecure: true
2823
otlphttp/prometheus:
2924
endpoint: "http://prometheus:9090/api/v1/otlp"
3025
tls:
3126
insecure: true
27+
opensearch:
28+
logs_index: otel
29+
http:
30+
endpoint: "http://opensearch:9200"
31+
tls:
32+
insecure: true
3233

3334
processors:
3435
batch:
@@ -64,4 +65,4 @@ service:
6465
logs:
6566
receivers: [otlp]
6667
processors: [batch]
67-
exporters: [otlp/logs, debug]
68+
exporters: [opensearch, debug]

0 commit comments

Comments
 (0)