| loki.storage |
object |
- Storage config. Providing this will automatically populate all necessary storage configs in the templated config. |
+ In case of using thanos storage, enable use_thanos_objstore and the configuration should be done inside the object_store section. |
{
"azure": {
@@ -6335,6 +6335,27 @@ null
"enableHttp2": true,
"requestTimeout": "0s"
},
+ "object_store": {
+ "azure": {
+ "account_key": null,
+ "account_name": null
+ },
+ "gcs": {
+ "bucket_name": null,
+ "service_account": null
+ },
+ "prefix": null,
+ "s3": {
+ "access_key_id": null,
+ "endpoint": null,
+ "http": {},
+ "insecure": false,
+ "region": null,
+ "secret_access_key": null,
+ "sse": {}
+ },
+ "type": "s3"
+ },
"s3": {
"accessKeyId": null,
"backoff_config": {},
@@ -6369,7 +6390,8 @@ null
"user_id": null,
"username": null
},
- "type": "s3"
+ "type": "s3",
+ "use_thanos_objstore": false
}
|
diff --git a/production/helm/loki/scenarios/simple-thanos-values.yaml b/production/helm/loki/scenarios/simple-thanos-values.yaml
new file mode 100644
index 0000000000000..814b1e43f0b35
--- /dev/null
+++ b/production/helm/loki/scenarios/simple-thanos-values.yaml
@@ -0,0 +1,53 @@
+---
+loki:
+ commonConfig:
+ replication_factor: 1
+ useTestSchema: true
+
+ storage:
+ type: s3
+
+ use_thanos_objstore: true
+
+ object_store:
+ type: s3
+
+ s3:
+ access_key_id: thanos-minio
+ secret_access_key: thanos-minio123
+ region: us-east-1
+ insecure: true
+ endpoint: http://minio.minio.svc.cluster.local:9000
+ http:
+ tls_config:
+ insecure_skip_verify: true
+
+ # GCS configuration (when type is "GCS")
+ gcs:
+ bucket_name: test-gcs # Name of the bucket
+ service_account: service-account-test.json # Optional service account JSON
+
+ # Azure configuration (when type is "AZURE")
+ azure:
+ account_name: azure-test # Storage account name
+ account_key: 1234567890 # Optional storage account key
+
+ bucketNames:
+ chunks: chunks_thanos
+ ruler: ruler_thanos
+ admin: admin_thanos
+
+enterprise:
+ enabled: true
+ adminApi:
+ enabled: true
+
+minio:
+ enabled: true
+
+read:
+ replicas: 1
+write:
+ replicas: 1
+backend:
+ replicas: 1
diff --git a/production/helm/loki/templates/_helpers.tpl b/production/helm/loki/templates/_helpers.tpl
index acc66905ddc23..c6f2e6ff3fd88 100644
--- a/production/helm/loki/templates/_helpers.tpl
+++ b/production/helm/loki/templates/_helpers.tpl
@@ -204,6 +204,10 @@ Docker image name for kubectl container
Generated storage config for loki common config
*/}}
{{- define "loki.commonStorageConfig" -}}
+{{- if .Values.loki.storage.use_thanos_objstore -}}
+object_store:
+ {{- include "loki.thanosStorageConfig" (dict "ctx" . "bucketName" .Values.loki.storage.bucketNames.chunks) | nindent 2 }}
+{{- else }}
{{- if .Values.minio.enabled -}}
s3:
endpoint: {{ include "loki.minio" $ }}
@@ -308,6 +312,7 @@ filesystem:
{{- end -}}
{{- end -}}
{{- end -}}
+{{- end -}}
{{/*
Storage config for ruler
@@ -424,10 +429,21 @@ ruler:
{{- end }}
{{- end }}
+{{/* Ruler Thanos Storage Config */}}
+{{- define "loki.rulerThanosStorageConfig" -}}
+{{- if and .Values.loki.storage.use_thanos_objstore .Values.ruler.enabled}}
+ backend: {{ .Values.loki.storage.object_store.type }}
+ {{- include "loki.thanosStorageConfig" (dict "ctx" . "bucketName" .Values.loki.storage.bucketNames.ruler) | nindent 2 }}
+{{- end }}
+{{- end }}
+
{{/* Enterprise Logs Admin API storage config */}}
{{- define "enterprise-logs.adminAPIStorageConfig" }}
storage:
- {{- if .Values.minio.enabled }}
+ {{- if .Values.loki.storage.use_thanos_objstore }}
+ backend: {{ .Values.loki.storage.object_store.type }}
+ {{- include "loki.thanosStorageConfig" (dict "ctx" . "bucketName" .Values.loki.storage.bucketNames.admin) | nindent 2 }}
+ {{- else if .Values.minio.enabled }}
backend: "s3"
s3:
bucket_name: admin
@@ -593,7 +609,6 @@ Generate list of ingress service paths based on deployment type
{{- end -}}
{{- end -}}
-
{{/*
Ingress service paths for distributed deployment
*/}}
@@ -1130,3 +1145,40 @@ This function needs to be called with a context object containing the following
{{- define "loki.configMapOrSecretContentHash" -}}
{{ get (include (print .ctx.Template.BasePath .name) .ctx | fromYaml) "data" | toYaml | sha256sum }}
{{- end }}
+
+{{/* Thanos object storage configuration helper to build
+the thanos_storage_config model*/}}
+{{- define "loki.thanosStorageConfig" -}}
+{{- $bucketName := .bucketName }}
+{{- with .ctx.Values.loki.storage.object_store }}
+{{- if eq .type "s3" }}
+s3:
+ {{- with .s3 }}
+ bucket_name: {{ $bucketName }}
+ endpoint: {{ .endpoint }}
+ access_key_id: {{ .access_key_id }}
+ secret_access_key: {{ .secret_access_key }}
+ region: {{ .region }}
+ insecure: {{ .insecure }}
+ http:
+ {{ toYaml .http | nindent 4 }}
+ sse:
+ {{ toYaml .sse | nindent 4 }}
+ {{- end }}
+{{- else if eq .type "gcs" }}
+gcs:
+ {{- with .gcs }}
+ bucket_name: {{ $bucketName }}
+ service_account: {{ .service_account }}
+ {{- end }}
+{{- else if eq .type "azure" }}
+azure:
+ {{- with .azure }}
+ container_name: {{ $bucketName }}
+ account_name: {{ .account_name }}
+ account_key: {{ .account_key }}
+ {{- end }}
+{{- end }}
+prefix: {{ .prefix }}
+{{- end }}
+{{- end }}
diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml
index b2972178bad85..81be6a3174894 100644
--- a/production/helm/loki/values.yaml
+++ b/production/helm/loki/values.yaml
@@ -209,6 +209,11 @@ loki:
{{ include "loki.rulerConfig" . }}
{{- end }}
+ {{- if and .Values.loki.storage.use_thanos_objstore .Values.ruler.enabled}}
+ ruler_storage:
+ {{- include "loki.rulerThanosStorageConfig" . | nindent 2 }}
+ {{- end }}
+
{{- if or .Values.tableManager.retention_deletes_enabled .Values.tableManager.retention_period }}
table_manager:
retention_deletes_enabled: {{ .Values.tableManager.retention_deletes_enabled }}
@@ -325,6 +330,7 @@ loki:
replication_factor: 3
compactor_address: '{{ include "loki.compactorAddress" . }}'
# -- Storage config. Providing this will automatically populate all necessary storage configs in the templated config.
+ # -- In case of using thanos storage, enable use_thanos_objstore and the configuration should be done inside the object_store section.
storage:
# Loki requires a bucket for chunks and the ruler. GEL requires a third bucket for the admin API.
# Please provide these values if you are using object storage.
@@ -384,6 +390,36 @@ loki:
chunks_directory: /var/loki/chunks
rules_directory: /var/loki/rules
admin_api_directory: /var/loki/admin
+
+ # Loki now supports using thanos storage clients for connecting to object storage backend.
+ # This will become the default way to configure storage in a future releases.
+ use_thanos_objstore: false
+
+ object_store:
+ # Type of object store. Valid options are: s3, gcs, azure
+ type: s3
+ prefix: null # Optional prefix for storage keys
+
+ # S3 configuration (when type is "s3")
+ s3:
+ endpoint: null # S3 endpoint URL
+ region: null # Optional region
+ access_key_id: null # Optional access key
+ secret_access_key: null # Optional secret key
+ insecure: false # Optional. Enable if using self-signed TLS
+ sse: {} # Optional server-side encryption configuration
+ http: {} # Optional HTTP client configuration
+
+ # GCS configuration (when type is "gcs")
+ gcs:
+ bucket_name: null # Name of the bucket
+ service_account: null # Optional service account JSON
+
+ # Azure configuration (when type is "azure")
+ azure:
+ account_name: null # Storage account name
+ account_key: null # Optional storage account key
+
# -- Configure memcached as an external cache for chunk and results cache. Disabled by default
# must enable and specify a host for each cache you would like to use.
memcached: