From fbc12316f6df35c7f3f40139498d9b6d5a0d0791 Mon Sep 17 00:00:00 2001 From: xiaochaoren1 Date: Fri, 11 Oct 2024 20:01:28 +0800 Subject: [PATCH] fix: modify exist attribute tag translation --- server/querier/engine/clickhouse/clickhouse_test.go | 6 +++--- server/querier/engine/clickhouse/filter.go | 2 +- server/querier/engine/clickhouse/group.go | 12 ++++++++---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/server/querier/engine/clickhouse/clickhouse_test.go b/server/querier/engine/clickhouse/clickhouse_test.go index 779a701bd72..f38d573c79f 100644 --- a/server/querier/engine/clickhouse/clickhouse_test.go +++ b/server/querier/engine/clickhouse/clickhouse_test.go @@ -175,10 +175,10 @@ var ( output: []string{"SELECT if(dictGet('flow_tag.pod_service_k8s_label_map', 'value', (toUInt64(service_id_0),'statefulset.kubernetes.io/pod-name'))!='', dictGet('flow_tag.pod_service_k8s_label_map', 'value', (toUInt64(service_id_0),'statefulset.kubernetes.io/pod-name')), dictGet('flow_tag.pod_k8s_label_map', 'value', (toUInt64(pod_id_0),'statefulset.kubernetes.io/pod-name')) ) AS `k8s.label.abc` FROM flow_log.`l4_flow_log` PREWHERE ((toUInt64(service_id_0) GLOBAL IN (SELECT id FROM flow_tag.pod_service_k8s_label_map WHERE value = 'opensource-loki-0' and key='statefulset.kubernetes.io/pod-name')) OR (toUInt64(pod_id_0) GLOBAL IN (SELECT id FROM flow_tag.pod_k8s_label_map WHERE value = 'opensource-loki-0' and key='statefulset.kubernetes.io/pod-name'))) AND (((toUInt64(service_id_0) GLOBAL IN (SELECT id FROM flow_tag.pod_service_k8s_label_map WHERE key='statefulset.kubernetes.io/pod-name')) OR (toUInt64(pod_id_0) GLOBAL IN (SELECT id FROM flow_tag.pod_k8s_label_map WHERE key='statefulset.kubernetes.io/pod-name')))) GROUP BY `k8s.label.abc` LIMIT 10000"}, }, { input: "select `attribute.cc` as `attribute.abc` from l7_flow_log where `attribute.abc`='opensource-loki-0' group by `attribute.abc`", - output: []string{"SELECT attribute_values[indexOf(attribute_names,'cc')] AS `attribute.abc` FROM flow_log.`l7_flow_log` PREWHERE attribute_values[indexOf(attribute_names,'cc')] = 'opensource-loki-0' AND (attribute_values[indexOf(attribute_names,'attribute.cc')] != '') GROUP BY `attribute.abc` LIMIT 10000"}, + output: []string{"SELECT attribute_values[indexOf(attribute_names,'cc')] AS `attribute.abc` FROM flow_log.`l7_flow_log` PREWHERE attribute_values[indexOf(attribute_names,'cc')] = 'opensource-loki-0' AND (attribute_values[indexOf(attribute_names,'cc')] != '') GROUP BY `attribute.abc` LIMIT 10000"}, }, { input: "select `tag.cc` as `tag.abc` from cpu where `tag.abc`='opensource-loki-0' group by `tag.abc`", - output: []string{"SELECT tag_values[indexOf(tag_names,'cc')] AS `tag.abc` FROM ext_metrics.`metrics` PREWHERE (virtual_table_name='cpu') AND tag_values[indexOf(tag_names,'cc')] = 'opensource-loki-0' AND (tag_values[indexOf(tag_names,'tag.cc')] != '') GROUP BY `tag.abc` LIMIT 10000"}, + output: []string{"SELECT tag_values[indexOf(tag_names,'cc')] AS `tag.abc` FROM ext_metrics.`metrics` PREWHERE (virtual_table_name='cpu') AND tag_values[indexOf(tag_names,'cc')] = 'opensource-loki-0' AND (tag_values[indexOf(tag_names,'cc')] != '') GROUP BY `tag.abc` LIMIT 10000"}, db: "ext_metrics", }, { input: "select `metrics.storageclass_annotations` AS `job_info` from prometheus_kube", @@ -281,7 +281,7 @@ var ( output: []string{"SELECT if(l3_device_type_0=1,l3_device_id_0, 0) AS `chost_id_0` FROM flow_log.`l4_flow_log` PREWHERE NOT (l3_device_type_0=1) LIMIT 1"}, }, { input: "SELECT response_code, `attribute.a.b`, `attribute.c.d` AS attr_c_d from l7_flow_log WHERE exist(response_code) AND exist(`attribute.a.b`) AND exist(`attribute.c.d`) LIMIT 1", - output: []string{"SELECT response_code, attribute_values[indexOf(attribute_names,'a.b')] AS `attribute.a.b`, attribute_values[indexOf(attribute_names,'c.d')] AS `attr_c_d` FROM flow_log.`l7_flow_log` PREWHERE ((isNotNull(response_code))) AND ((attribute_values[indexOf(attribute_names,'attribute.a.b')] != '')) AND ((attribute_values[indexOf(attribute_names,'attribute.c.d')] != '')) LIMIT 1"}, + output: []string{"SELECT response_code, attribute_values[indexOf(attribute_names,'a.b')] AS `attribute.a.b`, attribute_values[indexOf(attribute_names,'c.d')] AS `attr_c_d` FROM flow_log.`l7_flow_log` PREWHERE ((isNotNull(response_code))) AND ((attribute_values[indexOf(attribute_names,'a.b')] != '')) AND ((attribute_values[indexOf(attribute_names,'c.d')] != '')) LIMIT 1"}, }, { input: "SELECT `cloud.tag.xx_0` from l4_flow_log WHERE NOT exist(`cloud.tag.xx_0`) LIMIT 1", output: []string{"SELECT if(if(l3_device_type_0=1, dictGet('flow_tag.chost_cloud_tag_map', 'value', (toUInt64(l3_device_id_0),'xx')), '')!='',if(l3_device_type_0=1, dictGet('flow_tag.chost_cloud_tag_map', 'value', (toUInt64(l3_device_id_0),'xx')), ''), dictGet('flow_tag.pod_ns_cloud_tag_map', 'value', (toUInt64(pod_ns_id_0),'xx')) ) AS `cloud.tag.xx_0` FROM flow_log.`l4_flow_log` PREWHERE NOT (((toUInt64(l3_device_id_0) GLOBAL IN (SELECT id FROM flow_tag.chost_cloud_tag_map WHERE key='xx') AND l3_device_type_0=1) OR (toUInt64(pod_ns_id_0) GLOBAL IN (SELECT id FROM flow_tag.pod_ns_cloud_tag_map WHERE key='xx')))) LIMIT 1"}, diff --git a/server/querier/engine/clickhouse/filter.go b/server/querier/engine/clickhouse/filter.go index e5b6388ddcb..0e654711f34 100644 --- a/server/querier/engine/clickhouse/filter.go +++ b/server/querier/engine/clickhouse/filter.go @@ -160,7 +160,7 @@ func TransWhereTagFunction(db, table string, name string, args []string) (filter } else { // non-resource tags engine := &CHEngine{DB: db, Table: table} - notNullExpr, ok := GetNotNullFilter(resource, engine) + notNullExpr, ok := GetNotNullFilter(args[0], engine) if !ok { return } diff --git a/server/querier/engine/clickhouse/group.go b/server/querier/engine/clickhouse/group.go index c8c2ce9a308..7ebfb7b07e8 100644 --- a/server/querier/engine/clickhouse/group.go +++ b/server/querier/engine/clickhouse/group.go @@ -213,11 +213,13 @@ func GetNotNullFilter(name string, e *CHEngine) (view.Node, bool) { return &view.Expr{}, false } tagItem, ok = tag.GetTag("tag.", db, table, "default") - filter := fmt.Sprintf(tagItem.NotNullFilter, preAsTag) + filterName := strings.TrimPrefix(strings.Trim(preAsTag, "`"), "tag.") + filter := fmt.Sprintf(tagItem.NotNullFilter, filterName) return &view.Expr{Value: "(" + filter + ")"}, true } else if strings.HasPrefix(preAsTag, "attribute.") { tagItem, ok = tag.GetTag("attribute.", db, table, "default") - filter := fmt.Sprintf(tagItem.NotNullFilter, preAsTag) + filterName := strings.TrimPrefix(strings.Trim(preAsTag, "`"), "attribute.") + filter := fmt.Sprintf(tagItem.NotNullFilter, filterName) return &view.Expr{Value: "(" + filter + ")"}, true } else if common.IsValueInSliceString(preAsTag, []string{"request_id", "response_code", "span_kind", "request_length", "response_length", "sql_affected_rows"}) { filter := fmt.Sprintf("%s is not null", preAsTag) @@ -248,11 +250,13 @@ func GetNotNullFilter(name string, e *CHEngine) (view.Node, bool) { return &view.Expr{}, false } tagItem, ok = tag.GetTag("tag.", db, table, "default") - filter := fmt.Sprintf(tagItem.NotNullFilter, name) + filterName := strings.TrimPrefix(strings.Trim(name, "`"), "tag.") + filter := fmt.Sprintf(tagItem.NotNullFilter, filterName) return &view.Expr{Value: "(" + filter + ")"}, true } else if strings.HasPrefix(noBackQuoteName, "attribute.") { tagItem, ok = tag.GetTag("attribute.", db, table, "default") - filter := fmt.Sprintf(tagItem.NotNullFilter, name) + filterName := strings.TrimPrefix(strings.Trim(name, "`"), "attribute.") + filter := fmt.Sprintf(tagItem.NotNullFilter, filterName) return &view.Expr{Value: "(" + filter + ")"}, true } else if common.IsValueInSliceString(noBackQuoteName, []string{"request_id", "response_code", "span_kind", "request_length", "response_length", "sql_affected_rows"}) { filter := fmt.Sprintf("%s is not null", name)