diff --git a/server/querier/engine/clickhouse/clickhouse_test.go b/server/querier/engine/clickhouse/clickhouse_test.go index fe01eff80f2..3bb3a713629 100644 --- a/server/querier/engine/clickhouse/clickhouse_test.go +++ b/server/querier/engine/clickhouse/clickhouse_test.go @@ -175,10 +175,10 @@ var ( output: "SELECT if(dictGet(flow_tag.pod_service_k8s_label_map, 'value', (toUInt64(service_id_0),'statefulset.kubernetes.io/pod-name'))!='', dictGet(flow_tag.pod_service_k8s_label_map, 'value', (toUInt64(service_id_0),'statefulset.kubernetes.io/pod-name')), dictGet(flow_tag.pod_k8s_label_map, 'value', (toUInt64(pod_id_0),'statefulset.kubernetes.io/pod-name')) ) AS `k8s.label.abc` FROM flow_log.`l4_flow_log` PREWHERE ((toUInt64(service_id_0) IN (SELECT id FROM flow_tag.pod_service_k8s_label_map WHERE value = 'opensource-loki-0' and key='statefulset.kubernetes.io/pod-name')) OR (toUInt64(pod_id_0) IN (SELECT id FROM flow_tag.pod_k8s_label_map WHERE value = 'opensource-loki-0' and key='statefulset.kubernetes.io/pod-name'))) AND (((toUInt64(service_id_0) IN (SELECT id FROM flow_tag.pod_service_k8s_label_map WHERE key='statefulset.kubernetes.io/pod-name')) OR (toUInt64(pod_id_0) IN (SELECT id FROM flow_tag.pod_k8s_label_map WHERE key='statefulset.kubernetes.io/pod-name')))) GROUP BY `k8s.label.abc` LIMIT 10000", }, { input: "select `attribute.cc` as `attribute.abc` from l7_flow_log where `attribute.abc`='opensource-loki-0' group by `attribute.abc`", - output: "SELECT attribute_values[indexOf(attribute_names,'cc')] AS `attribute.abc` FROM flow_log.`l7_flow_log` PREWHERE attribute_values[indexOf(attribute_names,'cc')] = 'opensource-loki-0' AND (`attribute.abc` != '') GROUP BY `attribute.abc` LIMIT 10000", + output: "SELECT attribute_values[indexOf(attribute_names,'cc')] AS `attribute.abc` FROM flow_log.`l7_flow_log` PREWHERE attribute_values[indexOf(attribute_names,'cc')] = 'opensource-loki-0' AND (attribute_values[indexOf(attribute_names,'attribute.cc')] != '') GROUP BY `attribute.abc` LIMIT 10000", }, { input: "select `tag.cc` as `tag.abc` from cpu where `tag.abc`='opensource-loki-0' group by `tag.abc`", - output: "SELECT tag_values[indexOf(tag_names,'cc')] AS `tag.abc` FROM ext_metrics.`metrics` PREWHERE (virtual_table_name='cpu') AND tag_values[indexOf(tag_names,'cc')] = 'opensource-loki-0' AND (`tag.abc` != '') GROUP BY `tag.abc` LIMIT 10000", + output: "SELECT tag_values[indexOf(tag_names,'cc')] AS `tag.abc` FROM ext_metrics.`metrics` PREWHERE (virtual_table_name='cpu') AND tag_values[indexOf(tag_names,'cc')] = 'opensource-loki-0' AND (tag_values[indexOf(tag_names,'tag.cc')] != '') GROUP BY `tag.abc` LIMIT 10000", db: "ext_metrics", }, { input: "select `metrics.storageclass_annotations` AS `job_info` from prometheus_kube", @@ -280,8 +280,8 @@ var ( input: "SELECT chost_id_0 from l4_flow_log WHERE NOT exist(chost_0) LIMIT 1", output: "SELECT if(l3_device_type_0=1,l3_device_id_0, 0) AS `chost_id_0` FROM flow_log.`l4_flow_log` PREWHERE NOT (l3_device_type_0=1) LIMIT 1", }, { - input: "SELECT response_code from l4_flow_log WHERE exist(response_code) LIMIT 1", - output: "SELECT response_code FROM flow_log.`l4_flow_log` PREWHERE ((isNotNull(response_code))) LIMIT 1", + input: "SELECT response_code, `attribute.a.b`, `attribute.c.d` AS attr_c_d from l7_flow_log WHERE exist(response_code) AND exist(`attribute.a.b`) AND exist(`attribute.c.d`) LIMIT 1", + output: "SELECT response_code, attribute_values[indexOf(attribute_names,'a.b')] AS `attribute.a.b`, attribute_values[indexOf(attribute_names,'c.d')] AS `attr_c_d` FROM flow_log.`l7_flow_log` PREWHERE ((isNotNull(response_code))) AND ((attribute_values[indexOf(attribute_names,'attribute.a.b')] != '')) AND ((attribute_values[indexOf(attribute_names,'attribute.c.d')] != '')) LIMIT 1", }, { input: "SELECT `cloud.tag.xx_0` from l4_flow_log WHERE NOT exist(`cloud.tag.xx_0`) LIMIT 1", output: "SELECT if(if(l3_device_type_0=1, dictGet(flow_tag.chost_cloud_tag_map, 'value', (toUInt64(l3_device_id_0),'xx')), '')!='',if(l3_device_type_0=1, dictGet(flow_tag.chost_cloud_tag_map, 'value', (toUInt64(l3_device_id_0),'xx')), ''), dictGet(flow_tag.pod_ns_cloud_tag_map, 'value', (toUInt64(pod_ns_id_0),'xx')) ) AS `cloud.tag.xx_0` FROM flow_log.`l4_flow_log` PREWHERE NOT (((toUInt64(l3_device_id_0) IN (SELECT id FROM flow_tag.chost_cloud_tag_map WHERE key='xx') AND l3_device_type_0=1) OR (toUInt64(pod_ns_id_0) IN (SELECT id FROM flow_tag.pod_ns_cloud_tag_map WHERE key='xx')))) LIMIT 1", diff --git a/server/querier/engine/clickhouse/group.go b/server/querier/engine/clickhouse/group.go index d2bd091fb6b..4bba60ac46d 100644 --- a/server/querier/engine/clickhouse/group.go +++ b/server/querier/engine/clickhouse/group.go @@ -208,15 +208,19 @@ func GetNotNullFilter(name string, e *CHEngine) (view.Node, bool) { filter = fmt.Sprintf(tagItem.NotNullFilter, filterName, filterName) } return &view.Expr{Value: "(" + filter + ")"}, true - } else if strings.HasPrefix(preAsTag, "tag.") || strings.HasPrefix(preAsTag, "attribute.") { + } else if strings.HasPrefix(preAsTag, "tag.") { if db == chCommon.DB_NAME_PROMETHEUS { return &view.Expr{}, false } tagItem, ok = tag.GetTag("tag.", db, table, "default") - filter := fmt.Sprintf(tagItem.NotNullFilter, name) + filter := fmt.Sprintf(tagItem.NotNullFilter, preAsTag) + return &view.Expr{Value: "(" + filter + ")"}, true + } else if strings.HasPrefix(preAsTag, "attribute.") { + tagItem, ok = tag.GetTag("attribute.", db, table, "default") + filter := fmt.Sprintf(tagItem.NotNullFilter, preAsTag) return &view.Expr{Value: "(" + filter + ")"}, true } else if common.IsValueInSliceString(preAsTag, []string{"request_id", "response_code", "span_kind", "request_length", "response_length", "sql_affected_rows"}) { - filter := fmt.Sprintf("%s is not null", name) + filter := fmt.Sprintf("%s is not null", preAsTag) return &view.Expr{Value: "(" + filter + ")"}, true } return &view.Expr{}, false @@ -239,13 +243,17 @@ func GetNotNullFilter(name string, e *CHEngine) (view.Node, bool) { filter = fmt.Sprintf(tagItem.NotNullFilter, filterName, filterName) } return &view.Expr{Value: "(" + filter + ")"}, true - } else if strings.HasPrefix(noBackQuoteName, "tag.") || strings.HasPrefix(noBackQuoteName, "attribute.") { + } else if strings.HasPrefix(noBackQuoteName, "tag.") { if db == chCommon.DB_NAME_PROMETHEUS { return &view.Expr{}, false } tagItem, ok = tag.GetTag("tag.", db, table, "default") filter := fmt.Sprintf(tagItem.NotNullFilter, name) return &view.Expr{Value: "(" + filter + ")"}, true + } else if strings.HasPrefix(noBackQuoteName, "attribute.") { + tagItem, ok = tag.GetTag("attribute.", db, table, "default") + filter := fmt.Sprintf(tagItem.NotNullFilter, name) + return &view.Expr{Value: "(" + filter + ")"}, true } else if common.IsValueInSliceString(noBackQuoteName, []string{"request_id", "response_code", "span_kind", "request_length", "response_length", "sql_affected_rows"}) { filter := fmt.Sprintf("%s is not null", name) return &view.Expr{Value: "(" + filter + ")"}, true diff --git a/server/querier/engine/clickhouse/tag/translation.go b/server/querier/engine/clickhouse/tag/translation.go index 3bed9373f25..d87174b2041 100644 --- a/server/querier/engine/clickhouse/tag/translation.go +++ b/server/querier/engine/clickhouse/tag/translation.go @@ -758,7 +758,7 @@ func GenerateTagResoureMap() map[string]map[string]*Tag { tagResourceMap["tag."] = map[string]*Tag{ "default": NewTag( "tag_values[indexOf(tag_names,'%s')]", - "%s != ''", + "tag_values[indexOf(tag_names,'%s')] != ''", "tag_values[indexOf(tag_names,'%s')] %s %v", "%s(tag_values[indexOf(tag_names,'%s')],%v)", ), @@ -767,7 +767,7 @@ func GenerateTagResoureMap() map[string]map[string]*Tag { tagResourceMap["attribute."] = map[string]*Tag{ "default": NewTag( "attribute_values[indexOf(attribute_names,'%s')]", - "%s != ''", + "attribute_values[indexOf(attribute_names,'%s')] != ''", "attribute_values[indexOf(attribute_names,'%s')] %s %v", "%s(attribute_values[indexOf(attribute_names,'%s')],%v)", ),