fix: add agent group get api

server/agent_config/config.go

@@ -21,6 +21,9 @@ import _ "embed"
 //go:embed example.yaml
 var YamlAgentGroupConfig []byte
 
+//go:embed template.yaml
+var YamlAgentGroupConfigTemplate []byte
+
 type AgentGroupConfig struct {
 	VTapGroupID                       *string       `json:"VTAP_GROUP_ID" yaml:"vtap_group_id,omitempty"`
 	VTapGroupLcuuid                   *string       `json:"VTAP_GROUP_LCUUID" yaml:"vtap_group_lcuuid,omitempty"`

server/agent_config/db.go

@@ -16,6 +16,20 @@
 
 package agent_config
 
+import "time"
+
+type AgentGroupConfigYaml struct {
+	ID        int       `gorm:"primaryKey;column:id;type:int;not null" json:"ID"`
+	Lcuuid    string    `gorm:"column:lcuuid;type:char(64);default:not null" json:"LCUUID"`
+	Yaml      string    `gorm:"column:yaml;type:text;default:not null" json:"YAML"`
+	CreatedAt time.Time `gorm:"column:created_at;type:timestamp;default:not null" json:"CREATED_AT"`
+	UpdatedAt time.Time `gorm:"column:updated_at;type:timestamp;default:not null" json:"UPDATED_AT"`
+}
+
+func (AgentGroupConfigYaml) TableName() string {
+	return "agent_group_configuration"
+}
+
 type AgentGroupConfigModel struct {
 	ID                                int      `gorm:"primaryKey;column:id;type:int;not null" json:"ID"`
 	MaxCollectPps                     *int     `gorm:"column:max_collect_pps;type:int;default:null" json:"MAX_COLLECT_PPS"`

server/agent_config/template.yaml

@@ -3027,7 +3027,7 @@ inputs:
       # description:
       #   en: |-
       #     Used when deepflow-agent has only one k8s namespace query permission.
-      #   en: |-
+      #   ch: |-
       #     TODO
       # upgrade_from: static_config.kubernetes-namespace
       # TODO: 英文释义待明确。
@@ -4339,7 +4339,7 @@ processors:
         #     Used to extract the SpanID field in HTTP and RPC headers, supports filling
         #     in multiple values separated by commas. This feature can be turned off by
         #     setting it to empty.
-         #   ch: |-
+        #   ch: |-
         #     配置该参数后，deepflow-agent 会尝试从 HTTP 和 RPC header 中匹配特征字段，并将匹配到
         #     的结果填充到应用调用日志的`span_id`字段中，作为调用链追踪的特征值。参数支持填写多个不同的
         #     特征字段，中间用`,`分隔。

server/agent_config/template_json.go

@@ -0,0 +1,204 @@
+/*
+ * Copyright (c) 2024 Yunshan Networks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package agent_config
+
+import (
+	"fmt"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+	k8syaml "sigs.k8s.io/yaml"
+)
+
+func ParseTemplateYAMLToJson(yamlData []byte) ([]byte, error) {
+	formatedTemplate, err := formatTemplateYAML(yamlData)
+	if err != nil {
+		return nil, err
+	}
+	return k8syaml.YAMLToJSON(formatedTemplate)
+}
+
+func formatTemplateYAML(yamlData []byte) ([]byte, error) {
+	indentedLines, err := IndentTemplate(yamlData)
+	if err != nil {
+		return nil, err
+	}
+	return UncommentTemplate(indentedLines)
+}
+
+func IndentTemplate(yamlData []byte) ([]string, error) {
+	yamlStr := string(YamlAgentGroupConfigTemplate)
+	lines := strings.Split(yamlStr, "\n")
+
+	var indentedLines []string
+	var tempLines []string
+	for i := 0; i < len(lines); i++ {
+		if strings.Contains(lines[i], "---") {
+			j := i + 1
+			for ; j < len(lines); j++ {
+				if strings.Contains(lines[j], "---") {
+					break
+				}
+			}
+			i = j + 1
+			continue
+		}
+
+		if strings.Contains(lines[i], "#") {
+			if !strings.HasPrefix(strings.TrimSpace(lines[i]), "#") {
+				tempLines = append(tempLines, lines[i])
+			} else {
+				// add indent to config
+				tempLines = append(tempLines, "  "+lines[i])
+			}
+			continue
+		}
+
+		configNames := strings.Split(lines[i], ":")
+		if len(configNames) == 0 {
+			return nil, fmt.Errorf("line(index: %d, value: %s) split by \":\" failed", i, lines[i])
+		}
+		if len(tempLines) > 0 {
+			indentedLines = append(indentedLines, configNames[0]+"_comment:")
+			indentedLines = append(indentedLines, tempLines...)
+		}
+		indentedLines = append(indentedLines, lines[i])
+		tempLines = []string{}
+	}
+
+	return indentedLines, nil
+}
+
+// UncommentTemplate removes all lines containing "TODO" and uncomments all lines
+// by removing the "#" prefix. It is used to convert the agent configuration
+// template file to json format.
+func UncommentTemplate(indentedLines []string) ([]byte, error) {
+	var uncommentedLines []string
+	for i := 0; i < len(indentedLines); i++ {
+		line := indentedLines[i]
+		if strings.Contains(line, "TODO") {
+			continue
+		}
+		if !strings.HasPrefix(strings.TrimSpace(line), "#") && strings.Contains(line, "#") {
+			uncommentedLines = append(uncommentedLines, line)
+			continue
+		}
+
+		line = strings.ReplaceAll(line, "# ", "")
+		line = strings.ReplaceAll(line, "#", "")
+		uncommentedLines = append(uncommentedLines, line)
+	}
+	return []byte(strings.Join(uncommentedLines, "\n")), nil
+}
+
+func ParseJsonToYAMLAndValidate(jsonData map[string]interface{}) ([]byte, error) {
+	b, err := yaml.Marshal(jsonData)
+	if err != nil {
+		return nil, err
+	}
+	yamlData, err := k8syaml.JSONToYAML(b)
+	if err != nil {
+		return nil, err
+	}
+	var node yaml.Node
+	err = yaml.Unmarshal(yamlData, &node)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal data to yaml node error: %v", err)
+	}
+
+	formatedTemplate, err := formatTemplateYAML(YamlAgentGroupConfigTemplate)
+	if err != nil {
+		return nil, err
+	}
+	var nodeTemplate yaml.Node
+	err = yaml.Unmarshal(formatedTemplate, &nodeTemplate)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal template data to yaml node error: %v", err)
+	}
+
+	// 检查 node 中的每一项是否包含在 nodeTemplate 中
+	err = validateNodeAgainstTemplate(&node, &nodeTemplate)
+	if err != nil {
+		return nil, fmt.Errorf("验证配置失败: %v", err)
+	}
+
+	return yamlData, nil
+}
+
+func validateNodeAgainstTemplate(node, nodeTemplate *yaml.Node) error {
+	if node.Kind != nodeTemplate.Kind {
+		return fmt.Errorf("node kind mismatch: expected %v, got %v", nodeTemplate.Kind, node.Kind)
+	}
+
+	switch node.Kind {
+	case yaml.DocumentNode:
+		if len(node.Content) != len(nodeTemplate.Content) {
+			return fmt.Errorf("document node content length mismatch")
+		}
+		for i := range node.Content {
+			if err := validateNodeAgainstTemplate(node.Content[i], nodeTemplate.Content[i]); err != nil {
+				return err
+			}
+		}
+	case yaml.MappingNode:
+		nodeMap := make(map[string]*yaml.Node)
+		for i := 0; i < len(node.Content); i += 2 {
+			nodeMap[node.Content[i].Value] = node.Content[i+1]
+		}
+
+		templateMap := make(map[string]*yaml.Node)
+		for i := 0; i < len(nodeTemplate.Content); i += 2 {
+			templateMap[nodeTemplate.Content[i].Value] = nodeTemplate.Content[i+1]
+		}
+
+		for key, value := range nodeMap {
+			if templateValue, exists := templateMap[key]; exists {
+				if err := validateNodeAgainstTemplate(value, templateValue); err != nil {
+					return fmt.Errorf("invalid value for key '%s': %v", key, err)
+				}
+			} else {
+				return fmt.Errorf("unexpected key in configuration: %s", key)
+			}
+		}
+	case yaml.SequenceNode:
+		if len(node.Content) > 0 && len(nodeTemplate.Content) > 0 {
+			for _, item := range node.Content {
+				if err := validateNodeAgainstTemplate(item, nodeTemplate.Content[0]); err != nil {
+					return fmt.Errorf("invalid sequence item: %v", err)
+				}
+			}
+		} else if len(node.Content) > 0 && len(nodeTemplate.Content) == 0 {
+			return fmt.Errorf("unexpected sequence in configuration")
+		}
+	case yaml.ScalarNode:
+		if node.Tag != nodeTemplate.Tag {
+			return fmt.Errorf("scalar type mismatch: expected %s, got %s", nodeTemplate.Tag, node.Tag)
+		}
+	case yaml.AliasNode:
+		if nodeTemplate.Kind != yaml.AliasNode {
+			return fmt.Errorf("unexpected alias node")
+		}
+		// For alias nodes, we should validate the actual content they refer to
+		if err := validateNodeAgainstTemplate(node.Alias, nodeTemplate.Alias); err != nil {
+			return fmt.Errorf("invalid alias node: %v", err)
+		}
+	default:
+		return fmt.Errorf("unsupported node kind: %v", node.Kind)
+	}
+
+	return nil
+}

server/agent_config/template_json_test.go

@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2024 Yunshan Networks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package agent_config
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseTemplateYAMLToJson(t *testing.T) {
+	type args struct {
+		yamlData []byte
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "case01",
+			args: args{
+				yamlData: YamlAgentGroupConfigTemplate,
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ParseTemplateYAMLToJson(tt.args.yamlData)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("err: %v", err)
+				t.Errorf("ParseTemplateYAMLToJson() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			fmt.Println(got)
+			if err = os.WriteFile("template_2.json", got, os.ModePerm); err != nil {
+				t.Fatalf("Failed to write to file: %v", err)
+			}
+		})
+	}
+}
+
+func TestIndentAndUncommentTemplate(t *testing.T) {
+	type args struct {
+		yamlData []byte
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []string
+		wantErr bool
+	}{
+		{
+			name: "case01",
+			args: args{
+				yamlData: YamlAgentGroupConfigTemplate,
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			indentedLines, err := IndentTemplate(tt.args.yamlData)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("IndentTemplate() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+
+			uncommentedLines, err := UncommentTemplate(indentedLines)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("UncommentTemplate() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+
+			if err := os.WriteFile("template_formated.yaml", []byte(strings.Join(indentedLines, "\n")), os.ModePerm); err != nil {
+				t.Fatalf("Failed to write to file: %v", err)
+			}
+			if err := os.WriteFile("template_uncommented.yaml", uncommentedLines, os.ModePerm); err != nil {
+				t.Fatalf("Failed to write to file: %v", err)
+			}
+		})
+	}
+}
+
+func TestParseJsonToYAMLAndValidate(t *testing.T) {
+	type args struct {
+		jsonData map[string]interface{}
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []byte
+		wantErr bool
+	}{
+		{
+			name: "case01",
+			args: args{
+				jsonData: map[string]interface{}{
+					"global": map[string]interface{}{
+						"alerts": map[string]interface{}{
+							"check_core_file_disabled": true,
+						},
+					},
+				},
+			},
+			want: []byte(`global:
+  alerts:
+    check_core_file_disabled: true
+`),
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ParseJsonToYAMLAndValidate(tt.args.jsonData)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseJsonToYAMLAndValidate() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			assert.EqualValues(t, string(tt.want), string(got))
+			if err = os.WriteFile("template_3.yaml", got, os.ModePerm); err != nil {
+				t.Fatalf("Failed to write to file: %v", err)
+			}
+		})
+	}
+}