From b1a1f08de967b721bbde5a31977ebf90073d15d3 Mon Sep 17 00:00:00 2001 From: zhengya Date: Wed, 11 Oct 2023 10:58:10 +0800 Subject: [PATCH 01/22] [CONTROLLER/DB] splits model file --- server/controller/db/mysql/model.go | 805 ----------------- .../controller/db/mysql/platform_rsc_model.go | 827 ++++++++++++++++++ 2 files changed, 827 insertions(+), 805 deletions(-) create mode 100644 server/controller/db/mysql/platform_rsc_model.go diff --git a/server/controller/db/mysql/model.go b/server/controller/db/mysql/model.go index 817bc6fc9d5..0aa87ad83c9 100644 --- a/server/controller/db/mysql/model.go +++ b/server/controller/db/mysql/model.go @@ -24,813 +24,8 @@ import ( "fmt" "io" "time" - - "gorm.io/gorm" ) -type Base struct { - ID int `gorm:"primaryKey;autoIncrement;unique;column:id;type:int;not null" json:"ID"` - Lcuuid string `gorm:"unique;column:lcuuid;type:char(64)" json:"LCUUID"` - // TODO add CreatedAt/UpdatedAt/DeletedAt -} - -func (b Base) GetID() int { - return b.ID -} - -func (b *Base) SetID(id int) { - b.ID = id -} - -func (b Base) GetLcuuid() string { - return b.Lcuuid -} - -type OperatedTime struct { - CreatedAt time.Time `gorm:"autoCreateTime;column:created_at;type:datetime" json:"CREATED_AT"` - UpdatedAt time.Time `gorm:"autoUpdateTime;column:updated_at;type:datetime" json:"UPDATED_AT"` -} - -type SoftDeleteBase struct { - OperatedTime - DeletedAt gorm.DeletedAt `gorm:"column:deleted_at;type:datetime;default:null" json:"DELETED_AT"` -} - -type ResourceEvent struct { - ID int `gorm:"primaryKey;autoIncrement;unique;column:id;type:int;not null" json:"ID"` - Domain string `gorm:"column:domain;type:char(64);default:''" json:"DOMAIN"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` - ResourceLcuuid string `gorm:"column:resource_lcuuid;type:char(64);default:''" json:"RESOURCE_LCUUID"` - Content string `gorm:"column:content;type:text" json:"CONTENT"` - CreatedAt time.Time `gorm:"autoCreateTime;column:created_at;type:datetime" json:"CREATED_AT"` -} - -type DomainAdditionalResource struct { - ID int `gorm:"primaryKey;autoIncrement;unique;column:id;type:int;not null" json:"ID"` - Domain string `gorm:"column:domain;type:char(64);default:''" json:"DOMAIN"` - Content string `gorm:"column:content;type:longtext" json:"CONTENT"` - CompressedContent compressedBytes `gorm:"column:compressed_content;type:longblob" json:"COMPRESSED_CONTENT"` - CreatedAt time.Time `gorm:"autoCreateTime;column:created_at;type:datetime" json:"CREATED_AT"` -} -type Process struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - VTapID uint32 `gorm:"column:vtap_id;type:int;not null;default:0" json:"VTAP_ID"` - PID uint64 `gorm:"column:pid;type:int;not null;default:0" json:"PID"` - ProcessName string `gorm:"column:process_name;type:varchar(256);default:''" json:"PROCESS_NAME"` - CommandLine string `gorm:"column:command_line;type:text" json:"COMMAND_LINE"` - UserName string `gorm:"column:user_name;type:varchar(256);default:''" json:"USER_NAME"` - StartTime time.Time `gorm:"autoCreateTime;column:start_time;type:datetime" json:"START_TIME"` - OSAPPTags string `gorm:"column:os_app_tags;type:text" json:"OS_APP_TAGS"` - ContainerID string `gorm:"column:container_id;type:char(64);default:''" json:"CONTAINER_ID"` - NetnsID uint32 `gorm:"column:netns_id;type:int unsigned;default:0" json:"NETNS_ID"` // used to associate processes with cloud and container resources - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` - Domain string `gorm:"column:domain;type:char(64);default:''" json:"DOMAIN"` -} - -type Domain struct { - Base `gorm:"embedded"` - OperatedTime `gorm:"embedded"` - SyncedAt *time.Time `gorm:"column:synced_at" json:"SYNCED_AT"` - Name string `gorm:"column:name;type:varchar(64)" json:"NAME"` - IconID int `gorm:"column:icon_id;type:int" json:"ICON_ID"` - DisplayName string `gorm:"column:display_name;type:varchar(64);default:''" json:"DISPLAY_NAME"` - ClusterID string `gorm:"column:cluster_id;type:char(64)" json:"CLUSTER_ID"` - Type int `gorm:"column:type;type:int;default:0" json:"TYPE"` // 1.openstack 2.vsphere 3.nsp 4.tencent 5.filereader 6.aws 7.pingan 8.zstack 9.aliyun 10.huawei prv 11.k8s 12.simulation 13.huawei 14.qingcloud 15.qingcloud_private 16.F5 17.CMB_CMDB 18.azure 19.apsara_stack 20.tencent_tce 21.qingcloud_k8s 22.kingsoft_private 23.genesis 24.microsoft_acs 25.baidu_bce - Config string `gorm:"column:config;type:text" json:"CONFIG"` - ErrorMsg string `gorm:"column:error_msg;type:text" json:"ERROR_MSG"` - Enabled int `gorm:"column:enabled;type:int;not null;default:1" json:"ENABLED"` // 0.false 1.true - State int `gorm:"column:state;type:int;not null;default:1" json:"STATE"` // 1.normal 2.deleting 3.exception - ControllerIP string `gorm:"column:controller_ip;type:char(64)" json:"CONTROLLER_IP"` -} - -// TODO 最终可以与cloud模块命名统一,Domain -> DomainLcuuid - -type SubDomain struct { - Base `gorm:"embedded"` - OperatedTime `gorm:"embedded"` - SyncedAt *time.Time `gorm:"column:synced_at" json:"SYNCED_AT"` - Domain string `gorm:"column:domain;type:char(64);default:''" json:"DOMAIN"` - Name string `gorm:"column:name;type:varchar(64);default:''" json:"NAME"` - DisplayName string `gorm:"column:display_name;type:varchar(64);default:''" json:"DISPLAY_NAME"` - CreateMethod int `gorm:"column:create_method;type:int;default:0" json:"CREATE_METHOD"` // 0.learning 1.user_defined - ClusterID string `gorm:"column:cluster_id;type:char(64);default:''" json:"CLUSTER_ID"` - Config string `gorm:"column:config;type:text;default:''" json:"CONFIG"` - ErrorMsg string `gorm:"column:error_msg;type:text;default:''" json:"ERROR_MSG"` - Enabled int `gorm:"column:enabled;type:int;not null;default:1" json:"ENABLED"` // 0.false 1.true - State int `gorm:"column:state;type:int;not null;default:1" json:"STATE"` // 1.normal 2.deleting 3.exception -} - -type Region struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(64);default:''" json:"NAME"` - CreateMethod int `gorm:"column:create_method;type:int;default:0" json:"CREATE_METHOD"` // 0.learning 1.user_defined - Label string `gorm:"column:label;type:varchar(64);default:''" json:"LABEL"` - Longitude float64 `gorm:"column:longitude;type:double(7,4);default:null" json:"LONGITUDE"` - Latitude float64 `gorm:"column:latitude;type:double(7,4);default:null" json:"LATITUDE"` -} - -type AZ struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(64);default:''" json:"NAME"` - CreateMethod int `gorm:"column:create_method;type:int;default:0" json:"CREATE_METHOD"` // 0.learning 1.user_defined - Label string `gorm:"column:label;type:varchar(64);default:''" json:"LABEL"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - Domain string `gorm:"column:domain;type:char(64);default:''" json:"DOMAIN"` -} - -func (AZ) TableName() string { - return "az" -} - -type Host struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Type int `gorm:"column:type;type:int" json:"TYPE"` // 1.Server 3.Gateway 4.DFI - State int `gorm:"column:state;type:int" json:"STATE"` // 0.Temp 1.Creating 2.Complete 3.Modifying 4.Exception - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Alias string `gorm:"column:alias;type:char(64);default:''" json:"ALIAS"` - Description string `gorm:"column:description;type:varchar(256);default:''" json:"DESCRIPTION"` - IP string `gorm:"column:ip;type:char(64);default:''" json:"IP"` - HType int `gorm:"column:htype;type:int" json:"HTYPE"` // 1. Xen host 2. VMware host 3. KVM host 4. Public cloud host 5. Hyper-V - CreateMethod int `gorm:"column:create_method;type:int;default:0" json:"CREATE_METHOD"` // 0.learning 1.user_defined - UserName string `gorm:"column:user_name;type:varchar(64);default:''" json:"USER_NAME"` - UserPasswd string `gorm:"column:user_passwd;type:varchar(64);default:''" json:"USER_PASSWD"` - VCPUNum int `gorm:"column:vcpu_num;type:int;default:0" json:"VCPU_NUM"` - MemTotal int `gorm:"column:mem_total;type:int;default:0" json:"MEM_TOTAL"` // unit: M - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - Domain string `gorm:"column:domain;type:char(64);default:''" json:"DOMAIN"` - SyncedAt time.Time `gorm:"column:synced_at;type:datetime;not null;default:CURRENT_TIMESTAMP" json:"SYNCED_AT"` - ExtraInfo string `gorm:"column:extra_info;type:text;default:''" json:"EXTRA_INFO"` -} - -func (Host) TableName() string { - return "host_device" -} - -type VM struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - State int `gorm:"index:state_server_index;column:state;type:int;not null" json:"STATE"` // 0.Temp 1.Creating 2.Created 3.To run 4.Running 5.To suspend 6.Suspended 7.To resume 8. To stop 9.Stopped 10.Modifing 11.Exception 12.Destroying - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Alias string `gorm:"column:alias;type:char(64);default:''" json:"ALIAS"` - Label string `gorm:"column:label;type:char(64);default:''" json:"LABEL"` - CreateMethod int `gorm:"column:create_method;type:int;default:0" json:"CREATE_METHOD"` // 0.learning 1.user_defined - HType int `gorm:"column:htype;type:int;default:1" json:"HTYPE"` // 1.vm-c 2.bm-c 3.vm-n 4.bm-n 5.vm-s 6.bm-s - LaunchServer string `gorm:"index:state_server_index;column:launch_server;type:char(64);default:''" json:"LAUNCH_SERVER"` - CloudTags string `gorm:"column:cloud_tags;type:text;default:''" json:"CLOUD_TAGS"` // separated by , - VPCID int `gorm:"column:epc_id;type:int;default:0" json:"EPC_ID"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - UID string `gorm:"column:uid;type:char(64);default:''" json:"UID"` -} - -func (VM) TableName() string { - return "vm" -} - -type VMPodNodeConnection struct { - Base `gorm:"embedded"` - VMID int `gorm:"column:vm_id;type:int;default:null" json:"VM_ID"` - PodNodeID int `gorm:"column:pod_node_id;type:int;default:null" json:"POD_NODE_ID"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` -} - -func (VMPodNodeConnection) TableName() string { - return "vm_pod_node_connection" -} - -type VMSecurityGroup struct { - Base `gorm:"embedded"` - SecurityGroupID int `gorm:"column:sg_id;type:int;not null" json:"SG_ID"` - VMID int `gorm:"column:vm_id;type:int;not null" json:"VM_ID"` - Priority int `gorm:"column:priority;type:int;not null" json:"PRIORITY"` -} - -func (VMSecurityGroup) TableName() string { - return "vm_security_group" -} - -type Contact struct { - Base `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - CreateMethod int `gorm:"column:create_method;type:int;default:0" json:"CREATE_METHOD"` // 0.learning 1.user_defined - Mobile string `gorm:"column:mobile;type:char(13);default:''" json:"MOBILE"` - Email string `gorm:"column:email;type:varchar(128);default:''" json:"EMAIL"` - Company string `gorm:"column:company;type:varchar(128);default:''" json:"COMPANY"` - PushEmail string `gorm:"column:push_email;type:text;default:null" json:"PUSH_EMAIL"` - Domain string `gorm:"column:domain;type:char(64);default:''" json:"DOMAIN"` - AlarmPush int `gorm:"column:alarm_push;type:int;default:0" json:"ALARM_PUSH"` - ReportPush int `gorm:"column:report_push;type:int;default:0" json:"REPORT_PUSH"` - Deleted int `gorm:"column:deleted;type:int;default:0" json:"DELETED"` - CreatedAt time.Time `gorm:"column:created_at;type:datetime;not null;default:CURRENT_TIMESTAMP" json:"CREATED_AT"` - UpdatedAt time.Time `gorm:"column:updated_at;type:datetime;default:null" json:"UPDATED_AT"` -} - -type VPCContact struct { - Base `gorm:"embedded"` - CreateMethod int `gorm:"column:create_method;type:int;default:0" json:"CREATE_METHOD"` // 0.learning 1.user_defined - VPCID int `gorm:"column:epc_id;type:int;default:0" json:"VPC_ID"` - ContactID int `gorm:"column:contact_id;type:int;default:0" json:"CONTACT_ID"` -} - -func (VPCContact) TableName() string { - return "epc_contact" -} - -type VPC struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - CreateMethod int `gorm:"column:create_method;type:int;default:0" json:"CREATE_METHOD"` // 0.learning 1.user_defined - Label string `gorm:"column:label;type:varchar(64);default:''" json:"LABEL"` - Alias string `gorm:"column:alias;type:char(64);default:''" json:"ALIAS"` - Domain string `gorm:"column:domain;type:char(64);default:''" json:"DOMAIN"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` // TODO delete in future - TunnelID int `gorm:"column:tunnel_id;type:int;default:0" json:"TUNNEL_ID"` - Mode int `gorm:"column:mode;type:int;default:2" json:"MODE"` // 1:route, 2:transparent - CIDR string `gorm:"column:cidr;type:char(64);default:''" json:"CIDR"` - UID string `gorm:"column:uid;type:char(64);default:''" json:"UID"` -} - -func (VPC) TableName() string { - return "epc" -} - -type Network struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - State int `gorm:"column:state;type:int;not null" json:"STATE"` // 0.Temp 1.Creating 2.Created 3.Exception 4.Modifing 5.Destroying 6.Destroyed - NetType int `gorm:"column:net_type;type:int;default:4" json:"NET_TYPE"` // 1.CTRL 2.SERVICE 3.WAN 4.LAN - Name string `gorm:"column:name;type:varchar(256);not null" json:"NAME"` - CreateMethod int `gorm:"column:create_method;type:int;default:0" json:"CREATE_METHOD"` // 0.learning 1.user_defined - Label string `gorm:"column:label;type:varchar(64);default:''" json:"LABEL"` - Alias string `gorm:"column:alias;type:char(64);default:''" json:"ALIAS"` - Description string `gorm:"column:description;type:varchar(256);default:''" json:"DESCRIPTION"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` - ISP int `gorm:"column:isp;type:int;default:0" json:"ISP"` - VPCID int `gorm:"column:epc_id;type:int;default:0" json:"VPC_ID"` - SegmentationID int `gorm:"column:segmentation_id;type:int;default:0" json:"SEGMENTATION_ID"` - TunnelID int `gorm:"column:tunnel_id;type:int;default:0" json:"TUNNEL_ID"` - Shared bool `gorm:"column:shared;type:int;default:0" json:"SHARED"` - Topped int `gorm:"column:topped;type:int;default:0" json:"TOPPED"` - IsVIP int `gorm:"column:is_vip;type:int;default:0" json:"IS_VIP"` -} - -func (Network) TableName() string { - return "vl2" -} - -type Subnet struct { - Base `gorm:"embedded"` - Prefix string `gorm:"column:prefix;type:char(64);default:''" json:"PREFIX"` - Netmask string `gorm:"column:netmask;type:char(64);default:''" json:"NETMASK"` - NetworkID int `gorm:"column:vl2id;type:int;default:null" json:"VL2ID"` - NetIndex int `gorm:"column:net_index;type:int;default:0" json:"NET_INDEX"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Label string `gorm:"column:label;type:varchar(64);default:''" json:"LABEL"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` -} - -func (Subnet) TableName() string { - return "vl2_net" -} - -type VRouter struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - State int `gorm:"index:state_server_index;column:state;type:int;not null" json:"STATE"` // 0.Temp 1.Creating 2.Created 3.Exception 4.Modifing 5.Destroying 6.To run 7.Running 8.To stop 9.Stopped - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Label string `gorm:"column:label;type:char(64);default:''" json:"LABEL"` - Description string `gorm:"column:description;type:varchar(256);default:''" json:"DESCRIPTION"` - VPCID int `gorm:"column:epc_id;type:int;default:0" json:"EPC_ID"` - GWLaunchServer string `gorm:"index:state_server_index;column:gw_launch_server;type:char(64);default:''" json:"GW_LAUNCH_SERVER"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` // TODO delete in future -} - -func (VRouter) TableName() string { - return "vnet" -} - -type RoutingTable struct { - Base `gorm:"embedded"` - VRouterID int `gorm:"column:vnet_id;type:int;default:null" json:"VNET_ID"` - Destination string `gorm:"column:destination;type:text;default:''" json:"DESTINATION"` - NexthopType string `gorm:"column:nexthop_type;type:text;default:''" json:"NEXTHOP_TYPE"` - Nexthop string `gorm:"column:nexthop;type:text;default:''" json:"NEXTHOP"` -} - -type DHCPPort struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` - VPCID int `gorm:"column:epc_id;type:int;default:0" json:"VPC_ID"` -} - -func (DHCPPort) TableName() string { - return "dhcp_port" -} - -type VInterface struct { - Base `gorm:"embedded"` - Name string `gorm:"column:name;type:char(64);default:''" json:"NAME"` - Index int `gorm:"column:ifindex;type:int;not null" json:"IFINDEX"` - State int `gorm:"column:state;type:int;not null" json:"STATE"` // 1. Attached 2.Detached 3.Exception - CreateMethod int `gorm:"column:create_method;type:int;default:0" json:"CREATE_METHOD"` // 0.learning 1.user_defined - Type int `gorm:"column:iftype;type:int;default:0" json:"IFTYPE"` // 0.Unknown 1.Control 2.Service 3.WAN 4.LAN 5.Trunk 6.Tap 7.Tool - Mac string `gorm:"index:mac_index;column:mac;type:char(32);default:''" json:"MAC"` - VMac string `gorm:"column:vmac;type:char(32);default:''" json:"VMAC"` - TapMac string `gorm:"column:tap_mac;type:char(32);default:''" json:"TAP_MAC"` - NetworkID int `gorm:"column:subnetid;type:int;default:0" json:"SUBNETID"` // vl2 id - VlanTag int `gorm:"column:vlantag;type:int;default:0" json:"VLANTAG"` - DeviceType int `gorm:"column:devicetype;type:int;default:null" json:"DEVICETYPE"` // Type 0.unknown 1.vm 2.vgw 3.third-party-device 4.vmwaf 5.NSP-vgateway 6.host-device 7.network-device 9.DHCP-port 10.pod 11.pod_service 12. redis_instance 13. rds_instance 14. pod_node 15. load_balance 16. nat_gateway - DeviceID int `gorm:"column:deviceid;type:int;default:null" json:"DEVICEID"` // unknown: Senseless ID, vm: vm ID, vgw/NSP-vgateway: vnet ID, third-party-device: third_party_device ID, vmwaf: vmwaf ID, host-device: host_device ID, network-device: network_device ID - NetnsID uint32 `gorm:"column:netns_id;type:int unsigned;default:0" json:"NETNS_ID"` // used to associate processes with cloud and container resources - VtapID uint32 `gorm:"column:vtap_id;type:int;default:0" json:"VTAP_ID"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - CreatedAt time.Time `gorm:"column:created_at;type:timestamp;not null;default:CURRENT_TIMESTAMP" json:"CREATED_AT"` - UpdatedAt time.Time `gorm:"column:updated_at;type:timestamp;not null;default:CURRENT_TIMESTAMP" json:"UPDATED_AT"` -} - -func (VInterface) TableName() string { - return "vinterface" -} - -type LANIP struct { // TODO 添加region字段 - Base `gorm:"embedded"` - IP string `gorm:"column:ip;type:char(64);default:''" json:"IP"` - Netmask string `gorm:"column:netmask;type:char(64);default:''" json:"NETMASK"` - Gateway string `gorm:"column:gateway;type:char(64);default:''" json:"GATEWAY"` - CreateMethod int `gorm:"column:create_method;type:int;default:0" json:"CREATE_METHOD"` // 0.learning 1.user_defined - NetworkID int `gorm:"column:vl2id;type:int;default:null" json:"VL2ID"` - NetIndex int `gorm:"column:net_index;type:int;default:0" json:"NET_INDEX"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` - VInterfaceID int `gorm:"column:vifid;type:int;default:null" json:"VINTERFACE_ID"` - SubnetID int `gorm:"column:vl2_net_id;type:int;default:0" json:"SUBNET_ID"` - ISP int `gorm:"column:isp;type:int;default:0" json:"ISP"` // Used for multi-ISP access - CreatedAt time.Time `gorm:"column:created_at;type:timestamp;not null;default:CURRENT_TIMESTAMP" json:"CREATED_AT"` - UpdatedAt time.Time `gorm:"column:updated_at;type:timestamp;not null;default:CURRENT_TIMESTAMP" json:"UPDATED_AT"` -} - -func (LANIP) TableName() string { - return "vinterface_ip" -} - -type WANIP struct { - Base `gorm:"embedded"` - IP string `gorm:"column:ip;type:char(64);default:''" json:"IP"` - Alias string `gorm:"column:alias;type:char(64);default:''" json:"ALIAS"` - Netmask int `gorm:"column:netmask;type:int;default:null" json:"NETMASK"` - Gateway string `gorm:"column:gateway;type:char(64);default:''" json:"GATEWAY"` - CreateMethod int `gorm:"column:create_method;type:int;default:0" json:"CREATE_METHOD"` // 0.learning 1.user_defined - ISP int `gorm:"column:isp;type:int;default:null" json:"ISP"` - VInterfaceID int `gorm:"column:vifid;type:int;default:0" json:"VINTERFACE_ID"` - SubnetID int `gorm:"column:vl2_net_id;type:int;default:0" json:"SUBNET_ID"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - CreatedAt time.Time `gorm:"column:created_at;type:timestamp;not null;default:CURRENT_TIMESTAMP" json:"CREATED_AT"` - UpdatedAt time.Time `gorm:"column:updated_at;type:timestamp;not null;default:CURRENT_TIMESTAMP" json:"UPDATED_AT"` -} - -func (WANIP) TableName() string { - return "ip_resource" -} - -type FloatingIP struct { - Base `gorm:"embedded"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - VPCID int `gorm:"column:epc_id;type:int;default:0" json:"VPC_ID"` - NetworkID int `gorm:"column:vl2_id;type:int;default:null" json:"VL2_ID"` // TODO json字段是否能修改,需返回给前端? - VMID int `gorm:"column:vm_id;type:int;default:null" json:"VM_ID"` - IP string `gorm:"column:ip;type:char(64);default:''" json:"IP"` -} - -func (FloatingIP) TableName() string { - return "floatingip" -} - -type SecurityGroup struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Label string `gorm:"column:label;type:varchar(64);default:''" json:"LABEL"` - Alias string `gorm:"column:alias;type:char(64);default:''" json:"ALIAS"` - VPCID int `gorm:"column:epc_id;type:int;default:0" json:"EPC_ID"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - Topped int `gorm:"column:topped;type:int;default:0" json:"TOPPED"` -} - -type SecurityGroupRule struct { - Base `gorm:"embedded"` - SecurityGroupID int `gorm:"column:sg_id;type:int;not null" json:"SG_ID"` - Direction int `gorm:"column:direction;type:tinyint(1);not null;default:0" json:"DIRECTION"` // 0.Unknow 1.Ingress 2.Egress - Protocol string `gorm:"column:protocol;type:char(64);default:''" json:"PROTOCOL"` - EtherType int `gorm:"column:ethertype;type:tinyint(1);not null;default:0" json:"ETHERTYPE"` // 0.Unknow 1.IPv4 2.IPv6 - LocalPortRange string `gorm:"column:local_port_range;type:text;default:''" json:"LOCAL_PORT_RANGE"` - RemotePortRange string `gorm:"column:remote_port_range;type:text;default:''" json:"REMOTE_PORT_RANGE"` - Local string `gorm:"column:local;type:text;default:''" json:"LOCAL"` - Remote string `gorm:"column:remote;type:text;default:''" json:"REMOTE"` - Priority int `gorm:"column:priority;type:int;not null" json:"PRIORITY"` - Action int `gorm:"column:action;type:tinyint(1);not null;default:0" json:"ACTION"` // 0.Unknow 1.Accept 2.Drop -} - -type NATGateway struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Label string `gorm:"column:label;type:char(64);default:''" json:"LABEL"` - FloatingIPs string `gorm:"column:floating_ips;type:text;default:''" json:"FLOATING_IPS"` // separated by , - VPCID int `gorm:"column:epc_id;type:int;default:0" json:"EPC_ID"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` // TODO delete in future - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` - UID string `gorm:"column:uid;type:char(64);default:''" json:"UID"` -} - -func (NATGateway) TableName() string { - return "nat_gateway" -} - -type NATRule struct { - Base `gorm:"embedded"` - NATGatewayID int `gorm:"column:nat_id;type:int;default:0" json:"NAT_ID"` - Type string `gorm:"column:type;type:char(16);default:''" json:"TYPE"` - Protocol string `gorm:"column:protocol;type:char(64);default:''" json:"PROTOCOL"` - FloatingIP string `gorm:"column:floating_ip;type:char(64);default:''" json:"FLOATING_IP"` - FloatingIPPort int `gorm:"column:floating_ip_port;type:int;default:null" json:"FLOATING_IP_PORT"` - FixedIP string `gorm:"column:fixed_ip;type:char(64);default:''" json:"FIXED_IP"` - FixedIPPort int `gorm:"column:fixed_ip_port;type:int;default:null" json:"FIXED_IP_PORT"` - VInterfaceID int `gorm:"column:port_id;type:int;default:null" json:"PORT_ID"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` -} - -func (NATRule) TableName() string { - return "nat_rule" -} - -type NATVMConnection struct { - Base `gorm:"embedded"` - NATGatewayID int `gorm:"column:nat_id;type:int;default:null" json:"NAT_ID"` - VMID int `gorm:"column:vm_id;type:int;default:null" json:"VM_ID"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` -} - -func (NATVMConnection) TableName() string { - return "nat_vm_connection" -} - -type LB struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Label string `gorm:"column:label;type:char(64);default:''" json:"LABEL"` - Model int `gorm:"column:model;type:int;default:0" json:"MODEL"` // 1.Internal 2.External - VIP string `gorm:"column:vip;type:text;default:''" json:"VIP"` - VPCID int `gorm:"column:epc_id;type:int;default:0" json:"EPC_ID"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` // TODO delete in future - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` - UID string `gorm:"column:uid;type:char(64);default:''" json:"UID"` -} - -func (LB) TableName() string { - return "lb" -} - -type LBListener struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - LBID int `gorm:"column:lb_id;type:int;default:0" json:"LB_ID"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - IPs string `gorm:"column:ips;type:text;default:''" json:"IPS"` // separated by , - SNATIPs string `gorm:"column:snat_ips;type:text;default:''" json:"SNAT_IPS"` // separated by , - Label string `gorm:"column:label;type:char(64);default:''" json:"LABEL"` - Port int `gorm:"column:port;type:int;default:null" json:"PORT"` - Protocol string `gorm:"column:protocol;type:char(64);default:''" json:"PROTOCOL"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` -} - -func (LBListener) TableName() string { - return "lb_listener" -} - -type LBTargetServer struct { - Base `gorm:"embedded"` - LBID int `gorm:"column:lb_id;type:int;default:0" json:"LB_ID"` - LBListenerID int `gorm:"column:lb_listener_id;type:int;default:0" json:"LB_LISTENER_ID"` - VPCID int `gorm:"column:epc_id;type:int;default:0" json:"EPC_ID"` - Type int `gorm:"column:type;type:int;default:0" json:"TYPE"` // 1.VM 2.IP - IP string `gorm:"column:ip;type:char(64);default:''" json:"IP"` - VMID int `gorm:"column:vm_id;type:int;default:0" json:"VM_ID"` - Port int `gorm:"column:port;type:int;default:null" json:"PORT"` - Protocol string `gorm:"column:protocol;type:char(64);default:''" json:"PROTOCOL"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` -} - -func (LBTargetServer) TableName() string { - return "lb_target_server" -} - -type LBVMConnection struct { - Base `gorm:"embedded"` - LBID int `gorm:"column:lb_id;type:int;default:null" json:"LB_ID"` - VMID int `gorm:"column:vm_id;type:int;default:null" json:"VM_ID"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` -} - -func (LBVMConnection) TableName() string { - return "lb_vm_connection" -} - -type PeerConnection struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Label string `gorm:"column:label;type:char(64);default:''" json:"LABEL"` - LocalVPCID int `gorm:"column:local_epc_id;type:int;default:0" json:"LOCAL_EPC_ID"` - RemoteVPCID int `gorm:"column:remote_epc_id;type:int;default:0" json:"REMOTE_EPC_ID"` - LocalRegionID int `gorm:"column:local_region_id;type:int;default:0" json:"LOCAL_REGION_ID"` - RemoteRegionID int `gorm:"column:remote_region_id;type:int;default:0" json:"REMOTE_REGION_ID"` - CreateMethod int `gorm:"column:create_method;type:int;default:0" json:"CREATE_METHOD"` // 0.learning 1.user_defined - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` -} - -type CEN struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Label string `gorm:"column:label;type:char(64);default:''" json:"LABEL"` - Alias string `gorm:"column:alias;type:char(64);default:''" json:"ALIAS"` - VPCIDs string `gorm:"column:epc_ids;type:text;default:''" json:"EPC_IDS"` // separated by , - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` -} - -func (CEN) TableName() string { - return "cen" -} - -type RDSInstance struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Label string `gorm:"column:label;type:char(64);default:''" json:"LABEL"` - State int `gorm:"column:state;type:tinyint(1);not null;default:0" json:"STATE"` // 0. Unknown 1. Running 2. Recovering - Domain string `gorm:"column:domain;type:char(64);default:''" json:"DOMAIN"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` - VPCID int `gorm:"column:epc_id;type:int;default:0" json:"EPC_ID"` - Type int `gorm:"column:type;type:int;default:0" json:"TYPE"` // 0. Unknown 1. MySQL 2. SqlServer 3. PPAS 4. PostgreSQL 5. MariaDB - Version string `gorm:"column:version;type:char(64);default:''" json:"VERSION"` - Series int `gorm:"column:series;type:tinyint(1);not null;default:0" json:"SERIES"` // 0. Unknown 1. basic 2. HA - Model int `gorm:"column:model;type:tinyint(1);not null;default:0" json:"MODEL"` // 0. Unknown 1. Primary 2. Readonly 3. Temporary 4. Disaster recovery 5. share - UID string `gorm:"column:uid;type:char(64);default:''" json:"UID"` -} - -func (RDSInstance) TableName() string { - return "rds_instance" -} - -type RedisInstance struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Label string `gorm:"column:label;type:char(64);default:''" json:"LABEL"` - State int `gorm:"column:state;type:tinyint(1);not null;default:0" json:"STATE"` // 0. Unknown 1. Running 2. Recovering - Domain string `gorm:"column:domain;type:char(64);default:''" json:"DOMAIN"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` - VPCID int `gorm:"column:epc_id;type:int;default:0" json:"EPC_ID"` - Version string `gorm:"column:version;type:char(64);default:''" json:"VERSION"` - InternalHost string `gorm:"column:internal_host;type:varchar(128);default:''" json:"INTERNAL_HOST"` - PublicHost string `gorm:"column:public_host;type:varchar(128);default:''" json:"PUBLIC_HOST"` - UID string `gorm:"column:uid;type:char(64);default:''" json:"UID"` -} - -type VIP struct { - Base `gorm:"embedded"` - IP string `gorm:"column:ip;type:char(64);default:''" json:"IP"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` - VTapID uint32 `gorm:"column:vtap_id;type:int;not null;default:0" json:"VTAP_ID"` -} - -func (VIP) TableName() string { - return "vip" -} - -type PodCluster struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - ClusterName string `gorm:"column:cluster_name;type:varchar(256);default:''" json:"CLUSTER_NAME"` - Version string `gorm:"column:version;type:varchar(256);default:''" json:"VERSION"` - VPCID int `gorm:"column:epc_id;type:int;default:null" json:"VPC_ID"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` -} - -type PodNamespace struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Alias string `gorm:"column:alias;type:char(64);default:''" json:"ALIAS"` - CloudTags string `gorm:"column:cloud_tags;type:text;default:''" json:"CLOUD_TAGS"` // separated by , - PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" json:"POD_CLUSTER_ID"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` -} - -type PodNode struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Alias string `gorm:"column:alias;type:char(64);default:''" json:"ALIAS"` - Type int `gorm:"column:type;type:int;default:null" json:"TYPE"` // 1: Master 2: Node - ServerType int `gorm:"column:server_type;type:int;default:null" json:"SERVER_TYPE"` // 1: Host 2: VM - State int `gorm:"column:state;type:int;default:1" json:"STATE"` // 0: Exception 1: Normal - IP string `gorm:"column:ip;type:char(64);default:''" json:"IP"` - VCPUNum int `gorm:"column:vcpu_num;type:int;default:0" json:"VCPU_NUM"` - MemTotal int `gorm:"column:mem_total;type:int;default:0" json:"MEM_TOTAL"` // unit: M - PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" json:"POD_CLUSTER_ID"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` - VPCID int `gorm:"column:epc_id;type:int;default:null" json:"VPC_ID"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` -} - -type PodIngress struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Alias string `gorm:"column:alias;type:char(64);default:''" json:"ALIAS"` - PodNamespaceID int `gorm:"column:pod_namespace_id;type:int;default:null" json:"POD_NAMESPACE_ID"` - PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" json:"POD_CLUSTER_ID"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` -} - -type PodIngressRule struct { - Base `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Protocol string `gorm:"column:protocol;type:char(64);default:''" json:"PROTOCOL"` - Host string `gorm:"column:host;type:text;default:''" json:"HOST"` - PodIngressID int `gorm:"column:pod_ingress_id;type:int;default:null" json:"POD_INGRESS_ID"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` -} - -type PodIngressRuleBackend struct { - Base `gorm:"embedded"` - Path string `gorm:"column:path;type:text;default:''" json:"PATH"` - Port int `gorm:"column:port;type:int;default:null" json:"PORT"` - PodServiceID int `gorm:"column:pod_service_id;type:int;default:null" json:"POD_SERVICE_ID"` - PodIngressRuleID int `gorm:"column:pod_ingress_rule_id;type:int;default:null" json:"POD_INGRESS_RULE_ID"` - PodIngressID int `gorm:"column:pod_ingress_id;type:int;default:null" json:"POD_INGRESS_ID"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` -} - -type PodService struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Label string `gorm:"column:label;type:text;default:''" json:"LABEL"` // separated by , - Annotation string `gorm:"column:annotation;type:text;default:''" json:"ANNOTATION"` // separated by , - Alias string `gorm:"column:alias;type:char(64);default:''" json:"ALIAS"` - Type int `gorm:"column:type;type:int;default:null" json:"TYPE"` // 1: ClusterIP 2: NodePort - Selector string `gorm:"column:selector;type:text;default:''" json:"SELECTOR"` // separated by , - ServiceClusterIP string `gorm:"column:service_cluster_ip;type:char(64);default:''" json:"SERVICE_CLUSTER_IP"` - PodIngressID int `gorm:"column:pod_ingress_id;type:int;default:null" json:"POD_INGRESS_ID"` - PodNamespaceID int `gorm:"column:pod_namespace_id;type:int;default:null" json:"POD_NAMESPACE_ID"` - PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" json:"POD_CLUSTER_ID"` - VPCID int `gorm:"column:epc_id;type:int;default:null" json:"VPC_ID"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` -} - -type PodServicePort struct { - Base `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Protocol string `gorm:"column:protocol;type:char(64);default:''" json:"PROTOCOL"` - Port int `gorm:"column:port;type:int;default:null" json:"PORT"` - TargetPort int `gorm:"column:target_port;type:int;default:null" json:"TARGET_PORT"` - NodePort int `gorm:"column:node_port;type:int;default:null" json:"NODE_PORT"` - PodServiceID int `gorm:"column:pod_service_id;type:int;default:null" json:"POD_SERVICE_ID"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` -} - -type PodGroup struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Alias string `gorm:"column:alias;type:char(64);default:''" json:"ALIAS"` - Type int `gorm:"column:type;type:int;default:null" json:"TYPE"` // 1: Deployment 2: StatefulSet 3: ReplicationController - PodNum int `gorm:"column:pod_num;type:int;default:1" json:"POD_NUM"` - Label string `gorm:"column:label;type:text;default:''" json:"LABEL"` // separated by , - PodNamespaceID int `gorm:"column:pod_namespace_id;type:int;default:null" json:"POD_NAMESPACE_ID"` - PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" json:"POD_CLUSTER_ID"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` -} - -type PodGroupPort struct { - Base `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Protocol string `gorm:"column:protocol;type:char(64);default:''" json:"PROTOCOL"` - Port int `gorm:"column:port;type:int;default:null" json:"PORT"` - PodGroupID int `gorm:"column:pod_group_id;type:int;default:null" json:"POD_GROUP_ID"` - PodServiceID int `gorm:"column:pod_service_id;type:int;default:null" json:"POD_SERVICE_ID"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` -} - -type PodReplicaSet struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Alias string `gorm:"column:alias;type:char(64);default:''" json:"ALIAS"` - Label string `gorm:"column:label;type:text;default:''" json:"LABEL"` // separated by , - PodNum int `gorm:"column:pod_num;type:int;default:1" json:"POD_NUM"` - PodGroupID int `gorm:"column:pod_group_id;type:int;default:null" json:"POD_GROUP_ID"` - PodNamespaceID int `gorm:"column:pod_namespace_id;type:int;default:null" json:"POD_NAMESPACE_ID"` - PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" json:"POD_CLUSTER_ID"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` -} - -func (PodReplicaSet) TableName() string { - return "pod_rs" -} - -type PrometheusTarget struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Instance string `gorm:"column:instance;type:varchar(255);default:''" json:"INSTANCE"` - Job string `gorm:"column:job;type:varchar(255);default:''" json:"JOB"` - ScrapeURL string `gorm:"column:scrape_url;type:varchar(2083);default:''" json:"SCRAPE_URL"` - OtherLabels string `gorm:"column:other_labels;type:text;default:''" json:"OTHER_LABELS"` // separated by , - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` - PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" json:"POD_CLUSTER_ID"` - CreateMethod int `gorm:"column:create_method;type:tinyint(1);default:1" json:"CREATE_METHOD"` -} - -func (PrometheusTarget) TableName() string { - return "prometheus_target" -} - -type Pod struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` - Name string `gorm:"column:name;type:varchar(256);default:''" json:"NAME"` - Alias string `gorm:"column:alias;type:char(64);default:''" json:"ALIAS"` - State int `gorm:"column:state;type:int;not null" json:"STATE"` // 0.Exception 1.Running - Label string `gorm:"column:label;type:text;default:''" json:"LABEL"` // separated by , - Annotation string `gorm:"column:annotation;type:text;default:''" json:"ANNOTATION"` // separated by , - ENV string `gorm:"column:env;type:text;default:''" json:"ENV"` // separated by , - ContainerIDs string `gorm:"column:container_ids;type:text;default:''" json:"CONTAINER_IDS"` // separated by , - PodReplicaSetID int `gorm:"column:pod_rs_id;type:int;default:null" json:"POD_RS_ID"` - PodGroupID int `gorm:"column:pod_group_id;type:int;default:null" json:"POD_GROUP_ID"` - PodNamespaceID int `gorm:"column:pod_namespace_id;type:int;default:null" json:"POD_NAMESPACE_ID"` - PodNodeID int `gorm:"column:pod_node_id;type:int;default:null" json:"POD_NODE_ID"` - PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" json:"POD_CLUSTER_ID"` - VPCID int `gorm:"column:epc_id;type:int;default:null" json:"VPC_ID"` - AZ string `gorm:"column:az;type:char(64);default:''" json:"AZ"` - Region string `gorm:"column:region;type:char(64);default:''" json:"REGION"` - SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` - Domain string `gorm:"column:domain;type:char(64);not null" json:"DOMAIN"` -} - type Business struct { ID int `gorm:"primaryKey;column:id;type:int;not null" json:"ID"` Name string `gorm:"column:name;type:char(64);default:null" json:"NAME"` diff --git a/server/controller/db/mysql/platform_rsc_model.go b/server/controller/db/mysql/platform_rsc_model.go new file mode 100644 index 00000000000..076d8187532 --- /dev/null +++ b/server/controller/db/mysql/platform_rsc_model.go @@ -0,0 +1,827 @@ +/** + * Copyright (c) 2023 Yunshan Networks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mysql + +import ( + "time" + + "gorm.io/gorm" +) + +type ResourceEvent struct { + ID int `gorm:"primaryKey;autoIncrement;unique;column:id;type:int;not null" json:"ID"` + Domain string `gorm:"column:domain;type:char(64);default:''" json:"DOMAIN"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" json:"SUB_DOMAIN"` + ResourceLcuuid string `gorm:"column:resource_lcuuid;type:char(64);default:''" json:"RESOURCE_LCUUID"` + Content string `gorm:"column:content;type:text" json:"CONTENT"` + CreatedAt time.Time `gorm:"autoCreateTime;column:created_at;type:datetime" json:"CREATED_AT"` +} + +type DomainAdditionalResource struct { + ID int `gorm:"primaryKey;autoIncrement;unique;column:id;type:int;not null" json:"ID"` + Domain string `gorm:"column:domain;type:char(64);default:''" json:"DOMAIN"` + Content string `gorm:"column:content;type:longtext" json:"CONTENT"` + CompressedContent compressedBytes `gorm:"column:compressed_content;type:longblob" json:"COMPRESSED_CONTENT"` + CreatedAt time.Time `gorm:"autoCreateTime;column:created_at;type:datetime" json:"CREATED_AT"` +} + +type Base struct { + ID int `gorm:"primaryKey;autoIncrement;unique;column:id;type:int;not null" mapstructure:"ID"` + Lcuuid string `gorm:"unique;column:lcuuid;type:char(64)" mapstructure:"LCUUID"` + // TODO add CreatedAt/UpdatedAt/DeletedAt +} + +func (b Base) GetID() int { + return b.ID +} + +func (b *Base) SetID(id int) { + b.ID = id +} + +func (b Base) GetLcuuid() string { + return b.Lcuuid +} + +type OperatedTime struct { + CreatedAt time.Time `gorm:"autoCreateTime;column:created_at;type:datetime" mapstructure:"CREATED_AT"` + UpdatedAt time.Time `gorm:"autoUpdateTime;column:updated_at;type:datetime" mapstructure:"UPDATED_AT"` +} + +type SoftDeleteBase struct { + OperatedTime + DeletedAt gorm.DeletedAt `gorm:"column:deleted_at;type:datetime;default:null" mapstructure:"DELETED_AT"` +} + +type Process struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + VTapID uint32 `gorm:"column:vtap_id;type:int;not null;default:0" mapstructure:"VTAP_ID"` + PID uint64 `gorm:"column:pid;type:int;not null;default:0" mapstructure:"PID"` + ProcessName string `gorm:"column:process_name;type:varchar(256);default:''" mapstructure:"PROCESS_NAME"` + CommandLine string `gorm:"column:command_line;type:text" mapstructure:"COMMAND_LINE"` + UserName string `gorm:"column:user_name;type:varchar(256);default:''" mapstructure:"USER_NAME"` + StartTime time.Time `gorm:"autoCreateTime;column:start_time;type:datetime" mapstructure:"START_TIME"` + OSAPPTags string `gorm:"column:os_app_tags;type:text" mapstructure:"OS_APP_TAGS"` + ContainerID string `gorm:"column:container_id;type:char(64);default:''" mapstructure:"CONTAINER_ID"` + NetnsID uint32 `gorm:"column:netns_id;type:int unsigned;default:0" mapstructure:"NETNS_ID"` // used to associate processes with cloud and container resources + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` + Domain string `gorm:"column:domain;type:char(64);default:''" mapstructure:"DOMAIN"` +} + +type Domain struct { + Base `gorm:"embedded"` + OperatedTime `gorm:"embedded"` + SyncedAt *time.Time `gorm:"column:synced_at" mapstructure:"SYNCED_AT"` + Name string `gorm:"column:name;type:varchar(64)" mapstructure:"NAME"` + IconID int `gorm:"column:icon_id;type:int" mapstructure:"ICON_ID"` + DisplayName string `gorm:"column:display_name;type:varchar(64);default:''" mapstructure:"DISPLAY_NAME"` + ClusterID string `gorm:"column:cluster_id;type:char(64)" mapstructure:"CLUSTER_ID"` + Type int `gorm:"column:type;type:int;default:0" mapstructure:"TYPE"` // 1.openstack 2.vsphere 3.nsp 4.tencent 5.filereader 6.aws 7.pingan 8.zstack 9.aliyun 10.huawei prv 11.k8s 12.simulation 13.huawei 14.qingcloud 15.qingcloud_private 16.F5 17.CMB_CMDB 18.azure 19.apsara_stack 20.tencent_tce 21.qingcloud_k8s 22.kingsoft_private 23.genesis 24.microsoft_acs 25.baidu_bce + Config string `gorm:"column:config;type:text" mapstructure:"CONFIG"` + ErrorMsg string `gorm:"column:error_msg;type:text" mapstructure:"ERROR_MSG"` + Enabled int `gorm:"column:enabled;type:int;not null;default:1" mapstructure:"ENABLED"` // 0.false 1.true + State int `gorm:"column:state;type:int;not null;default:1" mapstructure:"STATE"` // 1.normal 2.deleting 3.exception + ControllerIP string `gorm:"column:controller_ip;type:char(64)" mapstructure:"CONTROLLER_IP"` +} + +// TODO 最终可以与cloud模块命名统一,Domain -> DomainLcuuid + +type SubDomain struct { + Base `gorm:"embedded"` + OperatedTime `gorm:"embedded"` + SyncedAt *time.Time `gorm:"column:synced_at" mapstructure:"SYNCED_AT"` + Domain string `gorm:"column:domain;type:char(64);default:''" mapstructure:"DOMAIN"` + Name string `gorm:"column:name;type:varchar(64);default:''" mapstructure:"NAME"` + DisplayName string `gorm:"column:display_name;type:varchar(64);default:''" mapstructure:"DISPLAY_NAME"` + CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined + ClusterID string `gorm:"column:cluster_id;type:char(64);default:''" mapstructure:"CLUSTER_ID"` + Config string `gorm:"column:config;type:text;default:''" mapstructure:"CONFIG"` + ErrorMsg string `gorm:"column:error_msg;type:text;default:''" mapstructure:"ERROR_MSG"` + Enabled int `gorm:"column:enabled;type:int;not null;default:1" mapstructure:"ENABLED"` // 0.false 1.true + State int `gorm:"column:state;type:int;not null;default:1" mapstructure:"STATE"` // 1.normal 2.deleting 3.exception +} + +type Region struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(64);default:''" mapstructure:"NAME"` + CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined + Label string `gorm:"column:label;type:varchar(64);default:''" mapstructure:"LABEL"` + Longitude float64 `gorm:"column:longitude;type:double(7,4);default:null" mapstructure:"LONGITUDE"` + Latitude float64 `gorm:"column:latitude;type:double(7,4);default:null" mapstructure:"LATITUDE"` +} + +type AZ struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(64);default:''" mapstructure:"NAME"` + CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined + Label string `gorm:"column:label;type:varchar(64);default:''" mapstructure:"LABEL"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + Domain string `gorm:"column:domain;type:char(64);default:''" mapstructure:"DOMAIN"` +} + +func (AZ) TableName() string { + return "az" +} + +type Host struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Type int `gorm:"column:type;type:int" mapstructure:"TYPE"` // 1.Server 3.Gateway 4.DFI + State int `gorm:"column:state;type:int" mapstructure:"STATE"` // 0.Temp 1.Creating 2.Complete 3.Modifying 4.Exception + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` + Description string `gorm:"column:description;type:varchar(256);default:''" mapstructure:"DESCRIPTION"` + IP string `gorm:"column:ip;type:char(64);default:''" mapstructure:"IP"` + HType int `gorm:"column:htype;type:int" mapstructure:"HTYPE"` // 1. Xen host 2. VMware host 3. KVM host 4. Public cloud host 5. Hyper-V + CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined + UserName string `gorm:"column:user_name;type:varchar(64);default:''" mapstructure:"USER_NAME"` + UserPasswd string `gorm:"column:user_passwd;type:varchar(64);default:''" mapstructure:"USER_PASSWD"` + VCPUNum int `gorm:"column:vcpu_num;type:int;default:0" mapstructure:"VCPU_NUM"` + MemTotal int `gorm:"column:mem_total;type:int;default:0" mapstructure:"MEM_TOTAL"` // unit: M + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + Domain string `gorm:"column:domain;type:char(64);default:''" mapstructure:"DOMAIN"` + SyncedAt time.Time `gorm:"column:synced_at;type:datetime;not null;default:CURRENT_TIMESTAMP" mapstructure:"SYNCED_AT"` + ExtraInfo string `gorm:"column:extra_info;type:text;default:''" mapstructure:"EXTRA_INFO"` +} + +func (Host) TableName() string { + return "host_device" +} + +type VM struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + State int `gorm:"index:state_server_index;column:state;type:int;not null" mapstructure:"STATE"` // 0.Temp 1.Creating 2.Created 3.To run 4.Running 5.To suspend 6.Suspended 7.To resume 8. To stop 9.Stopped 10.Modifing 11.Exception 12.Destroying + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` + Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` + CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined + HType int `gorm:"column:htype;type:int;default:1" mapstructure:"HTYPE"` // 1.vm-c 2.bm-c 3.vm-n 4.bm-n 5.vm-s 6.bm-s + LaunchServer string `gorm:"index:state_server_index;column:launch_server;type:char(64);default:''" mapstructure:"LAUNCH_SERVER"` + CloudTags string `gorm:"column:cloud_tags;type:text;default:''" mapstructure:"CLOUD_TAGS"` // separated by , + VPCID int `gorm:"column:epc_id;type:int;default:0" mapstructure:"EPC_ID"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + UID string `gorm:"column:uid;type:char(64);default:''" mapstructure:"UID"` +} + +func (VM) TableName() string { + return "vm" +} + +type VMPodNodeConnection struct { + Base `gorm:"embedded"` + VMID int `gorm:"column:vm_id;type:int;default:null" mapstructure:"VM_ID"` + PodNodeID int `gorm:"column:pod_node_id;type:int;default:null" mapstructure:"POD_NODE_ID"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` +} + +func (VMPodNodeConnection) TableName() string { + return "vm_pod_node_connection" +} + +type VMSecurityGroup struct { + Base `gorm:"embedded"` + SecurityGroupID int `gorm:"column:sg_id;type:int;not null" mapstructure:"SG_ID"` + VMID int `gorm:"column:vm_id;type:int;not null" mapstructure:"VM_ID"` + Priority int `gorm:"column:priority;type:int;not null" mapstructure:"PRIORITY"` +} + +func (VMSecurityGroup) TableName() string { + return "vm_security_group" +} + +type Contact struct { + Base `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined + Mobile string `gorm:"column:mobile;type:char(13);default:''" mapstructure:"MOBILE"` + Email string `gorm:"column:email;type:varchar(128);default:''" mapstructure:"EMAIL"` + Company string `gorm:"column:company;type:varchar(128);default:''" mapstructure:"COMPANY"` + PushEmail string `gorm:"column:push_email;type:text;default:null" mapstructure:"PUSH_EMAIL"` + Domain string `gorm:"column:domain;type:char(64);default:''" mapstructure:"DOMAIN"` + AlarmPush int `gorm:"column:alarm_push;type:int;default:0" mapstructure:"ALARM_PUSH"` + ReportPush int `gorm:"column:report_push;type:int;default:0" mapstructure:"REPORT_PUSH"` + Deleted int `gorm:"column:deleted;type:int;default:0" mapstructure:"DELETED"` + CreatedAt time.Time `gorm:"column:created_at;type:datetime;not null;default:CURRENT_TIMESTAMP" mapstructure:"CREATED_AT"` + UpdatedAt time.Time `gorm:"column:updated_at;type:datetime;default:null" mapstructure:"UPDATED_AT"` +} + +type VPCContact struct { + Base `gorm:"embedded"` + CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined + VPCID int `gorm:"column:epc_id;type:int;default:0" mapstructure:"VPC_ID"` + ContactID int `gorm:"column:contact_id;type:int;default:0" mapstructure:"CONTACT_ID"` +} + +func (VPCContact) TableName() string { + return "epc_contact" +} + +type VPC struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined + Label string `gorm:"column:label;type:varchar(64);default:''" mapstructure:"LABEL"` + Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` + Domain string `gorm:"column:domain;type:char(64);default:''" mapstructure:"DOMAIN"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` // TODO delete in future + TunnelID int `gorm:"column:tunnel_id;type:int;default:0" mapstructure:"TUNNEL_ID"` + Mode int `gorm:"column:mode;type:int;default:2" mapstructure:"MODE"` // 1:route, 2:transparent + CIDR string `gorm:"column:cidr;type:char(64);default:''" mapstructure:"CIDR"` + UID string `gorm:"column:uid;type:char(64);default:''" mapstructure:"UID"` +} + +func (VPC) TableName() string { + return "epc" +} + +type Network struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + State int `gorm:"column:state;type:int;not null" mapstructure:"STATE"` // 0.Temp 1.Creating 2.Created 3.Exception 4.Modifing 5.Destroying 6.Destroyed + NetType int `gorm:"column:net_type;type:int;default:4" mapstructure:"NET_TYPE"` // 1.CTRL 2.SERVICE 3.WAN 4.LAN + Name string `gorm:"column:name;type:varchar(256);not null" mapstructure:"NAME"` + CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined + Label string `gorm:"column:label;type:varchar(64);default:''" mapstructure:"LABEL"` + Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` + Description string `gorm:"column:description;type:varchar(256);default:''" mapstructure:"DESCRIPTION"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` + ISP int `gorm:"column:isp;type:int;default:0" mapstructure:"ISP"` + VPCID int `gorm:"column:epc_id;type:int;default:0" mapstructure:"VPC_ID"` + SegmentationID int `gorm:"column:segmentation_id;type:int;default:0" mapstructure:"SEGMENTATION_ID"` + TunnelID int `gorm:"column:tunnel_id;type:int;default:0" mapstructure:"TUNNEL_ID"` + Shared bool `gorm:"column:shared;type:int;default:0" mapstructure:"SHARED"` + Topped int `gorm:"column:topped;type:int;default:0" mapstructure:"TOPPED"` + IsVIP int `gorm:"column:is_vip;type:int;default:0" mapstructure:"IS_VIP"` +} + +func (Network) TableName() string { + return "vl2" +} + +type Subnet struct { + Base `gorm:"embedded"` + Prefix string `gorm:"column:prefix;type:char(64);default:''" mapstructure:"PREFIX"` + Netmask string `gorm:"column:netmask;type:char(64);default:''" mapstructure:"NETMASK"` + NetworkID int `gorm:"column:vl2id;type:int;default:null" mapstructure:"VL2ID"` + NetIndex int `gorm:"column:net_index;type:int;default:0" mapstructure:"NET_INDEX"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Label string `gorm:"column:label;type:varchar(64);default:''" mapstructure:"LABEL"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` +} + +func (Subnet) TableName() string { + return "vl2_net" +} + +type VRouter struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + State int `gorm:"index:state_server_index;column:state;type:int;not null" mapstructure:"STATE"` // 0.Temp 1.Creating 2.Created 3.Exception 4.Modifing 5.Destroying 6.To run 7.Running 8.To stop 9.Stopped + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` + Description string `gorm:"column:description;type:varchar(256);default:''" mapstructure:"DESCRIPTION"` + VPCID int `gorm:"column:epc_id;type:int;default:0" mapstructure:"EPC_ID"` + GWLaunchServer string `gorm:"index:state_server_index;column:gw_launch_server;type:char(64);default:''" mapstructure:"GW_LAUNCH_SERVER"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` // TODO delete in future +} + +func (VRouter) TableName() string { + return "vnet" +} + +type RoutingTable struct { + Base `gorm:"embedded"` + VRouterID int `gorm:"column:vnet_id;type:int;default:null" mapstructure:"VNET_ID"` + Destination string `gorm:"column:destination;type:text;default:''" mapstructure:"DESTINATION"` + NexthopType string `gorm:"column:nexthop_type;type:text;default:''" mapstructure:"NEXTHOP_TYPE"` + Nexthop string `gorm:"column:nexthop;type:text;default:''" mapstructure:"NEXTHOP"` +} + +type DHCPPort struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` + VPCID int `gorm:"column:epc_id;type:int;default:0" mapstructure:"VPC_ID"` +} + +func (DHCPPort) TableName() string { + return "dhcp_port" +} + +type VInterface struct { + Base `gorm:"embedded"` + Name string `gorm:"column:name;type:char(64);default:''" mapstructure:"NAME"` + Index int `gorm:"column:ifindex;type:int;not null" mapstructure:"IFINDEX"` + State int `gorm:"column:state;type:int;not null" mapstructure:"STATE"` // 1. Attached 2.Detached 3.Exception + CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined + Type int `gorm:"column:iftype;type:int;default:0" mapstructure:"IFTYPE"` // 0.Unknown 1.Control 2.Service 3.WAN 4.LAN 5.Trunk 6.Tap 7.Tool + Mac string `gorm:"index:mac_index;column:mac;type:char(32);default:''" mapstructure:"MAC"` + VMac string `gorm:"column:vmac;type:char(32);default:''" mapstructure:"VMAC"` + TapMac string `gorm:"column:tap_mac;type:char(32);default:''" mapstructure:"TAP_MAC"` + NetworkID int `gorm:"column:subnetid;type:int;default:0" mapstructure:"SUBNETID"` // vl2 id + VlanTag int `gorm:"column:vlantag;type:int;default:0" mapstructure:"VLANTAG"` + DeviceType int `gorm:"column:devicetype;type:int;default:null" mapstructure:"DEVICETYPE"` // Type 0.unknown 1.vm 2.vgw 3.third-party-device 4.vmwaf 5.NSP-vgateway 6.host-device 7.network-device 9.DHCP-port 10.pod 11.pod_service 12. redis_instance 13. rds_instance 14. pod_node 15. load_balance 16. nat_gateway + DeviceID int `gorm:"column:deviceid;type:int;default:null" mapstructure:"DEVICEID"` // unknown: Senseless ID, vm: vm ID, vgw/NSP-vgateway: vnet ID, third-party-device: third_party_device ID, vmwaf: vmwaf ID, host-device: host_device ID, network-device: network_device ID + NetnsID uint32 `gorm:"column:netns_id;type:int unsigned;default:0" mapstructure:"NETNS_ID"` // used to associate processes with cloud and container resources + VtapID uint32 `gorm:"column:vtap_id;type:int;default:0" mapstructure:"VTAP_ID"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + CreatedAt time.Time `gorm:"column:created_at;type:timestamp;not null;default:CURRENT_TIMESTAMP" mapstructure:"CREATED_AT"` + UpdatedAt time.Time `gorm:"column:updated_at;type:timestamp;not null;default:CURRENT_TIMESTAMP" mapstructure:"UPDATED_AT"` +} + +func (VInterface) TableName() string { + return "vinterface" +} + +type LANIP struct { // TODO 添加region字段 + Base `gorm:"embedded"` + IP string `gorm:"column:ip;type:char(64);default:''" mapstructure:"IP"` + Netmask string `gorm:"column:netmask;type:char(64);default:''" mapstructure:"NETMASK"` + Gateway string `gorm:"column:gateway;type:char(64);default:''" mapstructure:"GATEWAY"` + CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined + NetworkID int `gorm:"column:vl2id;type:int;default:null" mapstructure:"VL2ID"` + NetIndex int `gorm:"column:net_index;type:int;default:0" mapstructure:"NET_INDEX"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` + VInterfaceID int `gorm:"column:vifid;type:int;default:null" mapstructure:"VINTERFACE_ID"` + SubnetID int `gorm:"column:vl2_net_id;type:int;default:0" mapstructure:"SUBNET_ID"` + ISP int `gorm:"column:isp;type:int;default:0" mapstructure:"ISP"` // Used for multi-ISP access + CreatedAt time.Time `gorm:"column:created_at;type:timestamp;not null;default:CURRENT_TIMESTAMP" mapstructure:"CREATED_AT"` + UpdatedAt time.Time `gorm:"column:updated_at;type:timestamp;not null;default:CURRENT_TIMESTAMP" mapstructure:"UPDATED_AT"` +} + +func (LANIP) TableName() string { + return "vinterface_ip" +} + +type WANIP struct { + Base `gorm:"embedded"` + IP string `gorm:"column:ip;type:char(64);default:''" mapstructure:"IP"` + Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` + Netmask int `gorm:"column:netmask;type:int;default:null" mapstructure:"NETMASK"` + Gateway string `gorm:"column:gateway;type:char(64);default:''" mapstructure:"GATEWAY"` + CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined + ISP int `gorm:"column:isp;type:int;default:null" mapstructure:"ISP"` + VInterfaceID int `gorm:"column:vifid;type:int;default:0" mapstructure:"VINTERFACE_ID"` + SubnetID int `gorm:"column:vl2_net_id;type:int;default:0" mapstructure:"SUBNET_ID"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + CreatedAt time.Time `gorm:"column:created_at;type:timestamp;not null;default:CURRENT_TIMESTAMP" mapstructure:"CREATED_AT"` + UpdatedAt time.Time `gorm:"column:updated_at;type:timestamp;not null;default:CURRENT_TIMESTAMP" mapstructure:"UPDATED_AT"` +} + +func (WANIP) TableName() string { + return "ip_resource" +} + +type FloatingIP struct { + Base `gorm:"embedded"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + VPCID int `gorm:"column:epc_id;type:int;default:0" mapstructure:"VPC_ID"` + NetworkID int `gorm:"column:vl2_id;type:int;default:null" mapstructure:"VL2_ID"` // TODO json字段是否能修改,需返回给前端? + VMID int `gorm:"column:vm_id;type:int;default:null" mapstructure:"VM_ID"` + IP string `gorm:"column:ip;type:char(64);default:''" mapstructure:"IP"` +} + +func (FloatingIP) TableName() string { + return "floatingip" +} + +type SecurityGroup struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Label string `gorm:"column:label;type:varchar(64);default:''" mapstructure:"LABEL"` + Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` + VPCID int `gorm:"column:epc_id;type:int;default:0" mapstructure:"EPC_ID"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + Topped int `gorm:"column:topped;type:int;default:0" mapstructure:"TOPPED"` +} + +type SecurityGroupRule struct { + Base `gorm:"embedded"` + SecurityGroupID int `gorm:"column:sg_id;type:int;not null" mapstructure:"SG_ID"` + Direction int `gorm:"column:direction;type:tinyint(1);not null;default:0" mapstructure:"DIRECTION"` // 0.Unknow 1.Ingress 2.Egress + Protocol string `gorm:"column:protocol;type:char(64);default:''" mapstructure:"PROTOCOL"` + EtherType int `gorm:"column:ethertype;type:tinyint(1);not null;default:0" mapstructure:"ETHERTYPE"` // 0.Unknow 1.IPv4 2.IPv6 + LocalPortRange string `gorm:"column:local_port_range;type:text;default:''" mapstructure:"LOCAL_PORT_RANGE"` + RemotePortRange string `gorm:"column:remote_port_range;type:text;default:''" mapstructure:"REMOTE_PORT_RANGE"` + Local string `gorm:"column:local;type:text;default:''" mapstructure:"LOCAL"` + Remote string `gorm:"column:remote;type:text;default:''" mapstructure:"REMOTE"` + Priority int `gorm:"column:priority;type:int;not null" mapstructure:"PRIORITY"` + Action int `gorm:"column:action;type:tinyint(1);not null;default:0" mapstructure:"ACTION"` // 0.Unknow 1.Accept 2.Drop +} + +type NATGateway struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` + FloatingIPs string `gorm:"column:floating_ips;type:text;default:''" mapstructure:"FLOATING_IPS"` // separated by , + VPCID int `gorm:"column:epc_id;type:int;default:0" mapstructure:"EPC_ID"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` // TODO delete in future + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` + UID string `gorm:"column:uid;type:char(64);default:''" mapstructure:"UID"` +} + +func (NATGateway) TableName() string { + return "nat_gateway" +} + +type NATRule struct { + Base `gorm:"embedded"` + NATGatewayID int `gorm:"column:nat_id;type:int;default:0" mapstructure:"NAT_ID"` + Type string `gorm:"column:type;type:char(16);default:''" mapstructure:"TYPE"` + Protocol string `gorm:"column:protocol;type:char(64);default:''" mapstructure:"PROTOCOL"` + FloatingIP string `gorm:"column:floating_ip;type:char(64);default:''" mapstructure:"FLOATING_IP"` + FloatingIPPort int `gorm:"column:floating_ip_port;type:int;default:null" mapstructure:"FLOATING_IP_PORT"` + FixedIP string `gorm:"column:fixed_ip;type:char(64);default:''" mapstructure:"FIXED_IP"` + FixedIPPort int `gorm:"column:fixed_ip_port;type:int;default:null" mapstructure:"FIXED_IP_PORT"` + VInterfaceID int `gorm:"column:port_id;type:int;default:null" mapstructure:"PORT_ID"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` +} + +func (NATRule) TableName() string { + return "nat_rule" +} + +type NATVMConnection struct { + Base `gorm:"embedded"` + NATGatewayID int `gorm:"column:nat_id;type:int;default:null" mapstructure:"NAT_ID"` + VMID int `gorm:"column:vm_id;type:int;default:null" mapstructure:"VM_ID"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` +} + +func (NATVMConnection) TableName() string { + return "nat_vm_connection" +} + +type LB struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` + Model int `gorm:"column:model;type:int;default:0" mapstructure:"MODEL"` // 1.Internal 2.External + VIP string `gorm:"column:vip;type:text;default:''" mapstructure:"VIP"` + VPCID int `gorm:"column:epc_id;type:int;default:0" mapstructure:"EPC_ID"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` // TODO delete in future + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` + UID string `gorm:"column:uid;type:char(64);default:''" mapstructure:"UID"` +} + +func (LB) TableName() string { + return "lb" +} + +type LBListener struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + LBID int `gorm:"column:lb_id;type:int;default:0" mapstructure:"LB_ID"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + IPs string `gorm:"column:ips;type:text;default:''" mapstructure:"IPS"` // separated by , + SNATIPs string `gorm:"column:snat_ips;type:text;default:''" mapstructure:"SNAT_IPS"` // separated by , + Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` + Port int `gorm:"column:port;type:int;default:null" mapstructure:"PORT"` + Protocol string `gorm:"column:protocol;type:char(64);default:''" mapstructure:"PROTOCOL"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` +} + +func (LBListener) TableName() string { + return "lb_listener" +} + +type LBTargetServer struct { + Base `gorm:"embedded"` + LBID int `gorm:"column:lb_id;type:int;default:0" mapstructure:"LB_ID"` + LBListenerID int `gorm:"column:lb_listener_id;type:int;default:0" mapstructure:"LB_LISTENER_ID"` + VPCID int `gorm:"column:epc_id;type:int;default:0" mapstructure:"EPC_ID"` + Type int `gorm:"column:type;type:int;default:0" mapstructure:"TYPE"` // 1.VM 2.IP + IP string `gorm:"column:ip;type:char(64);default:''" mapstructure:"IP"` + VMID int `gorm:"column:vm_id;type:int;default:0" mapstructure:"VM_ID"` + Port int `gorm:"column:port;type:int;default:null" mapstructure:"PORT"` + Protocol string `gorm:"column:protocol;type:char(64);default:''" mapstructure:"PROTOCOL"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` +} + +func (LBTargetServer) TableName() string { + return "lb_target_server" +} + +type LBVMConnection struct { + Base `gorm:"embedded"` + LBID int `gorm:"column:lb_id;type:int;default:null" mapstructure:"LB_ID"` + VMID int `gorm:"column:vm_id;type:int;default:null" mapstructure:"VM_ID"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` +} + +func (LBVMConnection) TableName() string { + return "lb_vm_connection" +} + +type PeerConnection struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` + LocalVPCID int `gorm:"column:local_epc_id;type:int;default:0" mapstructure:"LOCAL_EPC_ID"` + RemoteVPCID int `gorm:"column:remote_epc_id;type:int;default:0" mapstructure:"REMOTE_EPC_ID"` + LocalRegionID int `gorm:"column:local_region_id;type:int;default:0" mapstructure:"LOCAL_REGION_ID"` + RemoteRegionID int `gorm:"column:remote_region_id;type:int;default:0" mapstructure:"REMOTE_REGION_ID"` + CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` +} + +type CEN struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` + Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` + VPCIDs string `gorm:"column:epc_ids;type:text;default:''" mapstructure:"EPC_IDS"` // separated by , + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` +} + +func (CEN) TableName() string { + return "cen" +} + +type RDSInstance struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` + State int `gorm:"column:state;type:tinyint(1);not null;default:0" mapstructure:"STATE"` // 0. Unknown 1. Running 2. Recovering + Domain string `gorm:"column:domain;type:char(64);default:''" mapstructure:"DOMAIN"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` + VPCID int `gorm:"column:epc_id;type:int;default:0" mapstructure:"EPC_ID"` + Type int `gorm:"column:type;type:int;default:0" mapstructure:"TYPE"` // 0. Unknown 1. MySQL 2. SqlServer 3. PPAS 4. PostgreSQL 5. MariaDB + Version string `gorm:"column:version;type:char(64);default:''" mapstructure:"VERSION"` + Series int `gorm:"column:series;type:tinyint(1);not null;default:0" mapstructure:"SERIES"` // 0. Unknown 1. basic 2. HA + Model int `gorm:"column:model;type:tinyint(1);not null;default:0" mapstructure:"MODEL"` // 0. Unknown 1. Primary 2. Readonly 3. Temporary 4. Disaster recovery 5. share + UID string `gorm:"column:uid;type:char(64);default:''" mapstructure:"UID"` +} + +func (RDSInstance) TableName() string { + return "rds_instance" +} + +type RedisInstance struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` + State int `gorm:"column:state;type:tinyint(1);not null;default:0" mapstructure:"STATE"` // 0. Unknown 1. Running 2. Recovering + Domain string `gorm:"column:domain;type:char(64);default:''" mapstructure:"DOMAIN"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` + VPCID int `gorm:"column:epc_id;type:int;default:0" mapstructure:"EPC_ID"` + Version string `gorm:"column:version;type:char(64);default:''" mapstructure:"VERSION"` + InternalHost string `gorm:"column:internal_host;type:varchar(128);default:''" mapstructure:"INTERNAL_HOST"` + PublicHost string `gorm:"column:public_host;type:varchar(128);default:''" mapstructure:"PUBLIC_HOST"` + UID string `gorm:"column:uid;type:char(64);default:''" mapstructure:"UID"` +} + +type VIP struct { + Base `gorm:"embedded"` + IP string `gorm:"column:ip;type:char(64);default:''" mapstructure:"IP"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` + VTapID uint32 `gorm:"column:vtap_id;type:int;not null;default:0" mapstructure:"VTAP_ID"` +} + +func (VIP) TableName() string { + return "vip" +} + +type PodCluster struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + ClusterName string `gorm:"column:cluster_name;type:varchar(256);default:''" mapstructure:"CLUSTER_NAME"` + Version string `gorm:"column:version;type:varchar(256);default:''" mapstructure:"VERSION"` + VPCID int `gorm:"column:epc_id;type:int;default:null" mapstructure:"VPC_ID"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` +} + +type PodNamespace struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` + CloudTags string `gorm:"column:cloud_tags;type:text;default:''" mapstructure:"CLOUD_TAGS"` // separated by , + PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" mapstructure:"POD_CLUSTER_ID"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` +} + +type PodNode struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` + Type int `gorm:"column:type;type:int;default:null" mapstructure:"TYPE"` // 1: Master 2: Node + ServerType int `gorm:"column:server_type;type:int;default:null" mapstructure:"SERVER_TYPE"` // 1: Host 2: VM + State int `gorm:"column:state;type:int;default:1" mapstructure:"STATE"` // 0: Exception 1: Normal + IP string `gorm:"column:ip;type:char(64);default:''" mapstructure:"IP"` + VCPUNum int `gorm:"column:vcpu_num;type:int;default:0" mapstructure:"VCPU_NUM"` + MemTotal int `gorm:"column:mem_total;type:int;default:0" mapstructure:"MEM_TOTAL"` // unit: M + PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" mapstructure:"POD_CLUSTER_ID"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` + VPCID int `gorm:"column:epc_id;type:int;default:null" mapstructure:"VPC_ID"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` +} + +type PodIngress struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` + PodNamespaceID int `gorm:"column:pod_namespace_id;type:int;default:null" mapstructure:"POD_NAMESPACE_ID"` + PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" mapstructure:"POD_CLUSTER_ID"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` +} + +type PodIngressRule struct { + Base `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Protocol string `gorm:"column:protocol;type:char(64);default:''" mapstructure:"PROTOCOL"` + Host string `gorm:"column:host;type:text;default:''" mapstructure:"HOST"` + PodIngressID int `gorm:"column:pod_ingress_id;type:int;default:null" mapstructure:"POD_INGRESS_ID"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` +} + +type PodIngressRuleBackend struct { + Base `gorm:"embedded"` + Path string `gorm:"column:path;type:text;default:''" mapstructure:"PATH"` + Port int `gorm:"column:port;type:int;default:null" mapstructure:"PORT"` + PodServiceID int `gorm:"column:pod_service_id;type:int;default:null" mapstructure:"POD_SERVICE_ID"` + PodIngressRuleID int `gorm:"column:pod_ingress_rule_id;type:int;default:null" mapstructure:"POD_INGRESS_RULE_ID"` + PodIngressID int `gorm:"column:pod_ingress_id;type:int;default:null" mapstructure:"POD_INGRESS_ID"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` +} + +type PodService struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Label string `gorm:"column:label;type:text;default:''" mapstructure:"LABEL"` // separated by , + Annotation string `gorm:"column:annotation;type:text;default:''" mapstructure:"ANNOTATION"` // separated by , + Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` + Type int `gorm:"column:type;type:int;default:null" mapstructure:"TYPE"` // 1: ClusterIP 2: NodePort + Selector string `gorm:"column:selector;type:text;default:''" mapstructure:"SELECTOR"` // separated by , + ServiceClusterIP string `gorm:"column:service_cluster_ip;type:char(64);default:''" mapstructure:"SERVICE_CLUSTER_IP"` + PodIngressID int `gorm:"column:pod_ingress_id;type:int;default:null" mapstructure:"POD_INGRESS_ID"` + PodNamespaceID int `gorm:"column:pod_namespace_id;type:int;default:null" mapstructure:"POD_NAMESPACE_ID"` + PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" mapstructure:"POD_CLUSTER_ID"` + VPCID int `gorm:"column:epc_id;type:int;default:null" mapstructure:"VPC_ID"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` +} + +type PodServicePort struct { + Base `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Protocol string `gorm:"column:protocol;type:char(64);default:''" mapstructure:"PROTOCOL"` + Port int `gorm:"column:port;type:int;default:null" mapstructure:"PORT"` + TargetPort int `gorm:"column:target_port;type:int;default:null" mapstructure:"TARGET_PORT"` + NodePort int `gorm:"column:node_port;type:int;default:null" mapstructure:"NODE_PORT"` + PodServiceID int `gorm:"column:pod_service_id;type:int;default:null" mapstructure:"POD_SERVICE_ID"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` +} + +type PodGroup struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` + Type int `gorm:"column:type;type:int;default:null" mapstructure:"TYPE"` // 1: Deployment 2: StatefulSet 3: ReplicationController + PodNum int `gorm:"column:pod_num;type:int;default:1" mapstructure:"POD_NUM"` + Label string `gorm:"column:label;type:text;default:''" mapstructure:"LABEL"` // separated by , + PodNamespaceID int `gorm:"column:pod_namespace_id;type:int;default:null" mapstructure:"POD_NAMESPACE_ID"` + PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" mapstructure:"POD_CLUSTER_ID"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` +} + +type PodGroupPort struct { + Base `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Protocol string `gorm:"column:protocol;type:char(64);default:''" mapstructure:"PROTOCOL"` + Port int `gorm:"column:port;type:int;default:null" mapstructure:"PORT"` + PodGroupID int `gorm:"column:pod_group_id;type:int;default:null" mapstructure:"POD_GROUP_ID"` + PodServiceID int `gorm:"column:pod_service_id;type:int;default:null" mapstructure:"POD_SERVICE_ID"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` +} + +type PodReplicaSet struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` + Label string `gorm:"column:label;type:text;default:''" mapstructure:"LABEL"` // separated by , + PodNum int `gorm:"column:pod_num;type:int;default:1" mapstructure:"POD_NUM"` + PodGroupID int `gorm:"column:pod_group_id;type:int;default:null" mapstructure:"POD_GROUP_ID"` + PodNamespaceID int `gorm:"column:pod_namespace_id;type:int;default:null" mapstructure:"POD_NAMESPACE_ID"` + PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" mapstructure:"POD_CLUSTER_ID"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` +} + +func (PodReplicaSet) TableName() string { + return "pod_rs" +} + +type PrometheusTarget struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Instance string `gorm:"column:instance;type:varchar(255);default:''" mapstructure:"INSTANCE"` + Job string `gorm:"column:job;type:varchar(255);default:''" mapstructure:"JOB"` + ScrapeURL string `gorm:"column:scrape_url;type:varchar(2083);default:''" mapstructure:"SCRAPE_URL"` + OtherLabels string `gorm:"column:other_labels;type:text;default:''" mapstructure:"OTHER_LABELS"` // separated by , + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` + PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" mapstructure:"POD_CLUSTER_ID"` + CreateMethod int `gorm:"column:create_method;type:tinyint(1);default:1" mapstructure:"CREATE_METHOD"` +} + +func (PrometheusTarget) TableName() string { + return "prometheus_target" +} + +type Pod struct { + Base `gorm:"embedded"` + SoftDeleteBase `gorm:"embedded"` + Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` + Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` + State int `gorm:"column:state;type:int;not null" mapstructure:"STATE"` // 0.Exception 1.Running + Label string `gorm:"column:label;type:text;default:''" mapstructure:"LABEL"` // separated by , + Annotation string `gorm:"column:annotation;type:text;default:''" mapstructure:"ANNOTATION"` // separated by , + ENV string `gorm:"column:env;type:text;default:''" mapstructure:"ENV"` // separated by , + ContainerIDs string `gorm:"column:container_ids;type:text;default:''" mapstructure:"CONTAINER_IDS"` // separated by , + PodReplicaSetID int `gorm:"column:pod_rs_id;type:int;default:null" mapstructure:"POD_RS_ID"` + PodGroupID int `gorm:"column:pod_group_id;type:int;default:null" mapstructure:"POD_GROUP_ID"` + PodNamespaceID int `gorm:"column:pod_namespace_id;type:int;default:null" mapstructure:"POD_NAMESPACE_ID"` + PodNodeID int `gorm:"column:pod_node_id;type:int;default:null" mapstructure:"POD_NODE_ID"` + PodClusterID int `gorm:"column:pod_cluster_id;type:int;default:null" mapstructure:"POD_CLUSTER_ID"` + VPCID int `gorm:"column:epc_id;type:int;default:null" mapstructure:"VPC_ID"` + AZ string `gorm:"column:az;type:char(64);default:''" mapstructure:"AZ"` + Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` + SubDomain string `gorm:"column:sub_domain;type:char(64);default:''" mapstructure:"SUB_DOMAIN"` + Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` +} From 298c4427b996f39fb99fd77b2635a7882e9bb729 Mon Sep 17 00:00:00 2001 From: zhengya Date: Wed, 11 Oct 2023 14:21:52 +0800 Subject: [PATCH 02/22] [CONTROLLER/PROMETHEUS] fixes updating label type bug --- server/controller/prometheus/cache/cache.go | 10 +++++----- server/controller/prometheus/cache/metric_label.go | 10 ++++++++-- server/controller/prometheus/cache/target.go | 1 - 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/server/controller/prometheus/cache/cache.go b/server/controller/prometheus/cache/cache.go index 2e679f9ad56..89d04ee53d0 100644 --- a/server/controller/prometheus/cache/cache.go +++ b/server/controller/prometheus/cache/cache.go @@ -218,19 +218,19 @@ func GetDebugCache(t controller.PrometheusCacheType) []byte { getLabel := func() { temp := map[string]interface{}{ "key_to_id": make(map[string]interface{}), - "id_to_key": make(map[int]string), + "id_to_key": make(map[int]interface{}), } - tempCache.Label.idToKey.Range(func(key, value any) bool { + tempCache.Label.keyToID.Range(func(key, value any) bool { temp["key_to_id"].(map[string]interface{})[marshal(key)] = value return true }) tempCache.Label.idToKey.Range(func(key, value any) bool { - temp["id_to_key"].(map[int]string)[key.(int)] = marshal(value) + temp["id_to_key"].(map[int]interface{})[key.(int)] = value return true }) if len(temp["key_to_id"].(map[string]interface{})) > 0 || - len(temp["id_to_key"].(map[int]string)) > 0 { + len(temp["id_to_key"].(map[int]interface{})) > 0 { content["label"] = temp } } @@ -265,7 +265,7 @@ func GetDebugCache(t controller.PrometheusCacheType) []byte { for item := range value.Iterator().C { keys[marshal(item)] = struct{}{} } - temp["target_id_to_label_names"].(map[string]interface{})[key] = keys + temp["metric_name_to_target_ids"].(map[string]interface{})[key] = keys } if len(temp["metric_target_keys"].(map[string]interface{})) > 0 || len(temp["metric_name_to_target_ids"].(map[string]interface{})) > 0 || diff --git a/server/controller/prometheus/cache/metric_label.go b/server/controller/prometheus/cache/metric_label.go index 1d9507e995e..574327d731a 100644 --- a/server/controller/prometheus/cache/metric_label.go +++ b/server/controller/prometheus/cache/metric_label.go @@ -67,7 +67,10 @@ func (mi *metricLabel) GetMetricNameIDToLabelIDs() *hashmap.Map[int, mapset.Set[ func (ml *metricLabel) Add(batch []*controller.PrometheusMetricLabel) { for _, item := range batch { for _, li := range item.GetLabelIds() { - ml.metricNameIDToLabelIDs.GetOrInsert(int(item.GetMetricNameId()), mapset.NewSet(int(li))) + ml.metricNameIDToLabelIDs.GetOrInsert(int(item.GetMetricNameId()), mapset.NewSet[int]()) + if lids, ok := ml.metricNameIDToLabelIDs.Get(int(item.GetMetricNameId())); ok { + lids.Add(int(li)) + } } } } @@ -79,7 +82,10 @@ func (ml *metricLabel) refresh(args ...interface{}) error { } for _, item := range metricLabels { if mni, ok := ml.metricNameCache.GetIDByName(item.MetricName); ok { - ml.metricNameIDToLabelIDs.GetOrInsert(mni, mapset.NewSet(item.LabelID)) + ml.metricNameIDToLabelIDs.GetOrInsert(mni, mapset.NewSet[int]()) + if lids, ok := ml.metricNameIDToLabelIDs.Get(mni); ok { + lids.Add(item.LabelID) + } } } return nil diff --git a/server/controller/prometheus/cache/target.go b/server/controller/prometheus/cache/target.go index 2d6736c1def..902a1f2dbff 100644 --- a/server/controller/prometheus/cache/target.go +++ b/server/controller/prometheus/cache/target.go @@ -135,7 +135,6 @@ func (t *targetIDToLabelNames) Load(id int) []string { type target struct { keyToTargetID *keyToTargetID targetIDToLabelNames *targetIDToLabelNames - idToLabels sync.Map } func newTarget() *target { From 25aa4a5b04123ace9f55dcd1cdd9031b17a6f87b Mon Sep 17 00:00:00 2001 From: Liu Chao Date: Wed, 11 Oct 2023 17:28:31 +0800 Subject: [PATCH 03/22] [querier] Support tag is equal to empty --- server/querier/engine/clickhouse/filter.go | 31 ++++++++++++++++++---- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/server/querier/engine/clickhouse/filter.go b/server/querier/engine/clickhouse/filter.go index 807c4672926..149cef8ada0 100644 --- a/server/querier/engine/clickhouse/filter.go +++ b/server/querier/engine/clickhouse/filter.go @@ -806,8 +806,14 @@ func GetPrometheusFilter(promTag, table, op, value string) (string, error) { } labelNameID, ok := Prometheus.LabelNameToID[nameNoPreffix] if !ok { - errorMessage := fmt.Sprintf("%s not found", nameNoPreffix) - return filter, errors.New(errorMessage) + if value == "" { + filter = "1=1" + } else { + filter = "1!=1" + } + debugMessage := fmt.Sprintf("%s not found", nameNoPreffix) + log.Debug(debugMessage) + return filter, nil } // Determine whether the tag is app_label or target_label isAppLabel := false @@ -815,6 +821,10 @@ func GetPrometheusFilter(promTag, table, op, value string) (string, error) { for _, appLabel := range appLabels { if appLabel.AppLabelName == nameNoPreffix { isAppLabel = true + if value == "" { + filter = fmt.Sprintf("app_label_value_id_%d = 0", appLabel.appLabelColumnIndex) + return filter, nil + } if strings.Contains(op, "match") { filter = fmt.Sprintf("toUInt64(app_label_value_id_%d) IN (SELECT label_value_id FROM flow_tag.app_label_live_view WHERE label_name_id=%d and %s(label_value,%s))", appLabel.appLabelColumnIndex, labelNameID, op, value) } else { @@ -846,8 +856,14 @@ func GetRemoteReadFilter(promTag, table, op, value, originFilter string, e *CHEn } labelNameID, ok := Prometheus.LabelNameToID[nameNoPreffix] if !ok { - errorMessage := fmt.Sprintf("%s not found", nameNoPreffix) - return filter, errors.New(errorMessage) + if value == "" { + filter = "1=1" + } else { + filter = "1!=1" + } + debugMessage := fmt.Sprintf("%s not found", nameNoPreffix) + log.Debug(debugMessage) + return filter, nil } prometheusSubqueryCache := GetPrometheusSubqueryCache() // Determine whether the tag is app_label or target_label @@ -863,6 +879,12 @@ func GetRemoteReadFilter(promTag, table, op, value, originFilter string, e *CHEn return filter, nil } } + if value == "" { + filter = fmt.Sprintf("app_label_value_id_%d = 0", appLabel.appLabelColumnIndex) + entryValue := common.EntryValue{Time: time.Now(), Filter: filter} + prometheusSubqueryCache.PrometheusSubqueryCache.Add(originFilter, entryValue) + return filter, nil + } // lru timeout if strings.Contains(op, "match") { @@ -909,7 +931,6 @@ func GetRemoteReadFilter(promTag, table, op, value, originFilter string, e *CHEn } targetLabelFilter := TargetLabelFilter{OriginFilter: originFilter, TransFilter: transFilter} e.TargetLabelFilters = append(e.TargetLabelFilters, targetLabelFilter) - // FIXME delete placeholders filter = "" } return filter, nil From 2856175ce46bd2a7f3fcd595e8672a4ab00d2915 Mon Sep 17 00:00:00 2001 From: Jiping Yin Date: Thu, 12 Oct 2023 10:35:20 +0800 Subject: [PATCH 04/22] [eBPF] Adjust refresh perf buffer frequency --- agent/src/ebpf/user/config.h | 14 +++++++------- agent/src/ebpf/user/tracer.c | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/agent/src/ebpf/user/config.h b/agent/src/ebpf/user/config.h index a96007e134f..ac9b48244a5 100644 --- a/agent/src/ebpf/user/config.h +++ b/agent/src/ebpf/user/config.h @@ -175,34 +175,34 @@ enum { */ /* - * timer ticks: ticks every 1 millisecond. + * tick every 10 millisecond * * unit: microseconds */ -#define EVENT_PERIOD_TIME_US 1000 +#define EVENT_TIMER_TICK_US 10000 /* * The kernel uses bundled burst to send data to the user. * The implementation method is that all CPUs trigger timeout checks and send * the data resident in the eBPF buffer. This value is the periodic time, unit - * is microseconds. + * is milliseconds. */ -#define KICK_KERN_PERIOD 10 // 10 milliseconds +#define KICK_KERN_PERIOD 10 // 10 ticks(100 milliseconds) /* * System boot time update cycle time, unit is milliseconds. */ -#define SYS_TIME_UPDATE_PERIOD 1000 // 1 seconds +#define SYS_TIME_UPDATE_PERIOD 1000 // 1000 ticks(10 seconds) /* * Check whether the eBPF Map exceeds the maximum value and use it to release * stale data (unit is milliseconds). */ -#define CHECK_MAP_EXCEEDED_PERIOD 1000 // 1 seconds +#define CHECK_MAP_EXCEEDED_PERIOD 100 // 100 ticks(1 seconds) /* * Used to check whether the kernel adaptation is successful, here is the * check cycle time (unit is milliseconds). */ -#define CHECK_KERN_ADAPT_PERIOD 1000 // 1 seconds +#define CHECK_KERN_ADAPT_PERIOD 100 // 100 ticks(1 seconds) #endif /* DF_EBPF_CONFIG_H */ diff --git a/agent/src/ebpf/user/tracer.c b/agent/src/ebpf/user/tracer.c index 3b9c30bf904..b59761b1cc6 100644 --- a/agent/src/ebpf/user/tracer.c +++ b/agent/src/ebpf/user/tracer.c @@ -1344,7 +1344,7 @@ static void period_process_main(__unused void *arg) for (;;) { period_events_process(); - usleep(EVENT_PERIOD_TIME_US); + usleep(EVENT_TIMER_TICK_US); } } From c8f118835de3331c92939857f0e852da0d7e7c85 Mon Sep 17 00:00:00 2001 From: Ericsssss Date: Thu, 12 Oct 2023 15:07:28 +0800 Subject: [PATCH 05/22] [Querier] modify pod_group_type enum error #22502 - run automation test (querier_sql & basic) pass --- server/querier/engine/clickhouse/tag/translation.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/querier/engine/clickhouse/tag/translation.go b/server/querier/engine/clickhouse/tag/translation.go index 24556be1360..dd73d879b22 100644 --- a/server/querier/engine/clickhouse/tag/translation.go +++ b/server/querier/engine/clickhouse/tag/translation.go @@ -44,7 +44,7 @@ var TAP_PORT_DEVICE_MAP = map[string]int{ } var INT_ENUM_TAG = []string{"close_type", "eth_type", "signal_source", "is_ipv4", "l7_ip_protocol", "type", "l7_protocol", "protocol", "response_status", "server_port", "status", "tap_port_type", "tunnel_tier", "tunnel_type", "instance_type", "nat_source", "role", "event_level", "policy_level", "policy_app_type"} -var INT_ENUM_PEER_TAG = []string{"resource_gl0_type", "resource_gl1_type", "resource_gl2_type", "tcp_flags_bit", "auto_instance_type", "auto_service_type", "pod_group_type"} +var INT_ENUM_PEER_TAG = []string{"resource_gl0_type", "resource_gl1_type", "resource_gl2_type", "tcp_flags_bit", "auto_instance_type", "auto_service_type"} var STRING_ENUM_TAG = []string{"tap_side", "event_type", "profile_language_type"} func GenerateTagResoureMap() map[string]map[string]*Tag { @@ -887,6 +887,12 @@ func GenerateTagResoureMap() map[string]map[string]*Tag { "toUInt64("+podGroupIDSuffix+") IN (SELECT id FROM flow_tag.pod_group_map WHERE pod_group_type %s %s)", "", ), + "enum": NewTag( + "dictGetOrDefault(flow_tag.int_enum_map, 'name', ('%s',toUInt64(dictGet(flow_tag.pod_group_map, 'pod_group_type', (toUInt64("+podGroupIDSuffix+"))))), dictGet(flow_tag.pod_group_map, 'pod_group_type', (toUInt64("+podGroupIDSuffix+"))))", + "", + "toUInt64(dictGet(flow_tag.pod_group_map, 'pod_group_type', (toUInt64("+podGroupIDSuffix+")))) IN (SELECT value FROM flow_tag.int_enum_map WHERE name %s %s and tag_name='%s')", + "toUInt64(dictGet(flow_tag.pod_group_map, 'pod_group_type', (toUInt64("+podGroupIDSuffix+")))) IN (SELECT value FROM flow_tag.int_enum_map WHERE %s(name,%s) and tag_name='%s')", + ), } } // enum_tag From b9a9d1faee54524f60b33a5cb231373b4f8fa8e9 Mon Sep 17 00:00:00 2001 From: Jiping Yin Date: Thu, 12 Oct 2023 19:03:15 +0800 Subject: [PATCH 06/22] [eBPF] Optimize fastcgi protocol inference (#4465) The purpose is to push all data to the upper layer if the sofa protocol is mistakenly inferred as the fastcgi protocol. --- agent/src/ebpf/kernel/include/protocol_inference.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/agent/src/ebpf/kernel/include/protocol_inference.h b/agent/src/ebpf/kernel/include/protocol_inference.h index 9f3c25bfdb8..394ad8e810c 100644 --- a/agent/src/ebpf/kernel/include/protocol_inference.h +++ b/agent/src/ebpf/kernel/include/protocol_inference.h @@ -1519,6 +1519,15 @@ infer_fastcgi_message(const char *buf, size_t count, return MSG_PRESTORE; } + if (is_socket_info_valid(conn_info->socket_info_ptr)) { + if (conn_info->socket_info_ptr->l7_proto != PROTO_FASTCGI) + return MSG_UNKNOWN; + if (header->type == FCGI_BEGIN_REQUEST || header->type == FCGI_PARAMS) + return MSG_REQUEST; + else + return MSG_RESPONSE; + } + if (fastcgi_header_common_check(header) && count > 8 && __bpf_ntohs(header->content_length) != 0) { if (header->type == FCGI_BEGIN_REQUEST) { From 14b322856cbc3439525c60f9935063a705e783a9 Mon Sep 17 00:00:00 2001 From: zhengya Date: Fri, 13 Oct 2023 15:46:44 +0800 Subject: [PATCH 07/22] [CONTROLLER/TAGRECORDER] limits vtap port name length --- server/controller/tagrecorder/ch_vtap_port.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/controller/tagrecorder/ch_vtap_port.go b/server/controller/tagrecorder/ch_vtap_port.go index 042d81145f9..69d3f6f9fd1 100644 --- a/server/controller/tagrecorder/ch_vtap_port.go +++ b/server/controller/tagrecorder/ch_vtap_port.go @@ -31,6 +31,8 @@ import ( "github.com/deepflowio/deepflow/server/controller/model" ) +const vTapPortNameLength = 256 + type ChVTapPort struct { UpdaterBase[mysql.ChVTapPort, VtapPortKey] } @@ -147,6 +149,9 @@ func (v *ChVTapPort) generateNewData() (map[VtapPortKey]mysql.ChVTapPort, bool) log.Debugf("duplicate name: %s (id: %d)", data.TapName, data.ID) } } + if len(vTapPort.Name) > vTapPortNameLength { + vTapPort.Name = vTapPort.Name[:vTapPortNameLength] + } if vTapPort.DeviceID == 0 && vTapPort.DeviceType == 0 && data.DeviceID != 0 && data.DeviceType != 0 { log.Debugf("device id: %d, device type: %d ", vTapPort.DeviceID, vTapPort.DeviceType) vTapPort.DeviceID = data.DeviceID @@ -195,6 +200,9 @@ func (v *ChVTapPort) generateNewData() (map[VtapPortKey]mysql.ChVTapPort, bool) log.Debugf("duplicate name: %s (id: %d)", data.TapName, data.ID) } } + if len(vTapPort.Name) > vTapPortNameLength { + vTapPort.Name = vTapPort.Name[:vTapPortNameLength] + } if vTapPort.DeviceID == 0 && vTapPort.DeviceType == 0 && data.DeviceID != 0 && data.DeviceType != 0 { log.Debugf("device id: %d, device type: %d ", vTapPort.DeviceID, vTapPort.DeviceType) vTapPort.DeviceID = data.DeviceID @@ -241,6 +249,9 @@ func (v *ChVTapPort) generateNewData() (map[VtapPortKey]mysql.ChVTapPort, bool) } else if !common.Contains(nameSlice, "lo") { vTapPort.Name = strings.Join([]string{"lo", vTapPort.Name}, ", ") } + if len(vTapPort.Name) > vTapPortNameLength { + vTapPort.Name = vTapPort.Name[:vTapPortNameLength] + } if vTapPort.DeviceID == 0 && vTapPort.DeviceType == 0 && deviceInfo.DeviceID != 0 && deviceInfo.DeviceType != 0 { log.Debugf("device id: %d, device type: %d ", vTapPort.DeviceID, vTapPort.DeviceType) vTapPort.DeviceID = deviceInfo.DeviceID From 2d23933eab9e73cb7eebf871ff27e181836a021d Mon Sep 17 00:00:00 2001 From: zhengya Date: Thu, 12 Oct 2023 17:50:25 +0800 Subject: [PATCH 08/22] [CONTROLLER/HTTP] fixes resource api resopnse bug --- .../controller/db/mysql/platform_rsc_model.go | 152 +++++++++--------- 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/server/controller/db/mysql/platform_rsc_model.go b/server/controller/db/mysql/platform_rsc_model.go index 076d8187532..e35b7f02b0c 100644 --- a/server/controller/db/mysql/platform_rsc_model.go +++ b/server/controller/db/mysql/platform_rsc_model.go @@ -68,8 +68,8 @@ type SoftDeleteBase struct { } type Process struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` VTapID uint32 `gorm:"column:vtap_id;type:int;not null;default:0" mapstructure:"VTAP_ID"` PID uint64 `gorm:"column:pid;type:int;not null;default:0" mapstructure:"PID"` @@ -85,8 +85,8 @@ type Process struct { } type Domain struct { - Base `gorm:"embedded"` - OperatedTime `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + OperatedTime `gorm:"embedded" mapstructure:",squash"` SyncedAt *time.Time `gorm:"column:synced_at" mapstructure:"SYNCED_AT"` Name string `gorm:"column:name;type:varchar(64)" mapstructure:"NAME"` IconID int `gorm:"column:icon_id;type:int" mapstructure:"ICON_ID"` @@ -103,8 +103,8 @@ type Domain struct { // TODO 最终可以与cloud模块命名统一,Domain -> DomainLcuuid type SubDomain struct { - Base `gorm:"embedded"` - OperatedTime `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + OperatedTime `gorm:"embedded" mapstructure:",squash"` SyncedAt *time.Time `gorm:"column:synced_at" mapstructure:"SYNCED_AT"` Domain string `gorm:"column:domain;type:char(64);default:''" mapstructure:"DOMAIN"` Name string `gorm:"column:name;type:varchar(64);default:''" mapstructure:"NAME"` @@ -118,8 +118,8 @@ type SubDomain struct { } type Region struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(64);default:''" mapstructure:"NAME"` CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined Label string `gorm:"column:label;type:varchar(64);default:''" mapstructure:"LABEL"` @@ -128,8 +128,8 @@ type Region struct { } type AZ struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(64);default:''" mapstructure:"NAME"` CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined Label string `gorm:"column:label;type:varchar(64);default:''" mapstructure:"LABEL"` @@ -142,8 +142,8 @@ func (AZ) TableName() string { } type Host struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Type int `gorm:"column:type;type:int" mapstructure:"TYPE"` // 1.Server 3.Gateway 4.DFI State int `gorm:"column:state;type:int" mapstructure:"STATE"` // 0.Temp 1.Creating 2.Complete 3.Modifying 4.Exception Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` @@ -168,8 +168,8 @@ func (Host) TableName() string { } type VM struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` State int `gorm:"index:state_server_index;column:state;type:int;not null" mapstructure:"STATE"` // 0.Temp 1.Creating 2.Created 3.To run 4.Running 5.To suspend 6.Suspended 7.To resume 8. To stop 9.Stopped 10.Modifing 11.Exception 12.Destroying Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` @@ -190,7 +190,7 @@ func (VM) TableName() string { } type VMPodNodeConnection struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` VMID int `gorm:"column:vm_id;type:int;default:null" mapstructure:"VM_ID"` PodNodeID int `gorm:"column:pod_node_id;type:int;default:null" mapstructure:"POD_NODE_ID"` Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` @@ -202,7 +202,7 @@ func (VMPodNodeConnection) TableName() string { } type VMSecurityGroup struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` SecurityGroupID int `gorm:"column:sg_id;type:int;not null" mapstructure:"SG_ID"` VMID int `gorm:"column:vm_id;type:int;not null" mapstructure:"VM_ID"` Priority int `gorm:"column:priority;type:int;not null" mapstructure:"PRIORITY"` @@ -213,7 +213,7 @@ func (VMSecurityGroup) TableName() string { } type Contact struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined Mobile string `gorm:"column:mobile;type:char(13);default:''" mapstructure:"MOBILE"` @@ -229,7 +229,7 @@ type Contact struct { } type VPCContact struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined VPCID int `gorm:"column:epc_id;type:int;default:0" mapstructure:"VPC_ID"` ContactID int `gorm:"column:contact_id;type:int;default:0" mapstructure:"CONTACT_ID"` @@ -240,8 +240,8 @@ func (VPCContact) TableName() string { } type VPC struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` CreateMethod int `gorm:"column:create_method;type:int;default:0" mapstructure:"CREATE_METHOD"` // 0.learning 1.user_defined Label string `gorm:"column:label;type:varchar(64);default:''" mapstructure:"LABEL"` @@ -260,8 +260,8 @@ func (VPC) TableName() string { } type Network struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` State int `gorm:"column:state;type:int;not null" mapstructure:"STATE"` // 0.Temp 1.Creating 2.Created 3.Exception 4.Modifing 5.Destroying 6.Destroyed NetType int `gorm:"column:net_type;type:int;default:4" mapstructure:"NET_TYPE"` // 1.CTRL 2.SERVICE 3.WAN 4.LAN Name string `gorm:"column:name;type:varchar(256);not null" mapstructure:"NAME"` @@ -287,7 +287,7 @@ func (Network) TableName() string { } type Subnet struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` Prefix string `gorm:"column:prefix;type:char(64);default:''" mapstructure:"PREFIX"` Netmask string `gorm:"column:netmask;type:char(64);default:''" mapstructure:"NETMASK"` NetworkID int `gorm:"column:vl2id;type:int;default:null" mapstructure:"VL2ID"` @@ -302,8 +302,8 @@ func (Subnet) TableName() string { } type VRouter struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` State int `gorm:"index:state_server_index;column:state;type:int;not null" mapstructure:"STATE"` // 0.Temp 1.Creating 2.Created 3.Exception 4.Modifing 5.Destroying 6.To run 7.Running 8.To stop 9.Stopped Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` @@ -320,7 +320,7 @@ func (VRouter) TableName() string { } type RoutingTable struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` VRouterID int `gorm:"column:vnet_id;type:int;default:null" mapstructure:"VNET_ID"` Destination string `gorm:"column:destination;type:text;default:''" mapstructure:"DESTINATION"` NexthopType string `gorm:"column:nexthop_type;type:text;default:''" mapstructure:"NEXTHOP_TYPE"` @@ -328,8 +328,8 @@ type RoutingTable struct { } type DHCPPort struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` @@ -342,7 +342,7 @@ func (DHCPPort) TableName() string { } type VInterface struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:char(64);default:''" mapstructure:"NAME"` Index int `gorm:"column:ifindex;type:int;not null" mapstructure:"IFINDEX"` State int `gorm:"column:state;type:int;not null" mapstructure:"STATE"` // 1. Attached 2.Detached 3.Exception @@ -369,7 +369,7 @@ func (VInterface) TableName() string { } type LANIP struct { // TODO 添加region字段 - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` IP string `gorm:"column:ip;type:char(64);default:''" mapstructure:"IP"` Netmask string `gorm:"column:netmask;type:char(64);default:''" mapstructure:"NETMASK"` Gateway string `gorm:"column:gateway;type:char(64);default:''" mapstructure:"GATEWAY"` @@ -390,7 +390,7 @@ func (LANIP) TableName() string { } type WANIP struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` IP string `gorm:"column:ip;type:char(64);default:''" mapstructure:"IP"` Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` Netmask int `gorm:"column:netmask;type:int;default:null" mapstructure:"NETMASK"` @@ -411,7 +411,7 @@ func (WANIP) TableName() string { } type FloatingIP struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` Region string `gorm:"column:region;type:char(64);default:''" mapstructure:"REGION"` VPCID int `gorm:"column:epc_id;type:int;default:0" mapstructure:"VPC_ID"` @@ -425,8 +425,8 @@ func (FloatingIP) TableName() string { } type SecurityGroup struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Label string `gorm:"column:label;type:varchar(64);default:''" mapstructure:"LABEL"` Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` @@ -437,7 +437,7 @@ type SecurityGroup struct { } type SecurityGroupRule struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` SecurityGroupID int `gorm:"column:sg_id;type:int;not null" mapstructure:"SG_ID"` Direction int `gorm:"column:direction;type:tinyint(1);not null;default:0" mapstructure:"DIRECTION"` // 0.Unknow 1.Ingress 2.Egress Protocol string `gorm:"column:protocol;type:char(64);default:''" mapstructure:"PROTOCOL"` @@ -451,8 +451,8 @@ type SecurityGroupRule struct { } type NATGateway struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` FloatingIPs string `gorm:"column:floating_ips;type:text;default:''" mapstructure:"FLOATING_IPS"` // separated by , @@ -468,7 +468,7 @@ func (NATGateway) TableName() string { } type NATRule struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` NATGatewayID int `gorm:"column:nat_id;type:int;default:0" mapstructure:"NAT_ID"` Type string `gorm:"column:type;type:char(16);default:''" mapstructure:"TYPE"` Protocol string `gorm:"column:protocol;type:char(64);default:''" mapstructure:"PROTOCOL"` @@ -485,7 +485,7 @@ func (NATRule) TableName() string { } type NATVMConnection struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` NATGatewayID int `gorm:"column:nat_id;type:int;default:null" mapstructure:"NAT_ID"` VMID int `gorm:"column:vm_id;type:int;default:null" mapstructure:"VM_ID"` Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` @@ -496,8 +496,8 @@ func (NATVMConnection) TableName() string { } type LB struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` Model int `gorm:"column:model;type:int;default:0" mapstructure:"MODEL"` // 1.Internal 2.External @@ -514,8 +514,8 @@ func (LB) TableName() string { } type LBListener struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` LBID int `gorm:"column:lb_id;type:int;default:0" mapstructure:"LB_ID"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` IPs string `gorm:"column:ips;type:text;default:''" mapstructure:"IPS"` // separated by , @@ -531,7 +531,7 @@ func (LBListener) TableName() string { } type LBTargetServer struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` LBID int `gorm:"column:lb_id;type:int;default:0" mapstructure:"LB_ID"` LBListenerID int `gorm:"column:lb_listener_id;type:int;default:0" mapstructure:"LB_LISTENER_ID"` VPCID int `gorm:"column:epc_id;type:int;default:0" mapstructure:"EPC_ID"` @@ -548,7 +548,7 @@ func (LBTargetServer) TableName() string { } type LBVMConnection struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` LBID int `gorm:"column:lb_id;type:int;default:null" mapstructure:"LB_ID"` VMID int `gorm:"column:vm_id;type:int;default:null" mapstructure:"VM_ID"` Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` @@ -559,8 +559,8 @@ func (LBVMConnection) TableName() string { } type PeerConnection struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` LocalVPCID int `gorm:"column:local_epc_id;type:int;default:0" mapstructure:"LOCAL_EPC_ID"` @@ -572,8 +572,8 @@ type PeerConnection struct { } type CEN struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` @@ -586,8 +586,8 @@ func (CEN) TableName() string { } type RDSInstance struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` State int `gorm:"column:state;type:tinyint(1);not null;default:0" mapstructure:"STATE"` // 0. Unknown 1. Running 2. Recovering @@ -607,8 +607,8 @@ func (RDSInstance) TableName() string { } type RedisInstance struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Label string `gorm:"column:label;type:char(64);default:''" mapstructure:"LABEL"` State int `gorm:"column:state;type:tinyint(1);not null;default:0" mapstructure:"STATE"` // 0. Unknown 1. Running 2. Recovering @@ -623,7 +623,7 @@ type RedisInstance struct { } type VIP struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` IP string `gorm:"column:ip;type:char(64);default:''" mapstructure:"IP"` Domain string `gorm:"column:domain;type:char(64);not null" mapstructure:"DOMAIN"` VTapID uint32 `gorm:"column:vtap_id;type:int;not null;default:0" mapstructure:"VTAP_ID"` @@ -634,8 +634,8 @@ func (VIP) TableName() string { } type PodCluster struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` ClusterName string `gorm:"column:cluster_name;type:varchar(256);default:''" mapstructure:"CLUSTER_NAME"` Version string `gorm:"column:version;type:varchar(256);default:''" mapstructure:"VERSION"` @@ -647,8 +647,8 @@ type PodCluster struct { } type PodNamespace struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` CloudTags string `gorm:"column:cloud_tags;type:text;default:''" mapstructure:"CLOUD_TAGS"` // separated by , @@ -660,8 +660,8 @@ type PodNamespace struct { } type PodNode struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` Type int `gorm:"column:type;type:int;default:null" mapstructure:"TYPE"` // 1: Master 2: Node @@ -679,8 +679,8 @@ type PodNode struct { } type PodIngress struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` PodNamespaceID int `gorm:"column:pod_namespace_id;type:int;default:null" mapstructure:"POD_NAMESPACE_ID"` @@ -692,7 +692,7 @@ type PodIngress struct { } type PodIngressRule struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Protocol string `gorm:"column:protocol;type:char(64);default:''" mapstructure:"PROTOCOL"` Host string `gorm:"column:host;type:text;default:''" mapstructure:"HOST"` @@ -701,7 +701,7 @@ type PodIngressRule struct { } type PodIngressRuleBackend struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` Path string `gorm:"column:path;type:text;default:''" mapstructure:"PATH"` Port int `gorm:"column:port;type:int;default:null" mapstructure:"PORT"` PodServiceID int `gorm:"column:pod_service_id;type:int;default:null" mapstructure:"POD_SERVICE_ID"` @@ -711,8 +711,8 @@ type PodIngressRuleBackend struct { } type PodService struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Label string `gorm:"column:label;type:text;default:''" mapstructure:"LABEL"` // separated by , Annotation string `gorm:"column:annotation;type:text;default:''" mapstructure:"ANNOTATION"` // separated by , @@ -731,7 +731,7 @@ type PodService struct { } type PodServicePort struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Protocol string `gorm:"column:protocol;type:char(64);default:''" mapstructure:"PROTOCOL"` Port int `gorm:"column:port;type:int;default:null" mapstructure:"PORT"` @@ -742,8 +742,8 @@ type PodServicePort struct { } type PodGroup struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` Type int `gorm:"column:type;type:int;default:null" mapstructure:"TYPE"` // 1: Deployment 2: StatefulSet 3: ReplicationController @@ -758,7 +758,7 @@ type PodGroup struct { } type PodGroupPort struct { - Base `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Protocol string `gorm:"column:protocol;type:char(64);default:''" mapstructure:"PROTOCOL"` Port int `gorm:"column:port;type:int;default:null" mapstructure:"PORT"` @@ -768,8 +768,8 @@ type PodGroupPort struct { } type PodReplicaSet struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` Label string `gorm:"column:label;type:text;default:''" mapstructure:"LABEL"` // separated by , @@ -788,8 +788,8 @@ func (PodReplicaSet) TableName() string { } type PrometheusTarget struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Instance string `gorm:"column:instance;type:varchar(255);default:''" mapstructure:"INSTANCE"` Job string `gorm:"column:job;type:varchar(255);default:''" mapstructure:"JOB"` ScrapeURL string `gorm:"column:scrape_url;type:varchar(2083);default:''" mapstructure:"SCRAPE_URL"` @@ -805,8 +805,8 @@ func (PrometheusTarget) TableName() string { } type Pod struct { - Base `gorm:"embedded"` - SoftDeleteBase `gorm:"embedded"` + Base `gorm:"embedded" mapstructure:",squash"` + SoftDeleteBase `gorm:"embedded" mapstructure:",squash"` Name string `gorm:"column:name;type:varchar(256);default:''" mapstructure:"NAME"` Alias string `gorm:"column:alias;type:char(64);default:''" mapstructure:"ALIAS"` State int `gorm:"column:state;type:int;not null" mapstructure:"STATE"` // 0.Exception 1.Running From d395da7fc61e44defda2f98e624ebc8da6555fef Mon Sep 17 00:00:00 2001 From: Ericsssss Date: Wed, 11 Oct 2023 17:51:13 +0800 Subject: [PATCH 09/22] [Tagrecorder] modify app_label_map - run automation test (basic) pass --- server/controller/tagrecorder/ch_app_label.go | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/server/controller/tagrecorder/ch_app_label.go b/server/controller/tagrecorder/ch_app_label.go index d3b30ce8e4d..4048294fbb8 100644 --- a/server/controller/tagrecorder/ch_app_label.go +++ b/server/controller/tagrecorder/ch_app_label.go @@ -17,6 +17,8 @@ package tagrecorder import ( + "golang.org/x/exp/slices" + "github.com/deepflowio/deepflow/server/controller/db/mysql" ) @@ -47,6 +49,7 @@ func (l *ChAPPLabel) generateNewData() (map[PrometheusAPPLabelKey]mysql.ChAPPLab if !ok { return nil, false } + appLabelSlice, ok := l.generateAPPLabelData() labelNameIDMap, valueNameIDMap, ok := l.generateNameIDData() if !ok { @@ -58,14 +61,17 @@ func (l *ChAPPLabel) generateNewData() (map[PrometheusAPPLabelKey]mysql.ChAPPLab labelID := prometheusMetricLabel.LabelID labelNameValueData := metricLabelIDNameValueMap[labelID] labelName := labelNameValueData["label_name"] - labelNameID := labelNameIDMap[labelName] - labelValue := labelNameValueData["label_value"] - labelValueID := valueNameIDMap[labelValue] - keyToItem[PrometheusAPPLabelKey{LabelNameID: labelNameID, LabelValueID: labelValueID}] = mysql.ChAPPLabel{ - LabelNameID: labelNameID, - LabelValue: labelValue, - LabelValueID: labelValueID, + if slices.Contains(appLabelSlice, labelName) { + labelNameID := labelNameIDMap[labelName] + labelValue := labelNameValueData["label_value"] + labelValueID := valueNameIDMap[labelValue] + keyToItem[PrometheusAPPLabelKey{LabelNameID: labelNameID, LabelValueID: labelValueID}] = mysql.ChAPPLabel{ + LabelNameID: labelNameID, + LabelValue: labelValue, + LabelValueID: labelValueID, + } } + } return keyToItem, true } @@ -101,6 +107,22 @@ func (l *ChAPPLabel) generateLabelIDNameValueData() (map[int]map[string]string, return metricLabelIDNameValueMap, true } +func (l *ChAPPLabel) generateAPPLabelData() ([]string, bool) { + appLabelSlice := []string{} + var prometheusAPPMetricAPPLabelLayouts []mysql.ChPrometheusMetricAPPLabelLayout + err := mysql.Db.Unscoped().Select("app_label_name").Group("app_label_name").Find(&prometheusAPPMetricAPPLabelLayouts).Error + + if err != nil { + log.Errorf(dbQueryResourceFailed(l.resourceTypeName, err)) + return appLabelSlice, false + } + + for _, prometheusAPPMetricAPPLabelLayout := range prometheusAPPMetricAPPLabelLayouts { + appLabelSlice = append(appLabelSlice, prometheusAPPMetricAPPLabelLayout.APPLabelName) + } + return appLabelSlice, true +} + func (l *ChAPPLabel) generateNameIDData() (map[string]int, map[string]int, bool) { labelNameIDMap := make(map[string]int) valueNameIDMap := make(map[string]int) From 55a1f67dd88615386f29a96bf9c14b762f076968 Mon Sep 17 00:00:00 2001 From: Ericsssss Date: Fri, 13 Oct 2023 17:40:28 +0800 Subject: [PATCH 10/22] [Querier] modify pod_group_type group by condition #22508 - run automation test (querier & basic) pass --- server/querier/engine/clickhouse/tag/translation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/querier/engine/clickhouse/tag/translation.go b/server/querier/engine/clickhouse/tag/translation.go index dd73d879b22..7ed4aa6623b 100644 --- a/server/querier/engine/clickhouse/tag/translation.go +++ b/server/querier/engine/clickhouse/tag/translation.go @@ -883,7 +883,7 @@ func GenerateTagResoureMap() map[string]map[string]*Tag { tagResourceMap[podGroupTypeSuffix] = map[string]*Tag{ "default": NewTag( "dictGet(flow_tag.pod_group_map, 'pod_group_type', (toUInt64("+podGroupIDSuffix+")))", - "", + podGroupIDSuffix+"!=0", "toUInt64("+podGroupIDSuffix+") IN (SELECT id FROM flow_tag.pod_group_map WHERE pod_group_type %s %s)", "", ), From 692d3bb19590446a1a98910ecb27b15d900d2cc9 Mon Sep 17 00:00:00 2001 From: huanchao Date: Sat, 7 Oct 2023 18:43:26 +0800 Subject: [PATCH 11/22] [Agent] fix init_flow and icmp srt when disorder --- agent/src/flow_generator/flow_map.rs | 10 +-- agent/src/flow_generator/perf/icmp.rs | 69 +++++++++++++------ .../metrics/flow_metrics/vtap_flow_port.ch | 4 +- .../metrics/flow_metrics/vtap_flow_port.en | 4 +- 4 files changed, 56 insertions(+), 31 deletions(-) diff --git a/agent/src/flow_generator/flow_map.rs b/agent/src/flow_generator/flow_map.rs index 444ae0e0030..6d3a1be6669 100644 --- a/agent/src/flow_generator/flow_map.rs +++ b/agent/src/flow_generator/flow_map.rs @@ -48,7 +48,6 @@ use super::{ use crate::{ common::{ - ebpf::EbpfType, endpoint::{ EndpointData, EndpointDataPov, EndpointInfo, EPC_FROM_DEEPFLOW, EPC_FROM_INTERNET, }, @@ -1009,12 +1008,6 @@ impl FlowMap { fn init_flow(&mut self, config: &Config, meta_packet: &mut MetaPacket) -> Box { let flow_config = config.flow; - match meta_packet.ebpf_type { - EbpfType::GoHttp2Uprobe | EbpfType::GoHttp2UprobeData => {} - _ => { - meta_packet.lookup_key.direction = PacketDirection::ClientToServer; - } - } let mut tagged_flow = TaggedFlow::default(); let lookup_key = &meta_packet.lookup_key; @@ -1574,6 +1567,9 @@ impl FlowMap { node.flow_state = FlowState::Established; // opening timeout node.timeout = config.flow.flow_timeout.opening; + if let Some(meta_flow_log) = node.meta_flow_log.as_mut() { + let _ = meta_flow_log.parse_l3(meta_packet); + } node } diff --git a/agent/src/flow_generator/perf/icmp.rs b/agent/src/flow_generator/perf/icmp.rs index 410060f1a31..de3e684f757 100644 --- a/agent/src/flow_generator/perf/icmp.rs +++ b/agent/src/flow_generator/perf/icmp.rs @@ -14,7 +14,7 @@ * limitations under the License. */ -use std::cmp::max; +use std::{cmp::max, collections::VecDeque}; use pnet::packet::{ icmp::{IcmpType, IcmpTypes}, @@ -45,8 +45,8 @@ pub struct IcmpPerf { srt_max: Timestamp, srt_sum: Timestamp, srt_count: u32, - last_requests: Vec, - last_replies: Vec, + last_requests: VecDeque, + last_replies: VecDeque, data_update_flag: bool, } @@ -54,6 +54,22 @@ impl IcmpPerf { pub fn new() -> Self { IcmpPerf::default() } + + fn set_srt(&mut self, srt: Timestamp) { + if srt <= ART_MAX { + self.srt_max = max(self.srt_max, srt); + self.srt_sum += srt; + self.srt_count += 1; + self.data_update_flag = true; + } + } + + fn reset(&mut self) { + self.srt_max = Timestamp::default(); + self.srt_sum = Timestamp::default(); + self.srt_count = 0; + self.data_update_flag = false; + } } impl L4FlowPerf for IcmpPerf { @@ -89,19 +105,14 @@ impl L4FlowPerf for IcmpPerf { { if pkt_timestamp <= self.last_replies[i].timestamp { let srt = Timestamp::from(self.last_replies[i].timestamp - pkt_timestamp); - if srt <= ART_MAX { - self.srt_max = max(self.srt_max, srt); - self.srt_sum += srt; - self.srt_count += 1; - self.data_update_flag = true; - } + self.set_srt(srt); } self.last_replies.remove(i); } else { if self.last_requests.len() >= MAX_CACHE_COUNT { - self.last_requests.remove(0); + let _ = self.last_requests.pop_front(); } - self.last_requests.push(LastIcmp { + self.last_requests.push_back(LastIcmp { timestamp: pkt_timestamp, id_and_seq: icmp_data.echo_id_seq, }); @@ -114,19 +125,14 @@ impl L4FlowPerf for IcmpPerf { { if pkt_timestamp >= self.last_requests[i].timestamp { let srt = Timestamp::from(pkt_timestamp - self.last_requests[i].timestamp); - if srt <= ART_MAX { - self.srt_max = max(self.srt_max, srt); - self.srt_sum += srt; - self.srt_count += 1; - self.data_update_flag = true; - } + self.set_srt(srt); } self.last_requests.remove(i); } else { if self.last_replies.len() >= MAX_CACHE_COUNT { - self.last_replies.remove(0); + let _ = self.last_replies.pop_front(); } - self.last_replies.push(LastIcmp { + self.last_replies.push_back(LastIcmp { timestamp: pkt_timestamp, id_and_seq: icmp_data.echo_id_seq, }); @@ -140,12 +146,35 @@ impl L4FlowPerf for IcmpPerf { } fn copy_and_reset_data(&mut self, _: bool) -> FlowPerfStats { + for request_index in (0..self.last_requests.len()).rev() { + let request = &self.last_requests[request_index]; + if let Some(reply_index) = self + .last_replies + .iter() + .position(|reply| request.id_and_seq == reply.id_and_seq) + { + let reply = &self.last_replies[reply_index]; + if request.timestamp <= reply.timestamp { + let srt = Timestamp::from(reply.timestamp - request.timestamp); + if srt <= ART_MAX { + self.srt_max = max(self.srt_max, srt); + self.srt_sum += srt; + self.srt_count += 1; + self.data_update_flag = true; + } + } + + self.last_requests.remove(request_index); + self.last_replies.remove(reply_index); + } + } + let mut stats = FlowPerfStats::default(); stats.l4_protocol = L4Protocol::Icmp; stats.tcp.srt_max = (self.srt_max.as_nanos() / Timestamp::from_micros(1).as_nanos()) as u32; stats.tcp.srt_sum = (self.srt_sum.as_nanos() / Timestamp::from_micros(1).as_nanos()) as u32; stats.tcp.srt_count = self.srt_count; - *self = IcmpPerf::default(); + self.reset(); stats } diff --git a/server/querier/db_descriptions/clickhouse/metrics/flow_metrics/vtap_flow_port.ch b/server/querier/db_descriptions/clickhouse/metrics/flow_metrics/vtap_flow_port.ch index 9e61bf1ce35..0f001b1a13e 100644 --- a/server/querier/db_descriptions/clickhouse/metrics/flow_metrics/vtap_flow_port.ch +++ b/server/querier/db_descriptions/clickhouse/metrics/flow_metrics/vtap_flow_port.ch @@ -64,13 +64,13 @@ server_half_close_flow , 断连-服务端半关 , 连接 , TCP 传 rtt , 平均 TCP 建连时延 , 微秒 , 统计周期内,所有 TCP 建连时延的平均值,一次时延的计算见文档描述 rtt_client , 平均 TCP 建连客户端时延 , 微秒 , 统计周期内,所有 TCP 建连客户端时延的平均值,一次时延的计算见文档描述 rtt_server , 平均 TCP 建连服务端时延 , 微秒 , 统计周期内,所有 TCP 建连服务端时延的平均值,一次时延的计算见文档描述 -srt , 平均 TCP 系统时延 , 微秒 , 统计周期内,所有 TCP 系统时延的平均值,一次时延的计算见文档描述 +srt , 平均 TCP/ICMP 系统时延 , 微秒 , 统计周期内,所有 TCP/ICMP 系统时延的平均值,一次时延的计算见文档描述 art , 平均数据时延 , 微秒 , 统计周期内,所有数据时延的平均值,数据时延包含 TCP/UDP,一次时延的计算见文档描述 cit , 平均客户端等待时延 , 微秒 , 统计周期内,所有客户端等待时延的平均值,数据时延仅包含 TCP,一次时延的计算见文档描述 rtt_max , 最大 TCP 建连时延 , 微秒 , 统计周期内,所有 TCP 建连时延的最大值,一次时延的计算见文档描述 rtt_client_max , 最大 TCP 建连客户端时延 , 微秒 , 统计周期内,所有 TCP 建连客户端时延的最大值,一次时延的计算见文档描述 rtt_server_max , 最大 TCP 建连服务端时延 , 微秒 , 统计周期内,所有 TCP 建连服务端时延的最大值,一次时延的计算见文档描述 -srt_max , 最大 TCP 系统时延 , 微秒 , 统计周期内,所有 TCP 系统时延的最大值,一次时延的计算见文档描述 +srt_max , 最大 TCP/ICMP 系统时延 , 微秒 , 统计周期内,所有 TCP/ICMP 系统时延的最大值,一次时延的计算见文档描述 art_max , 最大数据时延 , 微秒 , 统计周期内,所有数据时延的最大值,数据时延包含 TCP/UDP,一次时延的计算见文档描述 cit_max , 最大客户端等待时延 , 微秒 , 统计周期内,所有客户端等待时延的最大值,数据时延仅包含 TCP,一次时延的计算见文档描述 diff --git a/server/querier/db_descriptions/clickhouse/metrics/flow_metrics/vtap_flow_port.en b/server/querier/db_descriptions/clickhouse/metrics/flow_metrics/vtap_flow_port.en index bbb9fe0369f..5c90480107b 100644 --- a/server/querier/db_descriptions/clickhouse/metrics/flow_metrics/vtap_flow_port.en +++ b/server/querier/db_descriptions/clickhouse/metrics/flow_metrics/vtap_flow_port.en @@ -64,13 +64,13 @@ server_half_close_flow , Close - Server Half Close , Flow , rtt , Avg TCP Est. Delay , us , rtt_client , Avg TCP Est. Client Delay , us , rtt_server , Avg TCP Est. Server Delay , us , -srt , Avg TCP ACK Delay , us , +srt , Avg TCP/ICMP Response Delay , us , art , Avg Data Delay , us , cit , Avg Client Idle Delay , us , rtt_max , Max TCP Est. Delay , us , rtt_client_max , Max TCP Est. Client Delay , us , rtt_server_max , Max TCP Est. Server Delay , us , -srt_max , Max TCP ACK Delay , us , +srt_max , Max TCP/ICMP Response Delay , us , art_max , Max Data Delay , us , cit_max , Max Client Idle Delay , us , From 646f4e76f5d1bc9c765a8af6aaa1916fe211aba8 Mon Sep 17 00:00:00 2001 From: taloric Date: Wed, 11 Oct 2023 16:33:49 +0800 Subject: [PATCH 12/22] [querier] remove replica labels by query params --- .../querier/app/prometheus/config/config.go | 1 + .../app/prometheus/model/prometheus.go | 19 +++++++------ .../app/prometheus/router/prometheus.go | 5 ++-- .../app/prometheus/service/converters.go | 28 +++++++++++++------ server/server.yaml | 1 + 5 files changed, 33 insertions(+), 21 deletions(-) diff --git a/server/querier/app/prometheus/config/config.go b/server/querier/app/prometheus/config/config.go index ac14ea75056..829d3deadea 100644 --- a/server/querier/app/prometheus/config/config.go +++ b/server/querier/app/prometheus/config/config.go @@ -25,6 +25,7 @@ type Prometheus struct { RequestQueryWithDebug bool `default:"false" yaml:"request-query-with-debug"` ExternalTagCacheSize int `default:"1024" yaml:"external-tag-cache-size"` ExternalTagLoadInterval int `default:"300" yaml:"external-tag-load-interval"` + ThanosReplicaLabels []string `yaml:"thanos-replica-labels"` Cache PrometheusCache `yaml:"cache"` } diff --git a/server/querier/app/prometheus/model/prometheus.go b/server/querier/app/prometheus/model/prometheus.go index 9658a48e9bf..1504ab80776 100644 --- a/server/querier/app/prometheus/model/prometheus.go +++ b/server/querier/app/prometheus/model/prometheus.go @@ -23,15 +23,16 @@ import ( ) type PromQueryParams struct { - MetricsWithPrefix string - Promql string - StartTime string - EndTime string - Step string - Debug bool - Slimit string - Matchers []string - Context context.Context + MetricsWithPrefix string + Promql string + StartTime string + EndTime string + Step string + Debug bool + Slimit string + ThanosReplicaLabels []string + Matchers []string + Context context.Context } type PromQueryData struct { diff --git a/server/querier/app/prometheus/router/prometheus.go b/server/querier/app/prometheus/router/prometheus.go index 0ae302e9bdb..f3fc0bea873 100644 --- a/server/querier/app/prometheus/router/prometheus.go +++ b/server/querier/app/prometheus/router/prometheus.go @@ -46,6 +46,7 @@ func promQuery(svc *service.PrometheusService) gin.HandlerFunc { args.Slimit = c.Request.FormValue("slimit") debug := c.Request.FormValue("debug") args.Debug, _ = strconv.ParseBool(debug) + result, err := svc.PromInstantQueryService(&args, c.Request.Context()) if err != nil { c.JSON(500, &model.PromQueryResponse{Error: err.Error(), Status: _STATUS_FAIL}) @@ -73,7 +74,6 @@ func promQueryRange(svc *service.PrometheusService) gin.HandlerFunc { c.JSON(500, &model.PromQueryResponse{Error: err.Error(), Status: _STATUS_FAIL}) return } - //pp.Println(result) c.JSON(200, result) }) } @@ -92,9 +92,8 @@ func promReader(svc *service.PrometheusService) gin.HandlerFunc { c.JSON(500, err) return } - //pp.Println(req) + resp, err := svc.PromRemoteReadService(&req, c.Request.Context()) - //pp.Println(resp) if err != nil { c.JSON(500, err) return diff --git a/server/querier/app/prometheus/service/converters.go b/server/querier/app/prometheus/service/converters.go index 3fdebbb3c73..199ac67a3d3 100644 --- a/server/querier/app/prometheus/service/converters.go +++ b/server/querier/app/prometheus/service/converters.go @@ -592,10 +592,25 @@ func (p *prometheusReader) respTransToProm(ctx context.Context, metricsName stri // merge and serialize all tags as map key var deepflowNativeTagString, promTagJson string + var filterTagMap map[string]string // merge prometheus tags if tagIndex > -1 { promTagJson = values[tagIndex].(string) - deepflowNativeTagString = promTagJson + tagMap := make(map[string]string) + json.Unmarshal([]byte(promTagJson), &tagMap) + filterTagMap = make(map[string]string, len(tagMap)) + for k, v := range tagMap { + if k == "" || v == "" { + continue + } + // ignore replica labels if passed + if config.Cfg.Prometheus.ThanosReplicaLabels != nil && common.IsValueInSliceString(k, config.Cfg.Prometheus.ThanosReplicaLabels) { + continue + } + filterTagMap[k] = v + } + promFilterTagJson, _ := json.Marshal(filterTagMap) + deepflowNativeTagString = string(promFilterTagJson) } // merge deepflow autotagging tags @@ -622,14 +637,9 @@ func (p *prometheusReader) respTransToProm(ctx context.Context, metricsName stri // tag label pair var pairs []prompb.Label - if tagIndex > -1 { - tagMap := make(map[string]string) - json.Unmarshal([]byte(promTagJson), &tagMap) - pairs = make([]prompb.Label, 0, 1+len(tagMap)+len(allDeepFlowNativeTags)) - for k, v := range tagMap { - if k == "" || v == "" { - continue - } + if tagIndex > -1 && filterTagMap != nil { + pairs = make([]prompb.Label, 0, 1+len(filterTagMap)+len(allDeepFlowNativeTags)) + for k, v := range filterTagMap { if prefix == prefixTag { // prometheus tag for deepflow metrics pairs = append(pairs, prompb.Label{ diff --git a/server/server.yaml b/server/server.yaml index 273fef7881d..07bf4aeb460 100644 --- a/server/server.yaml +++ b/server/server.yaml @@ -280,6 +280,7 @@ querier: request-query-with-debug: true external-tag-cache-size: 1024 external-tag-load-interval: 300 + thanos-replica-labels: [] # remove duplicate replica labels when query data cache: enabled: true cache-item-size: 512000 # max size of cache item, unit: byte From 132b7ebb43bc982788e9433ed21503d471a1963e Mon Sep 17 00:00:00 2001 From: taloric Date: Fri, 13 Oct 2023 18:08:42 +0800 Subject: [PATCH 13/22] [querier] remove extend end time --- server/querier/app/prometheus/service/promql.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/querier/app/prometheus/service/promql.go b/server/querier/app/prometheus/service/promql.go index d999e7af303..a6c86aa5208 100644 --- a/server/querier/app/prometheus/service/promql.go +++ b/server/querier/app/prometheus/service/promql.go @@ -197,14 +197,9 @@ func (p *prometheusExecutor) promQueryRangeExecute(ctx context.Context, args *mo if start.Local().Unix()%int64(step.Seconds()) > int64(p.lookbackDelta.Seconds()) { start = time.Unix(start.Local().Unix()-start.Local().Unix()%int64(step.Seconds()), 0) } - if end.Local().Unix()%int64(step.Seconds()) > int64(p.lookbackDelta.Seconds()) { - end = time.Unix(end.Local().Unix()-end.Local().Unix()%int64(step.Seconds())+int64(step.Seconds()), 0) - } if int(step.Seconds())%86400 == 0 { year_start, month_start, day_start := start.Date() - year_end, month_end, day_end := end.Date() start = time.Date(year_start, month_start, day_start, 0, 0, 0, 0, start.Location()) - end = time.Date(year_end, month_end, day_end, 0, 0, 0, 0, end.Location()) } qry, err := engine.NewRangeQuery(queriable, nil, args.Promql, start, end, step) if qry == nil || err != nil { From b03278796b4dfe9c8fabc227d1fe8b7204ed33c0 Mon Sep 17 00:00:00 2001 From: yuanchao Date: Fri, 13 Oct 2023 19:55:58 +0800 Subject: [PATCH 14/22] [Agent] Fix mongo log request resource is null #22509 --- .../test/flow_generator/mongo/mongo.pcap | Bin 0 -> 15158 bytes .../test/flow_generator/mongo/mongo.result | 30 ++++ .../flow_generator/protocol_logs/sql/mongo.rs | 144 +++++++++++++++--- 3 files changed, 154 insertions(+), 20 deletions(-) create mode 100644 agent/resources/test/flow_generator/mongo/mongo.pcap create mode 100644 agent/resources/test/flow_generator/mongo/mongo.result diff --git a/agent/resources/test/flow_generator/mongo/mongo.pcap b/agent/resources/test/flow_generator/mongo/mongo.pcap new file mode 100644 index 0000000000000000000000000000000000000000..d5be4b29127d9e970350c0a89c580760a3e516ff GIT binary patch literal 15158 zcmeHO3ve98neKfU8?YT2Oe|wUO@wnYwpQ9*JuC~$_G-0j$(D9)?MgBRBW8DIS7Xi2 ztY>ERFckvHT@tu5PUQgeGByw62bkE#F&N@19?2y~D(A|bO+g|^6;*`GaiJ)ZkSpRN z_x-)Ak$0sNT#Cz4g+|q@o$j8mzyANfyZ^s?_RLGi9&$1_8^7Glft!yORLHN^yVyE> zr}W_$R~_rE!RLR@ymb}}F}AX@X$@O*!MQJ7IKG} z?it73&RG|A|Mnl^l<#o4W)kvO8j6sCna9r~Ge*c)ts_)IS8hA-z8XIHYoP0UFWDw6 zcxW7X-h4So=wryK)0wG2%HAZs9sJGyg@DGKyBcIGK_7J=Hjik21yV6irJu zxI$cJR!-(r(NYHGK+fuyHA_i~mdv)JdYLJjYRnQ1i`R8#`eju`P02tv4R(!PsT|+NDz$46iYBT;Fpup-Rg*ADo^{oA)&6=w z=qB4ik6R;InX~RtwA^r^&99&&2wu^cz_SHIR%OwYh2_;Ns{LN?3Smi5H{`3CQ#6wO ziX|tloFVfDh8r9D8tRzc=v9DP4?eOg)K>ev)n1R!*Wj&g^j7<-Wp#OVp+0JnHT?+I zbe??{TYlZ>qhw9t!zGsg_WqqG24J7>EVwssO|Z>YSbaAkzSQhuH^V+D<;UmvSaN1Z z_Yq=ic2dN0_RhkyHe!enR{~`vxY9+2IE#N9|De2b)R_h2Dj28B(zCj%r$>4UY9Pa! znKksJY?|#7lPfNLHvCfkhf9Pbqn9N09BFSHuq-1Ex0t#XO>BjbW6E}!${dv4iRZ<% zOnEIMmTWSnjykEJ*MLS^L@U|P>+k)hgI%cVNl}d}uqXFP&z!%f@dHkx&|rP138C!?BhpoqC;NS)7TNIF*a-r zWj3`+iH44dzgyp|rQ(Uf!E}|`Y^_^Auy$Z;?S^eZ#q3oFgH}zOe^W!SN9%0V!ad!? z8!~!Jpt-q9Y4+87yMEQ0{5!vRwmdPQhJSDs?E78$ z^}IF05_@6eA+qmN9d5P=-KJDJ|2pPWt|G)gHBE|G&fZPDY$HBEh_8Vv$&jwsF<U&x{LSlC=*4U^w zW|M8Hts9s1Yz((c#&5AWW+g+%3yxYHDgT(y7a)2+9Zv)`3po);CwbUPIAz^y=OMj{c}qUy)ZWRzx=*km>8vBBnabe$U7 zWcZf%YTeex#;w`fny&Z?wNYUI69$Xud)ZnK-+g{LS@(CfEG_jA4Wmgx0S=yf0I zb-{nwdcA`L5d+F9B%*CoZo>zJ>afCdh1L^Z=KcR3fuKLS}DrnY~dYv%MjptVSYAHk~TmI7jpxy98gb^4HUn z@2hQWs;l+am&}cS?(+POXkB@Wn|&SCDNT@)RoTL&bh<=JZ>r%_D@jU^Kr96*9V98O z0m_)~ZquqnrX!J4R7ufNI&SECZ*{4ln!Mf%iP4*2v;{LErBXr7jE){8Myq!e3+mL$ z{X3tjAVFPu+!oX>7}vo?Kv{$Kh?ULAiA>vH!uK&J&6!rLv6c2}WY@~R#$;WxzQJ49 zC`l>5UuX^~{~orJRaYd|M}TM%B;l)E|;E_HPcktjks|dIZacvG!t+WML{gqA^56c6|yI#WKF^66T)&r z@Oyo=%$d|PSw%&YpQJW(afODoLB-Iu40+Ux(08Y#m=AH$ALD6akYikntmrf>|1bhff)&_%D3yszF)qqPTQ>vIYSb!i)kRR7-mj5FGV2Wm;l#0A0{2w9pL?LJjIaNg)$bwAWcvP9}pg8th`?|`@wER zm7CLAE?1FSJCXv3Os+ ztt%90?du73#oD70p}sg9?Z*0A1MyJ2y)#rOid`FvhdTQr(Z1HuhOST$rLBdc&Op44 z3Ob^ZwSBG8?v{>FAI}SSM}l$S#|jlgkpPtiVzE$H95~?L*A|Gtx$P>{=!mW*q*xnh zQ%PSW&>0$goC--K5%j+Ya=12zobw*+ZI8D_yY0@weeMzhEwK&~LJ-Y%w6BeXTKfXQ zU?>LnrYxaYCSQhTNpd2WM$Y6>rrX5WLx`DQ9!%B&#UXB^B`Tq$(~3+X8->7_D+ge?zr^H=WN8o za42?VfWp^Q{I{2H+PZVX^u&F4A7rK8$L2X#yh8L7JKby;UKOQM@58lq^l_peEZM#IVK2Ooo#cI7e%kgv z?uYkr7~6_0fcB{iHvKOlv&_;l^<86%lyiNYbCZDf*h1b8#+X&~W;;U5^CMxw-R0%>w65oPRjOb_w zzgMJ+Ccoe3_o5J`)-cLxb`>>A#hNW^6;%}(GmYFplJ-dizN+e~pqPTDTS9)?k%SbM zGHmtTadE>{*p#~mV%>ds>tI1?);9_VdE&7_(*PlzvSq#&fF{z+W zm7+Q%j+m9zLa!{CxpW#m6tM7O;YK+tPtzTRQ1d!#{ZewzTFjVz|O0cLcVyI|&q8 zkm<6uKu1#f7?%feyb#9p7nsi6m#=UT>D%s~G7LUUq*vcoY&t)A0j85G349=WFE2Hv1QR*8}TT7jt3Q>P}onG zy?k%LjRRqDFe6bQ8O>skA%FcZ91rp(3$ITckFM?^>No9iv)ABNQkvkj{&e(vL_KtI zvFCAm6rRUv@;nxL^VC6m4?K^fyMWS)Mu@&m{bppkv28V&Klt(TAiXAM8hqb3zaLDo zA0lPuNsaAt)W}w{uvuJ9F(tOU5s+asm3^2gztMHmYr>G&mDxM zZ&)KeBPBM7RzLd!)yCFrjjAN_?3rt;o}(8=c-gE%5J{&;F2tOcoS}`4P7Z-+LecUs zs8Do~!UjE?g-wD$7TftaeRpQ5i7V_T^z_(n`k0_-M*$i+|Ag!rEAlug+I7IqZbc!b z2{vk0wsRZ(rxL3_t%iSY!Y?*2xZ1YSr%2g!90gaIU$&zE2Dg*=#eDaLI+PEW{&~|i z-zGx;0zzBhqEafgJRJQTAwsLaS!{VfH}~&+c5aE!`!Fo;De`$kXoE=Egrr+waOpZD|W^WJl?`j@oWe(lhdKJPkOY|qUsUj3h)tA<~_13vF}TH$LJeBR?& z{U0X8pFQDb-^F4}X@WoTx0UO;KXGDG#B#@Xr00tf$-TV@i7tEjUXGhjvS(Z3VEm(b zq>dO~e|XCM*ki;nG*Ikdym|*5j2lY!V;_Noah&#J!yrT~ZHAxm{n*6+<9_VyUn>%6 z;r3&0c0XiBX@Z^XsO;c&@T9pz0`5{n2qZdmdPnN9eti%V)2T)HiAdfX+1E60{?HOayy5cM zQ+XpB_{4N{Qp9q{DQ53(?7M~#$s3u2M3+6X|3YtMv3C2t#BkA~*;Abkf1enJ`itkD z*9S58SjFDR&{E7j*N`{T4tB)SX80NRMt<}1U-hRk~%YYuxDTA(z+zqzxri(B5FNfFB(XWsUW+452R z=5J`}jX=S6Fu!cQk{98|4q)N8L?)w+HkbEfUV57F*3+Y}Jl9N&58gM2sX(AK!Ogj0 z^aL>um%OL&#vxeZ8M4F$FXb7}oBs{`3hZe2QlP9uBGxuzdQGiJ{%?Q#If2@K|Nc2s zjr _HEADER_SIZE => { + _OP_MSG if payload.len() > _MSG_DOC_SECTION_OFFSET => { // OP_MSG let mut msg_body = MongoOpMsg::default(); - msg_body.decode(&payload[_HEADER_SIZE..])?; + // TODO: Message Flags + msg_body.decode(&payload[_MSG_DOC_SECTION_OFFSET..])?; match info.msg_type { LogMessageType::Response => { - info.response = msg_body.sections.doc.to_string(); - info.exception = msg_body.sections.c_string.unwrap_or(_UNKNOWN.to_string()); - info.response_code = - msg_body.sections.doc.get_f64("code").unwrap_or(0.0) as i32; + // The data structure of doc is Bson, which is a normal response when there is no errmsg in it + if msg_body.sections.doc.get_str("errmsg").is_err() { + info.response = msg_body.sections.doc.to_string(); + } else { + info.exception = + msg_body.sections.doc.get_str("errmsg").unwrap().to_string(); + } + if info.exception.len() == 0 { + info.exception = + msg_body.sections.c_string.unwrap_or(_UNKNOWN.to_string()); + } + info.response_code = msg_body.sections.doc.get_i32("code").unwrap_or(0); if info.response_code > 0 { self.perf_stats.as_mut().map(|p| p.inc_resp_err()); } @@ -315,13 +327,16 @@ impl MongoDBLog { } _OP_QUERY if payload.len() > 28 => { // "OP_QUERY" - info.exception = CStr::from_bytes_until_nul(&payload[..20]) - .map_err(|_| Error::L7ProtocolUnknown)? - .to_string_lossy() - .into_owned(); + let collection_name = + CStr::from_bytes_until_nul(&payload[_COLLECTION_NAME_OFFSET..]) + .map_err(|_| Error::L7ProtocolUnknown)? + .to_string_lossy() + .into_owned(); - let query = Document::from_reader(&payload[28 + info.exception.len() + 1..]) - .unwrap_or(Document::default()); + let query = Document::from_reader( + &payload[_QUERY_DOC_OFFSET + collection_name.len() + 1..], + ) + .unwrap_or(Document::default()); info.request = query.to_string(); } _OP_GET_MORE | _OP_DELETE if payload.len() > 20 => { @@ -415,18 +430,24 @@ pub struct MongoOpMsg { } impl MongoOpMsg { + const _KIND_OFFSET: usize = 0; + const _KIND_LEN: usize = 1; + const _DOC_LENGTH_OFFSET: usize = Self::_KIND_OFFSET + Self::_KIND_LEN; + const _DOC_LENGTH_LEN: usize = 4; + fn decode(&mut self, payload: &[u8]) -> Result { - if payload.len() < 9 { + if payload.len() < Self::_DOC_LENGTH_OFFSET + Self::_DOC_LENGTH_LEN { return Ok(false); } - // todo: decode flag let mut sections = Sections::default(); //sections.kind = payload[4]; - let section_len = bytes::read_u32_le(&payload[5..9]); - if payload.len() < 4 + section_len as usize { + let section_len = bytes::read_u32_le( + &payload[Self::_DOC_LENGTH_OFFSET..Self::_DOC_LENGTH_OFFSET + Self::_DOC_LENGTH_LEN], + ); + if payload.len() < Self::_DOC_LENGTH_LEN + section_len as usize { return Ok(false); } - let _ = sections.decode(&payload[4..]); + let _ = sections.decode(&payload); self.sections = sections; // todo: decode checksum Ok(true) @@ -455,8 +476,9 @@ impl Sections { 0 => { // Body self.kind_name = "BODY".to_string(); - let lenght = bytes::read_u32_le(&payload[1..5]); - if lenght != payload.len() as u32 - 1 { + let length = bytes::read_u32_le(&payload[1..5]); + // TODO: When ChecksumPresent is 1, there will be checksum in the payload + if length != payload.len() as u32 - 1 && length != payload.len() as u32 - 5 { return Ok(false); } self.doc = Document::from_reader(&payload[1..]).unwrap_or(Document::default()); @@ -567,3 +589,85 @@ pub struct MongoOpUpdate { zero: u32, } */ + +#[cfg(test)] +mod tests { + use std::path::Path; + use std::rc::Rc; + use std::{cell::RefCell, fs}; + + use super::*; + + use crate::{ + common::{flow::PacketDirection, l7_protocol_log::L7PerfCache, MetaPacket}, + flow_generator::L7_RRT_CACHE_CAPACITY, + utils::test::Capture, + }; + + const FILE_DIR: &str = "resources/test/flow_generator/mongo"; + + fn run(name: &str) -> String { + let capture = Capture::load_pcap(Path::new(FILE_DIR).join(name), None); + let log_cache = Rc::new(RefCell::new(L7PerfCache::new(L7_RRT_CACHE_CAPACITY))); + let mut packets = capture.as_meta_packets(); + if packets.is_empty() { + return "".to_string(); + } + + let mut output: String = String::new(); + let first_dst_port = packets[0].lookup_key.dst_port; + for packet in packets.iter_mut() { + packet.lookup_key.direction = if packet.lookup_key.dst_port == first_dst_port { + PacketDirection::ClientToServer + } else { + PacketDirection::ServerToClient + }; + let payload = match packet.get_l4_payload() { + Some(p) => p, + None => continue, + }; + + let mut mongo = MongoDBLog::default(); + let param = &ParseParam::new(packet as &MetaPacket, log_cache.clone(), true, true); + + let is_mongo = mongo.check_payload(payload, param); + let info = mongo.parse_payload(payload, param); + if let Ok(info) = info { + match info.unwrap_single() { + L7ProtocolInfo::MongoDBInfo(i) => { + output.push_str(&format!("{:?} is_mongo: {}\r\n", i, is_mongo)); + } + _ => unreachable!(), + } + } else { + output.push_str(&format!( + "{:?} is_mongo: {}\r\n", + MongoDBInfo::default(), + is_mongo + )); + } + } + output + } + + #[test] + fn check() { + let files = vec![("mongo.pcap", "mongo.result")]; + + for item in files.iter() { + let expected = fs::read_to_string(&Path::new(FILE_DIR).join(item.1)).unwrap(); + let output = run(item.0); + + if output != expected { + let output_path = Path::new("actual.txt"); + fs::write(&output_path, &output).unwrap(); + assert!( + output == expected, + "output different from expected {}, written to {:?}", + item.1, + output_path + ); + } + } + } +} From 76b61afade27de85d95800292fb260a8f8f0864c Mon Sep 17 00:00:00 2001 From: xiaoziv Date: Wed, 11 Oct 2023 07:54:13 +0000 Subject: [PATCH 15/22] [Agent] refactor optimize code --- agent/src/collector/l7_quadruple_generator.rs | 59 +++++++++---------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/agent/src/collector/l7_quadruple_generator.rs b/agent/src/collector/l7_quadruple_generator.rs index c1705224953..afb258db44a 100644 --- a/agent/src/collector/l7_quadruple_generator.rs +++ b/agent/src/collector/l7_quadruple_generator.rs @@ -270,7 +270,7 @@ impl SubQuadGen { flow: flow.clone(), l7_protocol: meter.l7_protocol, endpoint_hash, - endpoint: meter.endpoint.clone(), + endpoint: meter.endpoint, is_active_host0, is_active_host1, time_in_second: tagged_flow.flow.flow_stat_time, @@ -549,8 +549,8 @@ impl L7QuadrupleGenerator { ) { let mut second_inject = false; let mut minute_inject = false; - if let Some(s) = self.second_quad_gen.as_mut() { - if config.vtap_flow_1s_enabled { + if config.vtap_flow_1s_enabled { + if let Some(s) = self.second_quad_gen.as_mut() { second_inject = s.move_window(time_in_second); } } @@ -565,7 +565,11 @@ impl L7QuadrupleGenerator { let endpoint_hash = hash_endpoint(&l7_stats.endpoint) as u32; - let app_meter = Self::generate_app_meter(&l7_stats, config.l7_metrics_enabled); + let app_meter = if config.l7_metrics_enabled { + Self::generate_app_meter(&l7_stats) + } else { + AppMeter::default() + }; if second_inject { self.second_quad_gen.as_mut().unwrap().inject_app_meter( @@ -588,12 +592,7 @@ impl L7QuadrupleGenerator { } } - fn generate_app_meter(l7_stats: &L7Stats, l7_metrics_enabled: bool) -> AppMeter { - let mut app_meter = AppMeter::default(); - - if !l7_metrics_enabled { - return app_meter; - } + fn generate_app_meter(l7_stats: &L7Stats) -> AppMeter { let (close_type, direction_score) = if let Some(tagged_flow) = &l7_stats.flow { ( tagged_flow.flow.close_type, @@ -609,7 +608,7 @@ impl L7QuadrupleGenerator { SignalSource::Packet | SignalSource::EBPF | SignalSource::XFlow, ) => { // only L7Protocol is Unknown or Other and SignalSource != Otel will execute the following logic - app_meter = AppMeter { + AppMeter { traffic: AppTraffic { request: (close_type != CloseType::ForcedReport) as u32, response: (close_type != CloseType::ForcedReport) as u32, @@ -618,28 +617,24 @@ impl L7QuadrupleGenerator { ..Default::default() } } - (_, _) => { - app_meter = AppMeter { - traffic: AppTraffic { - request: stats.request_count, - response: stats.response_count, - direction_score: direction_score, - }, - latency: AppLatency { - rrt_max: stats.rrt_max, - rrt_sum: stats.rrt_sum as u64, - rrt_count: stats.rrt_count, - }, - anomaly: AppAnomaly { - client_error: stats.err_client_count, - server_error: stats.err_server_count, - timeout: stats.err_timeout, - }, - }; - } + (_, _) => AppMeter { + traffic: AppTraffic { + request: stats.request_count, + response: stats.response_count, + direction_score: direction_score, + }, + latency: AppLatency { + rrt_max: stats.rrt_max, + rrt_sum: stats.rrt_sum as u64, + rrt_count: stats.rrt_count, + }, + anomaly: AppAnomaly { + client_error: stats.err_client_count, + server_error: stats.err_server_count, + timeout: stats.err_timeout, + }, + }, } - - app_meter } fn handler_routine(&mut self) { From aa524d48bb57deeb92180e9151fb07c3fb821d28 Mon Sep 17 00:00:00 2001 From: zhuofeng Date: Thu, 12 Oct 2023 15:37:18 +0800 Subject: [PATCH 16/22] [ingester] l7_flow_log store syscall_coroutine_0/syscall_coroutine_1 --- server/go.mod | 2 +- server/go.sum | 4 ++-- server/ingester/ckissu/ckissu.go | 11 ++++++++++- server/ingester/common/const.go | 2 +- server/ingester/flow_log/log_data/l7_flow_log.go | 8 ++++++++ .../clickhouse/tag/flow_log/l7_flow_log | 2 ++ .../clickhouse/tag/flow_log/l7_flow_log.ch | 2 ++ .../clickhouse/tag/flow_log/l7_flow_log.en | 2 ++ 8 files changed, 28 insertions(+), 5 deletions(-) diff --git a/server/go.mod b/server/go.mod index 09a1ba3420a..2adde37171f 100644 --- a/server/go.mod +++ b/server/go.mod @@ -35,7 +35,7 @@ require ( github.com/cornelk/hashmap v1.0.8 github.com/deckarep/golang-set v1.8.0 github.com/deckarep/golang-set/v2 v2.1.0 - github.com/deepflowio/deepflow/message v0.0.0-20230927071530-6d75b95973e7 + github.com/deepflowio/deepflow/message v0.0.0-20231009120220-2f68ce239be3 github.com/deepflowio/deepflow/server/controller/cloud/kubernetes_gather/expand v0.0.0-00010101000000-000000000000 github.com/deepflowio/deepflow/server/controller/cloud/platform v0.0.0-00010101000000-000000000000 github.com/deepflowio/deepflow/server/controller/db/mysql/migrator v0.0.0-00010101000000-000000000000 diff --git a/server/go.sum b/server/go.sum index c9b053f6241..04229db40af 100644 --- a/server/go.sum +++ b/server/go.sum @@ -153,8 +153,8 @@ github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsP github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI= github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= -github.com/deepflowio/deepflow/message v0.0.0-20230927071530-6d75b95973e7 h1:rrvioA174n8n6g0Jk24vtb+8o1dpvT4fP8ASQfSPJEc= -github.com/deepflowio/deepflow/message v0.0.0-20230927071530-6d75b95973e7/go.mod h1:e+1lUMMlycCvFRKvlwt/y/0vxJnF8wVss3GyR1ARXY0= +github.com/deepflowio/deepflow/message v0.0.0-20231009120220-2f68ce239be3 h1:sR2fwjKCZvK3iP5WCfaoscD6mrPohlXf2r/JSRG3b18= +github.com/deepflowio/deepflow/message v0.0.0-20231009120220-2f68ce239be3/go.mod h1:e+1lUMMlycCvFRKvlwt/y/0vxJnF8wVss3GyR1ARXY0= github.com/deepflowio/tempopb v0.0.0-20230215110519-15853baf3a79 h1:erRwXyZiUZxxZX/Q1QHesZXgxjiunZUFy+ggCRimkD4= github.com/deepflowio/tempopb v0.0.0-20230215110519-15853baf3a79/go.mod h1:h2rkZ319TExIUGuK8a2dlcGd8qc6wdhsrcpXWaJQaQE= github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= diff --git a/server/ingester/ckissu/ckissu.go b/server/ingester/ckissu/ckissu.go index b0efbe3d9f0..defe0985c88 100644 --- a/server/ingester/ckissu/ckissu.go +++ b/server/ingester/ckissu/ckissu.go @@ -950,6 +950,15 @@ var ColumnDrops635 = []*ColumnDrops{ }, } +var ColumnAdd64 = []*ColumnAdds{ + &ColumnAdds{ + Dbs: []string{"flow_log"}, + Tables: []string{"l7_flow_log", "l7_flow_log_local"}, + ColumnNames: []string{"syscall_coroutine_0", "syscall_coroutine_1"}, + ColumnType: ckdb.UInt64, + }, +} + func getTables(connect *sql.DB, db, tableName string) ([]string, error) { sql := fmt.Sprintf("SHOW TABLES IN %s", db) rows, err := connect.Query(sql) @@ -1330,7 +1339,7 @@ func NewCKIssu(cfg *config.Config) (*Issu, error) { datasourceInfo: make(map[string]*DatasourceInfo), } - allVersionAdds := [][]*ColumnAdds{ColumnAdd610, ColumnAdd611, ColumnAdd612, ColumnAdd613, ColumnAdd615, ColumnAdd618, ColumnAdd620, ColumnAdd623, ColumnAdd625, ColumnAdd626, ColumnAdd633, ColumnAdd635} + allVersionAdds := [][]*ColumnAdds{ColumnAdd610, ColumnAdd611, ColumnAdd612, ColumnAdd613, ColumnAdd615, ColumnAdd618, ColumnAdd620, ColumnAdd623, ColumnAdd625, ColumnAdd626, ColumnAdd633, ColumnAdd635, ColumnAdd64} i.columnAdds = []*ColumnAdd{} for _, versionAdd := range allVersionAdds { for _, adds := range versionAdd { diff --git a/server/ingester/common/const.go b/server/ingester/common/const.go index 2399751b2dc..94774dea735 100644 --- a/server/ingester/common/const.go +++ b/server/ingester/common/const.go @@ -17,6 +17,6 @@ package common const ( - CK_VERSION = "v6.3.8.2" // 用于表示clickhouse的表版本号 + CK_VERSION = "v6.3.4.0" // 用于表示clickhouse的表版本号 DEFAULT_PCAP_DATA_PATH = "/var/lib/pcap" ) diff --git a/server/ingester/flow_log/log_data/l7_flow_log.go b/server/ingester/flow_log/log_data/l7_flow_log.go index 3ade31b9e2d..97a195b1ccc 100644 --- a/server/ingester/flow_log/log_data/l7_flow_log.go +++ b/server/ingester/flow_log/log_data/l7_flow_log.go @@ -81,6 +81,8 @@ type L7Base struct { SyscallTraceIDResponse uint64 SyscallThread0 uint32 SyscallThread1 uint32 + SyscallCoroutine0 uint64 + SyscallCoroutine1 uint64 SyscallCapSeq0 uint32 SyscallCapSeq1 uint32 } @@ -130,6 +132,8 @@ func L7BaseColumns() []*ckdb.Column { ckdb.NewColumn("syscall_trace_id_response", ckdb.UInt64).SetComment("SyscallTraceID-响应"), ckdb.NewColumn("syscall_thread_0", ckdb.UInt32).SetComment("Syscall线程-请求"), ckdb.NewColumn("syscall_thread_1", ckdb.UInt32).SetComment("Syscall线程-响应"), + ckdb.NewColumn("syscall_coroutine_0", ckdb.UInt64).SetComment("Request Syscall Coroutine"), + ckdb.NewColumn("syscall_coroutine_1", ckdb.UInt64).SetComment("Response Syscall Coroutine"), ckdb.NewColumn("syscall_cap_seq_0", ckdb.UInt32).SetComment("Syscall序列号-请求"), ckdb.NewColumn("syscall_cap_seq_1", ckdb.UInt32).SetComment("Syscall序列号-响应"), ) @@ -177,6 +181,8 @@ func (f *L7Base) WriteBlock(block *ckdb.Block) { f.SyscallTraceIDResponse, f.SyscallThread0, f.SyscallThread1, + f.SyscallCoroutine0, + f.SyscallCoroutine1, f.SyscallCapSeq0, f.SyscallCapSeq1) } @@ -538,6 +544,8 @@ func (b *L7Base) Fill(log *pb.AppProtoLogsData, platformData *grpc.PlatformInfoT b.SyscallTraceIDResponse = l.SyscallTraceIdResponse b.SyscallThread0 = l.SyscallTraceIdThread_0 b.SyscallThread1 = l.SyscallTraceIdThread_1 + b.SyscallCoroutine0 = l.SyscallCoroutine_0 + b.SyscallCoroutine1 = l.SyscallCoroutine_1 b.SyscallCapSeq0 = l.SyscallCapSeq_0 b.SyscallCapSeq1 = l.SyscallCapSeq_1 diff --git a/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log b/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log index fde27722e46..81f6f724b97 100644 --- a/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log +++ b/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log @@ -90,6 +90,8 @@ syscall_trace_id_request , syscall_trace_id_request , syscall_trace_id_request syscall_trace_id_response , syscall_trace_id_response , syscall_trace_id_response , int , , Tracing Info , 111 syscall_thread_0 , syscall_thread_0 , syscall_thread_0 , int , , Tracing Info , 111 syscall_thread_1 , syscall_thread_1 , syscall_thread_1 , int , , Tracing Info , 111 +syscall_coroutine_0 , syscall_coroutine_0 , syscall_coroutine_0 , int , , Tracing Info , 111 +syscall_coroutine_1 , syscall_coroutine_1 , syscall_coroutine_1 , int , , Tracing Info , 111 syscall_cap_seq_0 , syscall_cap_seq_0 , syscall_cap_seq_0 , int , , Tracing Info , 111 syscall_cap_seq_1 , syscall_cap_seq_1 , syscall_cap_seq_1 , int , , Tracing Info , 111 diff --git a/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log.ch b/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log.ch index 91a4fc251cf..92c59365c5f 100644 --- a/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log.ch +++ b/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log.ch @@ -90,6 +90,8 @@ syscall_trace_id_request , 请求 Syscall TraceID , syscall_trace_id_response , 响应 Syscall TraceID , syscall_thread_0 , 请求 Syscall 线程 , syscall_thread_1 , 响应 Syscall 线程 , +syscall_coroutine_0 , 请求 Syscall 协程 , +syscall_coroutine_1 , 响应 Syscall 协程 , syscall_cap_seq_0 , 请求 Syscall 序号 , syscall_cap_seq_1 , 响应 Syscall 序号 , diff --git a/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log.en b/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log.en index 60cf12ef192..539a533e722 100644 --- a/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log.en +++ b/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log.en @@ -90,6 +90,8 @@ syscall_trace_id_request , Req Syscall TraceID , syscall_trace_id_response , Resp Syscall TraceID , syscall_thread_0 , Req Syscall Thread , syscall_thread_1 , Resp Syscall Thread , +syscall_coroutine_0 , Req Syscall Coroutine , +syscall_coroutine_1 , Resp Syscall Coroutine , syscall_cap_seq_0 , Req Syscall CapSeq , syscall_cap_seq_1 , Resp Syscall CapSeq , From a9ec76940ac8d5b98e711b906836c81a570f6728 Mon Sep 17 00:00:00 2001 From: zhuofeng Date: Mon, 16 Oct 2023 15:41:24 +0800 Subject: [PATCH 17/22] [ingester] fix deepflow-stats name empty may cause panic --- server/ingester/ext_metrics/decoder/decoder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/ingester/ext_metrics/decoder/decoder.go b/server/ingester/ext_metrics/decoder/decoder.go index 40c0f550a6d..92b3e5d282c 100644 --- a/server/ingester/ext_metrics/decoder/decoder.go +++ b/server/ingester/ext_metrics/decoder/decoder.go @@ -194,7 +194,7 @@ func (d *Decoder) handleDeepflowStats(vtapID uint16, decoder *codec.SimpleDecode d.counter.ErrorCount++ return } - if err := pbStats.Unmarshal(bytes); err != nil { + if err := pbStats.Unmarshal(bytes); err != nil || pbStats.Name == "" { if d.counter.ErrorCount == 0 { log.Warningf("deepflow stats parse failed, err msg: %s", err) } From 3d11340de20c43518e00216704f068f892d30a29 Mon Sep 17 00:00:00 2001 From: Ericsssss Date: Mon, 16 Oct 2023 16:44:00 +0800 Subject: [PATCH 18/22] [Querier] modify pod_group_type showtagvalues error #22514 - run automation test(basic & querier_sql) pass --- server/controller/tagrecorder/const.go | 12 ++--- server/querier/engine/clickhouse/tag/const.go | 53 ++++++++++++------- .../engine/clickhouse/tag/description.go | 4 ++ 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/server/controller/tagrecorder/const.go b/server/controller/tagrecorder/const.go index 1949b4fac3d..b7dca15e280 100644 --- a/server/controller/tagrecorder/const.go +++ b/server/controller/tagrecorder/const.go @@ -756,12 +756,12 @@ var RESOURCE_TYPE_TO_NODE_TYPE = map[int]string{ common.VIF_DEVICE_TYPE_POD_GROUP: RESOURCE_TYPE_POD_GROUP, common.VIF_DEVICE_TYPE_SERVICE: RESOURCE_TYPE_SERVICE, common.VIF_DEVICE_TYPE_GPROCESS: RESOURCE_TYPE_GPROCESS, - common.VIF_DEVICE_TYPE_POD_GROUP_DEPLOYMENT: RESOURCE_TYPE_CH_POD_GROUP_DEPLOYMENT, - common.VIF_DEVICE_TYPE_POD_GROUP_STATEFULSET: RESOURCE_TYPE_CH_POD_GROUP_STATEFULSET, - common.VIF_DEVICE_TYPE_POD_GROUP_RC: RESOURCE_TYPE_CH_POD_GROUP_RC, - common.VIF_DEVICE_TYPE_POD_GROUP_DAEMON_SET: RESOURCE_TYPE_CH_POD_GROUP_DAEMON_SET, - common.VIF_DEVICE_TYPE_POD_GROUP_REPLICASET_CONTROLLER: RESOURCE_TYPE_CH_POD_GROUP_REPLICASET_CONTROLLER, - common.VIF_DEVICE_TYPE_POD_GROUP_CLONESET: RESOURCE_TYPE_CH_POD_GROUP_CLONESET, + common.VIF_DEVICE_TYPE_POD_GROUP_DEPLOYMENT: RESOURCE_TYPE_POD_GROUP, + common.VIF_DEVICE_TYPE_POD_GROUP_STATEFULSET: RESOURCE_TYPE_POD_GROUP, + common.VIF_DEVICE_TYPE_POD_GROUP_RC: RESOURCE_TYPE_POD_GROUP, + common.VIF_DEVICE_TYPE_POD_GROUP_DAEMON_SET: RESOURCE_TYPE_POD_GROUP, + common.VIF_DEVICE_TYPE_POD_GROUP_REPLICASET_CONTROLLER: RESOURCE_TYPE_POD_GROUP, + common.VIF_DEVICE_TYPE_POD_GROUP_CLONESET: RESOURCE_TYPE_POD_GROUP, common.VIF_DEVICE_TYPE_IP: RESOURCE_TYPE_IP, } diff --git a/server/querier/engine/clickhouse/tag/const.go b/server/querier/engine/clickhouse/tag/const.go index af4068d1d16..7ccf80bc19a 100644 --- a/server/querier/engine/clickhouse/tag/const.go +++ b/server/querier/engine/clickhouse/tag/const.go @@ -17,22 +17,28 @@ package tag const ( - VIF_DEVICE_TYPE_INTERNET = 0 - VIF_DEVICE_TYPE_VM = 1 - VIF_DEVICE_TYPE_VROUTER = 5 - VIF_DEVICE_TYPE_HOST = 6 - VIF_DEVICE_TYPE_DHCP_PORT = 9 - VIF_DEVICE_TYPE_POD = 10 - VIF_DEVICE_TYPE_POD_SERVICE = 11 - VIF_DEVICE_TYPE_REDIS_INSTANCE = 12 - VIF_DEVICE_TYPE_RDS_INSTANCE = 13 - VIF_DEVICE_TYPE_POD_NODE = 14 - VIF_DEVICE_TYPE_LB = 15 - VIF_DEVICE_TYPE_NAT_GATEWAY = 16 - VIF_DEVICE_TYPE_POD_GROUP = 101 - VIF_DEVICE_TYPE_SERVICE = 102 - VIF_DEVICE_TYPE_GPROCESS = 120 - VIF_DEVICE_TYPE_IP = 255 + VIF_DEVICE_TYPE_INTERNET = 0 + VIF_DEVICE_TYPE_VM = 1 + VIF_DEVICE_TYPE_VROUTER = 5 + VIF_DEVICE_TYPE_HOST = 6 + VIF_DEVICE_TYPE_DHCP_PORT = 9 + VIF_DEVICE_TYPE_POD = 10 + VIF_DEVICE_TYPE_POD_SERVICE = 11 + VIF_DEVICE_TYPE_REDIS_INSTANCE = 12 + VIF_DEVICE_TYPE_RDS_INSTANCE = 13 + VIF_DEVICE_TYPE_POD_NODE = 14 + VIF_DEVICE_TYPE_LB = 15 + VIF_DEVICE_TYPE_NAT_GATEWAY = 16 + VIF_DEVICE_TYPE_POD_GROUP = 101 + VIF_DEVICE_TYPE_SERVICE = 102 + VIF_DEVICE_TYPE_GPROCESS = 120 + VIF_DEVICE_TYPE_POD_GROUP_DEPLOYMENT = 130 + VIF_DEVICE_TYPE_POD_GROUP_STATEFULSET = 131 + VIF_DEVICE_TYPE_POD_GROUP_RC = 132 + VIF_DEVICE_TYPE_POD_GROUP_DAEMON_SET = 133 + VIF_DEVICE_TYPE_POD_GROUP_REPLICASET_CONTROLLER = 134 + VIF_DEVICE_TYPE_POD_GROUP_CLONESET = 135 + VIF_DEVICE_TYPE_IP = 255 ) const ( @@ -70,8 +76,19 @@ var AutoPodGroupMap = map[string]int{ } var AutoServiceMap = map[string]int{ - "pod_group": VIF_DEVICE_TYPE_POD_GROUP, - "service": VIF_DEVICE_TYPE_SERVICE, + "pod_group": VIF_DEVICE_TYPE_POD_GROUP, + "deployment": VIF_DEVICE_TYPE_POD_GROUP_DEPLOYMENT, + "stateful_set": VIF_DEVICE_TYPE_POD_GROUP_STATEFULSET, + "replication_controller": VIF_DEVICE_TYPE_POD_GROUP_RC, + "daemon_set": VIF_DEVICE_TYPE_POD_GROUP_DAEMON_SET, + "replica_set_controller": VIF_DEVICE_TYPE_POD_GROUP_REPLICASET_CONTROLLER, + "clone_set": VIF_DEVICE_TYPE_POD_GROUP_CLONESET, + "service": VIF_DEVICE_TYPE_SERVICE, +} + +var PodGroupTypeSlice = []string{ + "deployment", "stateful_set", "replication_controller", "daemon_set", + "replica_set_controller", "clone_set", } var NoLanguageTag = []string{ diff --git a/server/querier/engine/clickhouse/tag/description.go b/server/querier/engine/clickhouse/tag/description.go index 8e0d577a054..093c528c0ea 100644 --- a/server/querier/engine/clickhouse/tag/description.go +++ b/server/querier/engine/clickhouse/tag/description.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "golang.org/x/exp/slices" "regexp" "strconv" "strings" @@ -701,6 +702,9 @@ func GetTagResourceValues(db, table, rawSql string) (*common.Result, []string, e "auto_service": AutoServiceMap, } for resourceKey, resourceType := range autoMap[tag] { + if slices.Contains(PodGroupTypeSlice, resourceKey) { + continue + } resourceId := resourceKey + "_id" resourceName := resourceKey + "_name" if resourceKey == "service" { From 96b678d0db953c62e3c0e4d1e54b82636659bd08 Mon Sep 17 00:00:00 2001 From: Jiping Yin Date: Mon, 16 Oct 2023 21:50:56 +0800 Subject: [PATCH 19/22] [eBPF] Support debian 10.6 - 4.19.0-25-amd64 (#4483) --- agent/src/ebpf/kernel/include/task_struct_utils.h | 13 +++++++------ agent/src/ebpf/user/common.c | 9 +++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/agent/src/ebpf/kernel/include/task_struct_utils.h b/agent/src/ebpf/kernel/include/task_struct_utils.h index 82721c366c3..a411edb29c4 100644 --- a/agent/src/ebpf/kernel/include/task_struct_utils.h +++ b/agent/src/ebpf/kernel/include/task_struct_utils.h @@ -127,10 +127,9 @@ static __inline void *infer_and_get_socket_from_fd(int fd_num, // TAG: STRUCT_TASK_FILES_OFFSET // 成员 files 在 struct task_struct 中的偏移量 - // 0xd08 for kernel-devel-4.19.91-26.6.al7 +#ifdef LINUX_VER_5_2_PLUS // 0xa48 for 5.10.0-60.18.0.50.h322_1.hce2.aarch64 // 0xc60 for 5.10.0-106.18.0.68.oe2209.x86_64 -#ifdef LINUX_VER_5_2_PLUS int files_offset_array[] = { 0x790, 0xa80, 0xa88, 0xa90, 0xa98, 0xaa0, 0xaa8, 0xab0, 0xab8, 0xac0, 0xac8, 0xad0, 0xad8, 0xae0, 0xae8, 0xaf0, 0xaf8, 0xb00, 0xb08, 0xb10, @@ -140,12 +139,14 @@ static __inline void *infer_and_get_socket_from_fd(int fd_num, 0xcc8, 0xa48, 0xc60 }; #else + // 0xd08 for kernel-devel-4.19.91-26.6.al7 // 0x740 for 4.19.113-300.el7.x86_64 + // 0x6c0 for 4.19.0-25-amd64 #1 SMP Debian 4.19.289-2 (2023-08-08) int files_offset_array[] = { - 0x790, 0xa80, 0xa88, 0xa90, 0xa98, 0xaa0, 0xaa8, 0xab0, 0xab8, 0xac0, - 0xac8, 0xad0, 0xad8, 0xae0, 0xae8, 0xaf0, 0xaf8, 0xb00, 0xb08, 0xb10, - 0xb18, 0xb20, 0xb28, 0xb48, 0xb50, 0xb58, 0xb60, 0xb68, 0xb70, 0xb78, - 0xb80, 0xb88, 0xb90, 0xb98, 0xba0, 0x740, 0xbb8, 0xbc0, 0xbc8, 0xbd0, + 0x6c0, 0x790, 0xa80, 0xa88, 0xa90, 0xa98, 0xaa0, 0xaa8, 0xab0, 0xab8, + 0xac0, 0xac8, 0xad0, 0xad8, 0xae0, 0xae8, 0xaf0, 0xaf8, 0xb00, 0xb08, + 0xb10, 0xb18, 0xb20, 0xb28, 0xb48, 0xb50, 0xb58, 0xb60, 0xb68, 0xb70, + 0xb78, 0xb88, 0xb90, 0xb98, 0xba0, 0x740, 0xbb8, 0xbc0, 0xbc8, 0xbd0, 0xbd8, 0xbe0, 0xbe8, 0xbf0, 0xbf8, 0xc00, 0xc08, 0xc10, 0xcc8, 0xd08, }; #endif diff --git a/agent/src/ebpf/user/common.c b/agent/src/ebpf/user/common.c index ee7fbe266aa..678740bbf2a 100644 --- a/agent/src/ebpf/user/common.c +++ b/agent/src/ebpf/user/common.c @@ -523,6 +523,15 @@ int fetch_kernel_version(int *major, int *minor, int *patch) if (sscanf(sys_info.release, "%u.%u.%u", major, minor, patch) != 3) return ETR_INVAL; + // Get the real version of Debian + //#1 SMP Debian 4.19.289-2 (2023-08-08) + if (strstr(sys_info.version, "Debian")) { + int num; + if (sscanf(sys_info.version, "%*s %*s %*s %u.%u.%u-%u %*s", + major, minor, patch, &num) != 4) + return ETR_INVAL; + } + return ETR_OK; } From b26997bd2c9e80013eeb49d0e7731ab3d2335b89 Mon Sep 17 00:00:00 2001 From: huanchao Date: Thu, 12 Oct 2023 18:13:39 +0800 Subject: [PATCH 20/22] [Agent] Enhance the direction judgment of eBPF data --- agent/src/common/meta_packet.rs | 2 ++ agent/src/flow_generator/flow_map.rs | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/agent/src/common/meta_packet.rs b/agent/src/common/meta_packet.rs index 73e35267105..70ad42d9d51 100644 --- a/agent/src/common/meta_packet.rs +++ b/agent/src/common/meta_packet.rs @@ -169,6 +169,7 @@ pub struct MetaPacket<'a> { pub process_kname: [u8; PACKET_KNAME_MAX_PADDING], // kernel process name // for PcapAssembler pub flow_id: u64, + pub socket_role: u8, /********** for GPID **********/ pub gpid_0: u32, @@ -932,6 +933,7 @@ impl<'a> MetaPacket<'a> { packet.thread_id = data.thread_id; packet.coroutine_id = data.coroutine_id; packet.syscall_trace_id = data.syscall_trace_id_call; + packet.socket_role = data.socket_role; #[cfg(target_arch = "aarch64")] ptr::copy( data.process_kname.as_ptr() as *const u8, diff --git a/agent/src/flow_generator/flow_map.rs b/agent/src/flow_generator/flow_map.rs index 6d3a1be6669..dde431c0fca 100644 --- a/agent/src/flow_generator/flow_map.rs +++ b/agent/src/flow_generator/flow_map.rs @@ -1082,6 +1082,11 @@ impl FlowMap { ], signal_source: meta_packet.signal_source, is_active_service, + direction_score: if meta_packet.socket_role > 0 { + ServiceTable::MAX_SCORE + } else { + 0 + }, ..Default::default() }; tagged_flow.flow = flow; From 8c8bd68179d1941ef4647595640f39c2360b7a64 Mon Sep 17 00:00:00 2001 From: yunwei37 <1067852565@qq.com> Date: Mon, 16 Oct 2023 10:10:14 +0000 Subject: [PATCH 21/22] [Agent]: fix typo for socket --- agent/src/ebpf/user/ctrl_tracer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agent/src/ebpf/user/ctrl_tracer.c b/agent/src/ebpf/user/ctrl_tracer.c index 8bb7a43c9a1..e8500375c73 100644 --- a/agent/src/ebpf/user/ctrl_tracer.c +++ b/agent/src/ebpf/user/ctrl_tracer.c @@ -202,7 +202,7 @@ static inline int msg_send(int clt_fd, res = sendn(clt_fd, data, data_len, MSG_NOSIGNAL); if (data_len != res) { fprintf(stderr, - "[%s] scoket msg body send error -- %d/%d sent\n", + "[%s] socket msg body send error -- %d/%d sent\n", __func__, res, data_len); return -1; } @@ -308,7 +308,7 @@ static int sockopt_send(enum sockopt_type type, sockoptid_t cmd, const void *in, clt_fd = socket(PF_UNIX, SOCK_STREAM, 0); res = connect(clt_fd, (struct sockaddr *)&clt_addr, sizeof(clt_addr)); if (-1 == res) { - fprintf(stderr, "[%s] scoket msg connection error: %s\n", + fprintf(stderr, "[%s] socket msg connection error: %s\n", __func__, strerror(errno)); free(msg); return -1; From 77465898ad7edb4d3776a1ea5ce117a14851d27b Mon Sep 17 00:00:00 2001 From: Jiping Yin Date: Tue, 17 Oct 2023 00:53:19 +0800 Subject: [PATCH 22/22] [eBPF] Fix container id acquisition failure (#4495) Co-authored-by: XIANG Yang --- agent/src/ebpf/test/Makefile | 2 +- agent/src/ebpf/test/test_fetch_container_id.c | 67 +++++++++++++++++ agent/src/ebpf/user/common.c | 75 +++++++++++-------- agent/src/ebpf/user/common.h | 1 + 4 files changed, 112 insertions(+), 33 deletions(-) create mode 100644 agent/src/ebpf/test/test_fetch_container_id.c diff --git a/agent/src/ebpf/test/Makefile b/agent/src/ebpf/test/Makefile index 26694943f12..7ac379a856d 100644 --- a/agent/src/ebpf/test/Makefile +++ b/agent/src/ebpf/test/Makefile @@ -24,7 +24,7 @@ ARCH ?= $(shell uname -m) CC ?= gcc CFLAGS ?= -std=gnu99 --static -g -O2 -ffunction-sections -fdata-sections -fPIC -fno-omit-frame-pointer -Wall -Wno-sign-compare -Wno-unused-parameter -Wno-missing-field-initializers -EXECS := test_symbol test_offset test_insns_cnt test_bihash test_vec +EXECS := test_symbol test_offset test_insns_cnt test_bihash test_vec test_fetch_container_id ifeq ($(ARCH), x86_64) #-lbcc -lstdc++ LDLIBS += ../libtrace.a -ljattach -lbcc_bpf -lGoReSym -lbddisasm -ldwarf -lelf -lz -lpthread -lbcc -lstdc++ -ldl diff --git a/agent/src/ebpf/test/test_fetch_container_id.c b/agent/src/ebpf/test/test_fetch_container_id.c new file mode 100644 index 00000000000..14d6257d359 --- /dev/null +++ b/agent/src/ebpf/test/test_fetch_container_id.c @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2022 Yunshan Networks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../user/common.h" +#include "../user/mem.h" +#include "../user/log.h" +#include "../user/types.h" + +static const char *cgroup_str_0 = + "8:cpuset:/docker/3386444dafd452389a80af5e5c1dc92fda06e4064770d945ea0eb3d242642bcc"; +static const char *cgroup_str_1 = + "6:pids:/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod55263d4e_9ca4_4f08_ac8e_5df9de111281.slice/cri-containerd-de8109b1bb55c08b50a9d4d3c17cffe3fc7b85b1c25b0b41dbe45e1d386bc6ba.scope"; +static const char *cgroup_str_2 = + "10:pids:/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod804d9406_ab49_4e7c_a7f1_bfddfb186df2.clice/docker-3386444dafd452389a80af5e5c1dc92fda06e4064770d945ea0eb3d242642aaa.scope"; +static const char *cgroup_str_3 = + "5:devices:/kubepods/burstable/pod599a0779-5f40-4779-9d11-82ecce3e6662/3386444dafd452389a80af5e5c1dc92fda06e4064770d945ea0eb3d242642bbb"; +static const char *cgroup_str_err = + "6:devices:/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxsystem.slice/containerd.service"; + +static int test_cgr_str(const char *cgr_str) +{ + char c_id[65]; + char test_str[2048]; + memset(test_str, 0, sizeof(test_str)); + memcpy(test_str, cgr_str, strlen(cgr_str)); + if (fetch_container_id_from_str(test_str, c_id, sizeof(c_id))) + return -1; + if (!(strlen(c_id) == 64 && strstr(test_str, c_id))) + return -1; + printf("\nsource : %s \n fetch : %s\n", cgr_str, c_id); + return 0; +} + +int main(void) +{ + int ret; + ret = test_cgr_str(cgroup_str_0); + if (ret != 0) + return ret; + ret = test_cgr_str(cgroup_str_1); + if (ret != 0) + return ret; + ret = test_cgr_str(cgroup_str_2); + if (ret != 0) + return ret; + ret = test_cgr_str(cgroup_str_3); + if (ret != 0) + return ret; + ret = test_cgr_str(cgroup_str_err); + if (ret == 0) + return -1; + + return 0; +} diff --git a/agent/src/ebpf/user/common.c b/agent/src/ebpf/user/common.c index 678740bbf2a..337de1248cf 100644 --- a/agent/src/ebpf/user/common.c +++ b/agent/src/ebpf/user/common.c @@ -43,6 +43,8 @@ #include "log.h" #include "string.h" +#define MAXLINE 1024 + static u64 g_sys_btime_msecs; bool is_core_kernel(void) @@ -164,7 +166,6 @@ uint32_t get_sys_uptime(void) static void exec_clear_residual_probes(const char *events_file, const char *type_name) { -#define MAXLINE 1024 struct probe_elem { struct list_head list; char event[MAXLINE]; @@ -979,50 +980,60 @@ int exec_command(const char *cmd, const char *args) return -1; } -int fetch_container_id(pid_t pid, char *id, int copy_bytes) +int fetch_container_id_from_str(char *buff, char *id, int copy_bytes) { - static const int scope_len = 5; static const int cid_len = 64; - char file[PATH_MAX], buff[4096]; - int fd; - memset(buff, 0, sizeof(buff)); - snprintf(file, sizeof(file), "/proc/%d/cgroup", pid); - if (access(file, F_OK)) + char *p; + + if ((p = strstr(buff, ".scope"))) + *p = '\0'; + else + p = buff + strlen(buff); + + if (strlen(buff) < cid_len) return -1; - fd = open(file, O_RDONLY); - if (fd <= 2) + p -= cid_len; + + if (strchr(p, '.') || strchr(p, '-') || strchr(p, '/')) return -1; - read(fd, buff, sizeof(buff)); - close(fd); + if (strlen(p) != cid_len) + return -1; - char *p; - if ((p = strchr(buff, '\n')) == NULL) + memset(id, 0, copy_bytes); + memcpy_s_inline((void *)id, copy_bytes, (void *)p, cid_len); + + return 0; +} + +int fetch_container_id(pid_t pid, char *id, int copy_bytes) +{ + char file[PATH_MAX], buff[MAXLINE]; + memset(buff, 0, sizeof(buff)); + snprintf(file, sizeof(file), "/proc/%d/cgroup", pid); + if (access(file, F_OK)) return -1; - *p = '\0'; - if (strlen(buff) < (scope_len + 1 + cid_len)) { + FILE *fp; + char *lf; + if ((fp = fopen(file, "r")) == NULL) { return -1; } - if ((p = strstr(buff, ".scope"))) { - *p = '\0'; - p = strstr(buff, "containerd-"); - if (p == NULL) - return -1; - p += strlen("containerd-"); - goto done; + while ((fgets(buff, sizeof(buff), fp)) != NULL) { + if ((lf = strchr(buff, '\n'))) + *lf = '\0'; + // includes "pids" | "cpuset" | "devices" | "memory" | "cpu" + if (strstr(buff, "pids") || strstr(buff, "cpuset") + || strstr(buff, "devices") || strstr(buff, "memory") + || strstr(buff, "cpu")) { + break; + } - } else if ((p = strstr(buff, "/docker/"))) { - p += strlen("/docker/"); - goto done; } - return -1; -done: - if (strlen(p) != cid_len) - return -1; - memcpy_s_inline((void *)id, copy_bytes, (void *)p, cid_len); - return 0; + fclose(fp); + + return fetch_container_id_from_str(buff, id, copy_bytes); } diff --git a/agent/src/ebpf/user/common.h b/agent/src/ebpf/user/common.h index c0ab7ef716b..1bbf96ca907 100644 --- a/agent/src/ebpf/user/common.h +++ b/agent/src/ebpf/user/common.h @@ -276,5 +276,6 @@ void df_exit_ns(int fd); int gen_file_from_mem(const char *mem_ptr, int write_bytes, const char *path); int exec_command(const char *cmd, const char *args); u64 current_sys_time_secs(void); +int fetch_container_id_from_str(char *buff, char *id, int copy_bytes); int fetch_container_id(pid_t pid, char *id, int copy_bytes); #endif /* DF_COMMON_H */