Skip to content

Commit f069dcd

Browse files
jfb8856606jfb8856606
committed
Support KNI ratelimit.
1 parent 5c144b1 commit f069dcd

File tree

6 files changed

+150
-21
lines changed

6 files changed

+150
-21
lines changed

‎config.ini

+8
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,14 @@ gateway=192.168.1.1
219219
# The format is same as port_list
220220
#tcp_port=80,443
221221
#udp_port=53
222+
# KNI ratelimit value, default: 0, means disable ratelimit.
223+
# example:
224+
# The total speed limit for a single process entering the kni ring is 10,000 QPS,
225+
# 1000 QPS for general packets, 9000 QPS for console packets (ospf/arp, etc.)
226+
# The total speed limit for kni forwarding to the kernel is 20,000 QPS.
227+
#console_packets_ratelimit=0
228+
#general_packets_ratelimit=0
229+
#kernel_packets_ratelimit=0
222230

223231
# FreeBSD network performance tuning configurations.
224232
# Most native FreeBSD configurations are supported.

‎lib/ff_config.c

+11
Original file line numberDiff line numberDiff line change
@@ -909,6 +909,12 @@ ini_parse_handler(void* user, const char* section, const char* name,
909909
pconfig->kni.enable= atoi(value);
910910
} else if (MATCH("kni", "type")) {
911911
pconfig->kni.type= atoi(value);
912+
} else if (MATCH("kni", "console_packets_ratelimit")) {
913+
pconfig->kni.console_packets_ratelimit= atoi(value);
914+
} else if (MATCH("kni", "general_packets_ratelimit")) {
915+
pconfig->kni.general_packets_ratelimit= atoi(value);
916+
} else if (MATCH("kni", "kernel_packets_ratelimit")) {
917+
pconfig->kni.kernel_packets_ratelimit= atoi(value);
912918
} else if (MATCH("kni", "kni_action")) {
913919
pconfig->kni.kni_action= strdup(value);
914920
} else if (MATCH("kni", "method")) {
@@ -1247,6 +1253,11 @@ ff_default_config(struct ff_config *cfg)
12471253
cfg->dpdk.promiscuous = 1;
12481254
cfg->dpdk.pkt_tx_delay = BURST_TX_DRAIN_US;
12491255

1256+
/* KNI ratelimit default disabled */
1257+
//cfg->kni.console_packets_ratelimit = KNI_RATELIMT_CONSOLE;
1258+
//cfg->kni.general_packets_ratelimit = KNI_RATELIMT_GENERAL;
1259+
//cfg->kni.kernel_packets_ratelimit = KNI_RATELIMT_KERNEL;
1260+
12501261
cfg->freebsd.hz = 100;
12511262
cfg->freebsd.physmem = 1048576*256;
12521263
cfg->freebsd.fd_reserve = 0;

‎lib/ff_config.h

+13
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,16 @@ extern "C" {
3838
#define DPDK_MAX_VLAN_FILTER 128
3939
#define PCAP_SNAP_MINLEN 94
4040
#define PCAP_SAVE_MINLEN (2<<22)
41+
/*
42+
* KNI ratelimit default value.
43+
* The total speed limit for a single process entering the kni ring is 10,000 QPS,
44+
* 1000 QPS for general packets, 9000 QPS for console packets (ospf/arp, etc.)
45+
* The total speed limit for kni forwarding to the kernel is 20,000 QPS.
46+
*/
47+
#define KNI_RATELIMT_PROCESS (10000)
48+
#define KNI_RATELIMT_GENERAL (1000)
49+
#define KNI_RATELIMT_CONSOLE (KNI_RATELIMT_PROCESS - KNI_RATELIMT_GENERAL)
50+
#define KNI_RATELIMT_KERNEL (KNI_RATELIMT_PROCESS * 2)
4151

4252
extern int dpdk_argc;
4353
extern char *dpdk_argv[DPDK_CONFIG_NUM + 1];
@@ -241,6 +251,9 @@ struct ff_config {
241251
struct {
242252
int enable;
243253
int type;
254+
int console_packets_ratelimit;
255+
int general_packets_ratelimit;
256+
int kernel_packets_ratelimit;
244257
char *kni_action;
245258
char *method;
246259
char *tcp_port;

‎lib/ff_dpdk_if.c

+39-5
Original file line numberDiff line numberDiff line change
@@ -1327,8 +1327,14 @@ protocol_filter(const void *data, uint16_t len)
13271327
len -= sizeof(struct rte_vlan_hdr);
13281328
}
13291329

1330-
if(ether_type == RTE_ETHER_TYPE_ARP)
1330+
if(ether_type == RTE_ETHER_TYPE_ARP) {
13311331
return FILTER_ARP;
1332+
}
1333+
1334+
/* Multicast protocol, such as stp(used by zebra), is forwarded to kni and has a separate speed limit */
1335+
if (rte_is_multicast_ether_addr(&hdr->dst_addr)) {
1336+
return FILTER_MULTI;
1337+
}
13321338

13331339
#if (!defined(__FreeBSD__) && defined(INET6) ) || \
13341340
( defined(__FreeBSD__) && defined(INET6) && defined(FF_KNI))
@@ -1520,7 +1526,7 @@ process_packets(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **bufs,
15201526
mbuf_clone = pktmbuf_deep_clone(rtem, mbuf_pool);
15211527
if(mbuf_clone) {
15221528
ff_add_vlan_tag(mbuf_clone);
1523-
ff_kni_enqueue(port_id, mbuf_clone);
1529+
ff_kni_enqueue(filter, port_id, mbuf_clone);
15241530
}
15251531
}
15261532
#endif
@@ -1529,15 +1535,15 @@ process_packets(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **bufs,
15291535
} else if (enable_kni) {
15301536
if (knictl_action == FF_KNICTL_ACTION_ALL_TO_KNI){
15311537
ff_add_vlan_tag(rtem);
1532-
ff_kni_enqueue(port_id, rtem);
1538+
ff_kni_enqueue(filter, port_id, rtem);
15331539
} else if (knictl_action == FF_KNICTL_ACTION_ALL_TO_FF){
15341540
ff_veth_input(ctx, rtem);
15351541
} else if (knictl_action == FF_KNICTL_ACTION_DEFAULT){
15361542
if (enable_kni &&
15371543
((filter == FILTER_KNI && kni_accept) ||
1538-
(filter == FILTER_UNKNOWN && !kni_accept)) ) {
1544+
((filter == FILTER_UNKNOWN || filter >= FILTER_OSPF) && !kni_accept)) ) {
15391545
ff_add_vlan_tag(rtem);
1540-
ff_kni_enqueue(port_id, rtem);
1546+
ff_kni_enqueue(filter, port_id, rtem);
15411547
} else {
15421548
ff_veth_input(ctx, rtem);
15431549
}
@@ -2056,6 +2062,34 @@ main_loop(void *arg)
20562062
cur_tsc = rte_rdtsc();
20572063
if (unlikely(freebsd_clock.expire < cur_tsc)) {
20582064
rte_timer_manage();
2065+
2066+
#ifdef FF_KNI
2067+
/* reset kni ratelimt */
2068+
if (enable_kni &&
2069+
(ff_global_cfg.kni.console_packets_ratelimit ||
2070+
ff_global_cfg.kni.general_packets_ratelimit ||
2071+
ff_global_cfg.kni.kernel_packets_ratelimit)) {
2072+
static time_t last_sec = 0;
2073+
time_t sec;
2074+
long nsec;
2075+
2076+
ff_get_current_time(&sec, &nsec);
2077+
if (sec > last_sec) {
2078+
if (kni_rate_limt.gerneal_packets > ff_global_cfg.kni.general_packets_ratelimit ||
2079+
kni_rate_limt.console_packets > ff_global_cfg.kni.console_packets_ratelimit ||
2080+
kni_rate_limt.kernel_packets > ff_global_cfg.kni.kernel_packets_ratelimit) {
2081+
printf("kni ratelimit, general:%lu/%d, console:%lu/%d, kernel:%lu/%d, last sec:%ld, sec:%ld\n",
2082+
kni_rate_limt.gerneal_packets, ff_global_cfg.kni.general_packets_ratelimit,
2083+
kni_rate_limt.console_packets, ff_global_cfg.kni.console_packets_ratelimit,
2084+
kni_rate_limt.kernel_packets, ff_global_cfg.kni.kernel_packets_ratelimit, last_sec, sec);
2085+
}
2086+
last_sec = sec;
2087+
kni_rate_limt.gerneal_packets = 0;
2088+
kni_rate_limt.console_packets = 0;
2089+
kni_rate_limt.kernel_packets = 0;
2090+
}
2091+
}
2092+
#endif
20592093
}
20602094

20612095
idle = 1;

‎lib/ff_dpdk_kni.c

+61-13
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@
4444
#include "ff_dpdk_kni.h"
4545
#include "ff_config.h"
4646

47+
#ifndef IPPROTO_OSPFIGP
48+
#define IPPROTO_OSPFIGP 89 /**< OSPFIGP */
49+
#endif
50+
4751
/* Callback for request of changing MTU */
4852
/* Total octets in ethernet header */
4953
#define KNI_ENET_HEADER_SIZE 14
@@ -92,6 +96,8 @@ struct kni_interface_stats {
9296
struct rte_ring **kni_rp;
9397
struct kni_interface_stats **kni_stat;
9498

99+
struct kni_ratelimit kni_rate_limt = {0, 0, 0};
100+
95101
static void
96102
set_bitmap(uint16_t port, unsigned char *bitmap)
97103
{
@@ -219,24 +225,38 @@ kni_process_tx(uint16_t port_id, uint16_t queue_id,
219225
struct rte_mbuf **pkts_burst, unsigned count)
220226
{
221227
/* read packet from kni ring(phy port) and transmit to kni */
222-
uint16_t nb_tx, nb_kni_tx = 0;
228+
uint16_t nb_tx, nb_to_tx, nb_kni_tx;
223229
nb_tx = rte_ring_dequeue_burst(kni_rp[port_id], (void **)pkts_burst, count, NULL);
224230

231+
/*
232+
* The total ratelimit forwarded to the kernel, may a few more packets being sent, but it doesn’t matter,
233+
* If there are too many processes, there is also the possibility that the control packet will be ratelimited.
234+
*/
235+
if (ff_global_cfg.kni.kernel_packets_ratelimit) {
236+
if (likely(kni_rate_limt.kernel_packets < ff_global_cfg.kni.kernel_packets_ratelimit)) {
237+
nb_to_tx = nb_tx;
238+
} else {
239+
nb_to_tx = 0;
240+
}
241+
kni_rate_limt.kernel_packets += nb_tx;
242+
} else {
243+
nb_to_tx = nb_tx;
244+
}
245+
225246
#ifdef FF_KNI_KNI
226247
if (ff_global_cfg.kni.type == KNI_TYPE_KNI) {
227248
/* NB.
228249
* if nb_tx is 0,it must call rte_kni_tx_burst
229250
* must Call regularly rte_kni_tx_burst(kni, NULL, 0).
230251
* detail https://embedded.communities.intel.com/thread/6668
231252
*/
232-
nb_kni_tx = rte_kni_tx_burst(kni_stat[port_id]->kni, pkts_burst, nb_tx);
253+
nb_kni_tx = rte_kni_tx_burst(kni_stat[port_id]->kni, pkts_burst, nb_to_tx);
233254
rte_kni_handle_request(kni_stat[port_id]->kni);
234255
} else if (ff_global_cfg.kni.type == KNI_TYPE_VIRTIO)
235256
#endif
236257
{
237-
nb_kni_tx = rte_eth_tx_burst(kni_stat[port_id]->port_id, 0, pkts_burst, nb_tx);
258+
nb_kni_tx = rte_eth_tx_burst(kni_stat[port_id]->port_id, 0, pkts_burst, nb_to_tx);
238259
}
239-
240260
if(nb_kni_tx < nb_tx) {
241261
uint16_t i;
242262
for(i = nb_kni_tx; i < nb_tx; ++i)
@@ -419,22 +439,28 @@ protocol_filter_ip(const void *data, uint16_t len, uint16_t eth_frame_type)
419439
next_len = len - hdr_len;
420440

421441
switch (proto) {
442+
#ifdef FF_KNI
443+
/* The opsf protocol is forwarded to kni and the ratelimited separately */
444+
case IPPROTO_OSPFIGP:
445+
return FILTER_OSPF;
446+
#endif
447+
422448
case IPPROTO_TCP:
423449
#ifdef FF_KNI
424450
if (!enable_kni)
425-
break;
426-
#else
427-
break;
428451
#endif
452+
break;
453+
429454
return protocol_filter_tcp(next, next_len);
455+
430456
case IPPROTO_UDP:
431457
#ifdef FF_KNI
432458
if (!enable_kni)
433-
break;
434-
#else
435-
break;
436459
#endif
460+
break;
461+
437462
return protocol_filter_udp(next, next_len);
463+
438464
case IPPROTO_IPIP:
439465
return protocol_filter_ip(next, next_len, RTE_ETHER_TYPE_IPV4);
440466
#ifdef INET6
@@ -628,12 +654,34 @@ ff_kni_process(uint16_t port_id, uint16_t queue_id,
628654

629655
/* enqueue the packet, and own it */
630656
int
631-
ff_kni_enqueue(uint16_t port_id, struct rte_mbuf *pkt)
657+
ff_kni_enqueue(enum FilterReturn filter, uint16_t port_id, struct rte_mbuf *pkt)
632658
{
659+
if (filter >= FILTER_ARP) {
660+
if (ff_global_cfg.kni.console_packets_ratelimit) {
661+
kni_rate_limt.console_packets++;
662+
if (kni_rate_limt.console_packets > ff_global_cfg.kni.console_packets_ratelimit) {
663+
goto error;
664+
}
665+
}
666+
} else {
667+
if (ff_global_cfg.kni.general_packets_ratelimit) {
668+
kni_rate_limt.gerneal_packets++;
669+
if (kni_rate_limt.gerneal_packets > ff_global_cfg.kni.general_packets_ratelimit) {
670+
goto error;
671+
}
672+
}
673+
}
674+
633675
int ret = rte_ring_enqueue(kni_rp[port_id], pkt);
634-
if (ret < 0)
635-
rte_pktmbuf_free(pkt);
676+
if (ret < 0) {
677+
goto error;
678+
}
636679

637680
return 0;
681+
682+
error:
683+
rte_pktmbuf_free(pkt);
684+
685+
return -1;
638686
}
639687

‎lib/ff_dpdk_kni.h

+18-3
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,28 @@
3434
extern int enable_kni;
3535
extern int nb_dev_ports;
3636

37+
struct kni_ratelimit {
38+
/* Important control plane packets enqueue to kni ring, such as arp, stp, ospf, etc. statistics for each process. */
39+
uint64_t console_packets;
40+
41+
/* gerneal packets enqueue to kni ring, such ICMP pkts, statistics for each process. */
42+
uint64_t gerneal_packets;
43+
44+
/* All packets forwarded to the kernel, statistics for primary process. */
45+
uint64_t kernel_packets;
46+
};
47+
48+
extern struct kni_ratelimit kni_rate_limt;
49+
3750
enum FilterReturn {
3851
FILTER_UNKNOWN = -1,
39-
FILTER_ARP = 1,
40-
FILTER_KNI = 2,
52+
FILTER_KNI = 1,
53+
FILTER_ARP = 2,
4154
#ifdef INET6
4255
FILTER_NDP = 3, // Neighbor Solicitation/Advertisement, Router Solicitation/Advertisement/Redirect
4356
#endif
57+
FILTER_OSPF = 4,
58+
FILTER_MULTI = 5,
4459
};
4560

4661
void ff_kni_init(uint16_t nb_ports, int type, const char *tcp_ports,
@@ -54,7 +69,7 @@ void ff_kni_process(uint16_t port_id, uint16_t queue_id,
5469

5570
enum FilterReturn ff_kni_proto_filter(const void *data, uint16_t len, uint16_t eth_frame_type);
5671

57-
int ff_kni_enqueue(uint16_t port_id, struct rte_mbuf *pkt);
72+
int ff_kni_enqueue(enum FilterReturn filter, uint16_t port_id, struct rte_mbuf *pkt);
5873

5974

6075
#endif /* ifndef _FSTACK_DPDK_KNI_H */

0 commit comments

Comments
 (0)