METRON-822 Improve Fastcapa Performance (nickwallen) closes apache#509

metron-deployment/roles/fastcapa/defaults/main.yml

@@ -17,17 +17,25 @@
 ---
 # dpdk
 dpdk_home: "/usr/local/dpdk"
-dpdk_version: "16.11"
-dpdk_sdk: "/root/dpdk-{{ dpdk_version }}"
+dpdk_version: "16.11.1"
+dpdk_sdk: "/root/dpdk-stable-{{ dpdk_version }}"
 dpdk_src_url: "http://fast.dpdk.org/rel/dpdk-{{ dpdk_version }}.tar.xz"
 dpdk_target: "x86_64-native-linuxapp-gcc"
 num_huge_pages: 512
 extra_cflags: -g
 
 # fastcapa
 fastcapa_work_dir: /root/fastcapa
+fastcapa_build_dir: "{{ fastcapa_work_dir }}/build/app/"
 fastcapa_prefix: /usr/local/bin
 fastcapa_ld_library_path: /usr/local/lib
+fastcapa_bin: fastcapa
+
+# fastcapa settings
 fastcapa_portmask: 0x01
 fastcapa_kafka_config: /etc/fastcapa.conf
-fastcapa_bin: fastcapa
+fastcapa_topic: pcap
+fastcapa_burst_size: 32
+fastcapa_nb_rx_desc: 1024
+fastcapa_nb_rx_queue: 1
+fastcapa_tx_ring_size: 2048

metron-deployment/roles/fastcapa/tasks/fastcapa.yml

@@ -30,7 +30,7 @@
     LD_LIBRARY_PATH: "{{ fastcapa_ld_library_path }}"
 
 - name: Install fastcapa
-  shell: "cp {{ fastcapa_work_dir }}/src/build/app/{{ fastcapa_bin }} {{ fastcapa_prefix }}"
+  shell: "cp {{ fastcapa_build_dir }}/{{ fastcapa_bin }} {{ fastcapa_prefix }}"
   args:
     chdir: "{{ fastcapa_work_dir }}"
     creates: "{{ fastcapa_prefix }}/{{ fastcapa_bin }}"

metron-deployment/roles/fastcapa/templates/fastcapa

@@ -34,6 +34,14 @@ DAEMONLOG=/var/log/$NAME.log
 NOW=`date`
 DAEMON_PATH="{{ dpdk_sdk }}"
 
+PORT_MASK="{{ fastcapa_portmask }}"
+KAFKA_TOPIC="{{ fastcapa_topic }}"
+KAFKA_CONFIG="{{ fastcapa_kafka_config }}"
+BURST_SIZE="{{ fastcapa_burst_size }}"
+NB_RX_DESC="{{ fastcapa_nb_rx_desc }}"
+NB_RX_QUEUE="{{ fastcapa_nb_rx_queue }}"
+TX_RING_SIZE="{{ fastcapa_tx_ring_size }}"
+
 case "$1" in
   start)
     printf "%-50s" "Starting $NAME..."
@@ -42,9 +50,14 @@ case "$1" in
     cd $DAEMON_PATH
     DAEMON="{{ fastcapa_prefix }}/{{ fastcapa_bin }}"
     DAEMONOPTS+=" -- "
-    DAEMONOPTS+="-p {{ fastcapa_portmask }} "
-    DAEMONOPTS+="-t {{ fastcapa_topic }} "
-    DAEMONOPTS+="-c {{ fastcapa_kafka_config }} "
+    DAEMONOPTS+="-p $PORTMASK "
+    DAEMONOPTS+="-t $KAFKA_TOPIC "
+    DAEMONOPTS+="-c $KAFKA_CONFIG "
+    DAEMONOPTS+="-b $BURST_SIZE "
+    DAEMONOPTS+="-r $NB_RX_DESC "
+    DAEMONOPTS+="-q $NB_RX_QUEUE "
+    DAEMONOPTS+="-x $TX_RING_SIZE "
+
 
     PID=`$DAEMON $DAEMONOPTS >> $DAEMONLOG 2>&1 & echo $!`
     if [ -z $PID ]; then

metron-deployment/roles/fastcapa/templates/fastcapa.conf

@@ -27,7 +27,7 @@ metadata.broker.list = {{ kafka_broker_url }}
 client.id = metron-fastcapa
 
 # max number of messages allowed on the producer queue
-queue.buffering.max.messages = 1000
+queue.buffering.max.messages = 1000000
 
 # maximum time, in milliseconds, for buffering data on the producer queue
 queue.buffering.max.ms = 3000
@@ -45,7 +45,7 @@ message.send.max.retries = 5
 retry.backoff.ms = 250
 
 # how often statistics are emitted; 0 = never
-statistics.interval.ms = 0
+statistics.interval.ms = 5000
 
 # only provide delivery reports for failed messages
 delivery.report.only.error = false

metron-deployment/vagrant/fastcapa-test-platform/vars/main.yml

@@ -15,16 +15,28 @@
 #  limitations under the License.
 #
 ---
+
+# 0.9.4+ required for fastcapa
+librdkafka_version: 0.9.4
+librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz 
+
 dpdk_device: ["00:08.0"]
 dpdk_target: "x86_64-native-linuxapp-gcc"
 num_huge_pages: 512
-fastcapa_portmask: "0x01"
-fastcapa_topic: pcap
 kafka_broker_url: source:9092
 zookeeper_url: source:2181
 pcap_replay_interface: eth1
 kafka_broker_home: /usr/hdp/current/kafka-broker/
 
+# fastcapa settings
+fastcapa_portmask: 0x01
+fastcapa_kafka_config: /etc/fastcapa.conf
+fastcapa_topic: pcap
+fastcapa_burst_size: 32
+fastcapa_nb_rx_desc: 1024
+fastcapa_nb_rx_queue: 1
+fastcapa_tx_ring_size: 2048
+
 # dummy variables for pycapa's dependence on ambari_gather_facts
 cluster_name: dummy
 namenode_host: dummy

metron-sensors/fastcapa/.gitignore

@@ -1,3 +1,5 @@
 roles
 .vagrant
 *.retry
+build
+*.out

metron-sensors/fastcapa/Makefile

@@ -18,9 +18,10 @@
 all: src
 
 src:
-	cd src; make
+	cd src; make O=../build
 
 clean:
 	cd src; make clean
+	rm -rf build
 
 .PHONY: all src clean