diff --git a/lib/logstash/codecs/netflow.rb b/lib/logstash/codecs/netflow.rb index f00c1b4..facaefe 100644 --- a/lib/logstash/codecs/netflow.rb +++ b/lib/logstash/codecs/netflow.rb @@ -472,6 +472,29 @@ def string_field(field, type, length) field end # def string_field + def get_rfc6759_application_id_class(field,length) + case length + when 2 + field[0] = :Application_Id16 + when 3 + field[0] = :Application_Id24 + when 4 + field[0] = :Application_Id32 + when 5 + field[0] = :Application_Id40 + when 7 + field[0] = :Application_Id56 + when 8 + field[0] = :Application_Id64 + when 9 + field[0] = :Application_Id72 + else + @logger.warn("Unsupported application_id length encountered, skipping", :field => field, :length => length) + nil + end + field[0] + end + def netflow_field_for(type, length, template_id) if @netflow_fields.include?(type) field = @netflow_fields[type].clone @@ -509,23 +532,7 @@ def netflow_field_for(type, length, template_id) end field[0] = uint_field(length, field[0]) when :application_id - case length - when 2 - field[0] = :Application_Id16 - when 3 - field[0] = :Application_Id24 - when 4 - field[0] = :Application_Id32 - when 5 - field[0] = :Application_Id40 - when 8 - field[0] = :Application_Id64 - when 9 - field[0] = :Application_Id72 - else - @logger.warn("Unsupported application_id length encountered, skipping", :field => field, :length => length) - nil - end + field[0] = get_rfc6759_application_id_class(field,length) when :skip field += [nil, {:length => length.to_i}] when :string @@ -573,6 +580,8 @@ def ipfix_field_for(type, enterprise, length) field[0] = uint_field(length, 4) when :uint16 field[0] = uint_field(length, 2) + when :application_id + field[0] = get_rfc6759_application_id_class(field,length) end @logger.debug("Definition complete", :field => field) diff --git a/lib/logstash/codecs/netflow/util.rb b/lib/logstash/codecs/netflow/util.rb index 62ddcb7..fd6cbf6 100644 --- a/lib/logstash/codecs/netflow/util.rb +++ b/lib/logstash/codecs/netflow/util.rb @@ -139,10 +139,11 @@ def set(val) end def get - self.classification_id.to_s + ":" + self.selector_id.to_s + self.classification_id.to_s + ".." + self.selector_id.to_s end end + class Application_Id24 < BinData::Primitive endian :big uint8 :classification_id @@ -156,10 +157,11 @@ def set(val) end def get - self.classification_id.to_s + ":" + self.selector_id.to_s + self.classification_id.to_s + ".." + self.selector_id.to_s end end + class Application_Id32 < BinData::Primitive endian :big uint8 :classification_id @@ -173,10 +175,11 @@ def set(val) end def get - self.classification_id.to_s + ":" + self.selector_id.to_s + self.classification_id.to_s + ".." + self.selector_id.to_s end end + class Application_Id40 < BinData::Primitive endian :big uint8 :classification_id @@ -190,41 +193,130 @@ def set(val) end def get - self.classification_id.to_s + ":" + self.selector_id.to_s + self.classification_id.to_s + ".." + self.selector_id.to_s + end +end + + +class Appid56PanaL7Pen < BinData::Record + # RFC6759 chapter 4.1: PANA-L7-PEN + # This implements the "application ids MAY be encoded in a smaller number of bytes" + # Used in Application_Id56 choice statement + endian :big + uint32 :pen_id + uint16 :selector_id +end + + +class Application_Id56 < BinData::Primitive + endian :big + uint8 :classification_id + choice :selector_id, :selection => :classification_id do + # for classification engine id 20 we switch to Appid64PanaL7Pen to decode + appid56_pana_l7_pen 20 + uint48 :default + end + + def set(val) + unless val.nil? + self.classification_id=val.to_i<<48 + if self.classification_id == 20 + # classification engine id 20 (PANA_L7_PEN) contains a 4-byte PEN: + self.pen_id = val.to_i-((val.to_i>>48)<<48)>>16 + self.selector_id = val.to_i-((val.to_i>>16)<<16) + else + self.selector_id = val.to_i-((val.to_i>>48)<<48) + end + end + end + + def get + if self.classification_id == 20 + self.classification_id.to_s + ".." + self.selector_id[:pen_id].to_s + ".." + self.selector_id[:selector_id].to_s + else + self.classification_id.to_s + ".." + self.selector_id.to_s + end end end + +class Appid64PanaL7Pen < BinData::Record + # RFC6759 chapter 4.1: PANA-L7-PEN + # This implements the 3 bytes default selector id length + # Used in Application_Id64 choice statement + endian :big + uint32 :pen_id + uint24 :selector_id +end + class Application_Id64 < BinData::Primitive endian :big uint8 :classification_id - uint56 :selector_id + choice :selector_id, :selection => :classification_id do + # for classification engine id 20 we switch to Appid64PanaL7Pen to decode + appid64_pana_l7_pen 20 + uint56 :default + end def set(val) unless val.nil? self.classification_id=val.to_i<<56 - self.selector_id = val.to_i-((val.to_i>>56)<<56) + if self.classification_id == 20 + # classification engine id 20 (PANA_L7_PEN) contains a 4-byte PEN: + self.pen_id = val.to_i-((val.to_i>>56)<<56)>>24 + self.selector_id = val.to_i-((val.to_i>>24)<<24) + else + self.selector_id = val.to_i-((val.to_i>>56)<<56) + end end end def get - self.classification_id.to_s + ":" + self.selector_id.to_s + if self.classification_id == 20 + self.classification_id.to_s + ".." + self.selector_id[:pen_id].to_s + ".." + self.selector_id[:selector_id].to_s + else + self.classification_id.to_s + ".." + self.selector_id.to_s + end end end +class Appid72PanaL7Pen < BinData::Record + # RFC6759 chapter 4.1: PANA-L7-PEN + # This implements the "application ids MAY be encoded with a larger length" + # Used in Application_Id72 choice statement + endian :big + uint32 :pen_id + uint32 :selector_id +end + class Application_Id72 < BinData::Primitive endian :big uint8 :classification_id - uint64 :selector_id + choice :selector_id, :selection => :classification_id do + # for classification engine id 20 we switch to Appid72PanaL7Pen to decode + appid72_pana_l7_pen 20 + uint64 :default + end def set(val) unless val.nil? - self.classification_id=val.to_i<<64 - self.selector_id = val.to_i-((val.to_i>>64)<<64) + self.classification_id = val.to_i<<64 + if self.classification_id == 20 + # classification engine id 20 (PANA_L7_PEN) contains a 4-byte PEN: + self.pen_id = val.to_i-((val.to_i>>64)<<64)>>32 + self.selector_id = val.to_i-((val.to_i>>32)<<32) + else + self.selector_id = val.to_i-((val.to_i>>64)<<64) + end end end def get - self.classification_id.to_s + ":" + self.selector_id.to_s + if self.classification_id == 20 + self.classification_id.to_s + ".." + self.selector_id[:pen_id].to_s + ".." + self.selector_id[:selector_id].to_s + else + self.classification_id.to_s + ".." + self.selector_id.to_s + end end end diff --git a/spec/codecs/netflow9_test_fortigate_fortios_542_appid_data258_262.dat b/spec/codecs/netflow9_test_fortigate_fortios_542_appid_data258_262.dat new file mode 100644 index 0000000..bdc5af0 Binary files /dev/null and b/spec/codecs/netflow9_test_fortigate_fortios_542_appid_data258_262.dat differ diff --git a/spec/codecs/netflow9_test_fortigate_fortios_542_appid_tpl258-269.dat b/spec/codecs/netflow9_test_fortigate_fortios_542_appid_tpl258-269.dat new file mode 100644 index 0000000..9a5782f Binary files /dev/null and b/spec/codecs/netflow9_test_fortigate_fortios_542_appid_tpl258-269.dat differ diff --git a/spec/codecs/netflow_spec.rb b/spec/codecs/netflow_spec.rb index 5ebdc3f..4dab0b5 100644 --- a/spec/codecs/netflow_spec.rb +++ b/spec/codecs/netflow_spec.rb @@ -1024,8 +1024,6 @@ end - - context "Netflow 9 IE150 IE151" do let(:data) do packets = [] @@ -1103,6 +1101,66 @@ end + + context "Netflow 9 Fortigate FortiOS 54x appid" do + let(:data) do + packets = [] + packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_fortigate_fortios_542_appid_tpl258-269.dat"), :mode => "rb") + packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_fortigate_fortios_542_appid_data258_262.dat"), :mode => "rb") + end + + let(:json_events) do + events = [] + events << <<-END + { + "netflow": { + "output_snmp": 2, + "forwarding_status": { + "reason": 0, + "status": 1 + }, + "xlate_src_port": 45380, + "in_pkts": 6, + "ipv4_dst_addr": "182.50.136.239", + "first_switched": "2018-05-11T00:54:10.999Z", + "flowset_id": 262, + "l4_src_port": 45380, + "xlate_dst_port": 0, + "version": 9, + "application_id": "20..12356..36660", + "flow_seq_num": 350, + "ipv4_src_addr": "192.168.100.151", + "in_bytes": 748, + "protocol": 6, + "flow_end_reason": 3, + "last_switched": "2018-05-11T00:54:10.999Z", + "input_snmp": 8, + "out_pkts": 6, + "out_bytes": 748, + "xlate_src_addr_ipv4": "10.0.0.250", + "xlate_dst_addr_ipv4": "0.0.0.0", + "l4_dst_port": 80 + }, + "@timestamp": "2018-05-11T00:54:11.000Z", + "@version": "1" + } + END + events.map{|event| event.gsub(/\s+/, "")} + end + + it "should decode raw data" do + expect(decode.size).to eq(17) + expect(decode[1].get("[netflow][application_id]")).to eq("20..12356..40568") + expect(decode[2].get("[netflow][application_id]")).to eq("20..12356..40568") + expect(decode[16].get("[netflow][application_id]")).to eq("20..12356..0") + end + + it "should serialize to json" do + expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0])) + end + + end + context "IPFIX Nokia BRAS" do let(:data) do packets = [] @@ -1352,7 +1410,7 @@ "l4_src_port": 0, "nprobe_proto_name": "\u0000\u00c1\u0000\u0000\u0001\u00ac\u0010\u0000d\u00e4O\u00ef\u00ff\u00ff\u00fa\u0007", "version": 9, - "application_id": "0:82", + "application_id": "0..82", "flow_seq_num": 2, "ipv4_src_addr": "0.0.0.0", "protocol": 0, @@ -1372,7 +1430,7 @@ it "should decode raw data" do expect(decode.size).to eq(1) expect(decode[0].get("[netflow][nprobe_proto]")).to eq(82) - expect(decode[0].get("[netflow][application_id]")).to eq("0:82") + expect(decode[0].get("[netflow][application_id]")).to eq("0..82") expect(decode[0].get("[netflow][in_bytes]")).to eq(82) end @@ -2291,7 +2349,7 @@ "application_description": "ARGUS", "flowset_id": 260, "version": 9, - "application_id": "1:13" + "application_id": "1..13" }, "@timestamp": "2017-02-14T11:09:59.000Z", "@version": "1" @@ -2302,7 +2360,7 @@ it "should decode raw data" do expect(decode.size).to eq(15) - expect(decode[14].get("[netflow][application_id]")).to eq("1:13") + expect(decode[14].get("[netflow][application_id]")).to eq("1..13") expect(decode[14].get("[netflow][application_description]")).to eq("ARGUS") end @@ -2345,7 +2403,7 @@ "udp_dst_port": 161, "src_mask": 0, "version": 9, - "application_id": "5:38", + "application_id": "5..38", "flow_seq_num": 1509134, "ipv4_src_addr": "10.10.172.60", "in_src_mac": "00:18:19:9e:6c:01", @@ -2362,7 +2420,7 @@ it "should decode raw data" do expect(decode.size).to eq(5) - expect(decode[4].get("[netflow][application_id]")).to eq("5:38") + expect(decode[4].get("[netflow][application_id]")).to eq("5..38") end it "should serialize to json" do @@ -2388,7 +2446,7 @@ "staMacAddress": "34:02:86:75:c0:51", "flowset_id": 261, "version": 9, - "application_id": "13:431", + "application_id": "13..431", "flow_seq_num": 78, "in_bytes": 80973880, "postIpDiffServCodePoint": 0, @@ -2405,7 +2463,7 @@ it "should decode raw data" do expect(decode.size).to eq(19) - expect(decode[18].get("[netflow][application_id]")).to eq("13:431") + expect(decode[18].get("[netflow][application_id]")).to eq("13..431") end it "should serialize to json" do