Skip to content

Commit

Permalink
Added support for RFC6759 decoding of application_id field (#145)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jorritfolmer
jorritfolmer authored May 31, 2018
1 parent eb977bd commit 7a31f32
Show file tree
Hide file tree
Showing 5 changed files with 197 additions and 38 deletions.
43 changes: 26 additions & 17 deletions lib/logstash/codecs/netflow.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,29 @@ def string_field(field, type, length)
field
end # def string_field

def get_rfc6759_application_id_class(field,length)
case length
when 2
field[0] = :Application_Id16
when 3
field[0] = :Application_Id24
when 4
field[0] = :Application_Id32
when 5
field[0] = :Application_Id40
when 7
field[0] = :Application_Id56
when 8
field[0] = :Application_Id64
when 9
field[0] = :Application_Id72
else
@logger.warn("Unsupported application_id length encountered, skipping", :field => field, :length => length)
nil
end
field[0]
end

def netflow_field_for(type, length, template_id)
if @netflow_fields.include?(type)
field = @netflow_fields[type].clone
Expand Down Expand Up @@ -509,23 +532,7 @@ def netflow_field_for(type, length, template_id)
end
field[0] = uint_field(length, field[0])
when :application_id
case length
when 2
field[0] = :Application_Id16
when 3
field[0] = :Application_Id24
when 4
field[0] = :Application_Id32
when 5
field[0] = :Application_Id40
when 8
field[0] = :Application_Id64
when 9
field[0] = :Application_Id72
else
@logger.warn("Unsupported application_id length encountered, skipping", :field => field, :length => length)
nil
end
field[0] = get_rfc6759_application_id_class(field,length)
when :skip
field += [nil, {:length => length.to_i}]
when :string
Expand Down Expand Up @@ -573,6 +580,8 @@ def ipfix_field_for(type, enterprise, length)
field[0] = uint_field(length, 4)
when :uint16
field[0] = uint_field(length, 2)
when :application_id
field[0] = get_rfc6759_application_id_class(field,length)
end

@logger.debug("Definition complete", :field => field)
Expand Down
114 changes: 103 additions & 11 deletions lib/logstash/codecs/netflow/util.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,10 +139,11 @@ def set(val)
end
def get
self.classification_id.to_s + ":" + self.selector_id.to_s
self.classification_id.to_s + ".." + self.selector_id.to_s
end
end
class Application_Id24 < BinData::Primitive
endian :big
uint8 :classification_id
Expand All @@ -156,10 +157,11 @@ def set(val)
end
def get
self.classification_id.to_s + ":" + self.selector_id.to_s
self.classification_id.to_s + ".." + self.selector_id.to_s
end
end
class Application_Id32 < BinData::Primitive
endian :big
uint8 :classification_id
Expand All @@ -173,10 +175,11 @@ def set(val)
end
def get
self.classification_id.to_s + ":" + self.selector_id.to_s
self.classification_id.to_s + ".." + self.selector_id.to_s
end
end
class Application_Id40 < BinData::Primitive
endian :big
uint8 :classification_id
Expand All @@ -190,41 +193,130 @@ def set(val)
end
def get
self.classification_id.to_s + ":" + self.selector_id.to_s
self.classification_id.to_s + ".." + self.selector_id.to_s
end
end
class Appid56PanaL7Pen < BinData::Record
# RFC6759 chapter 4.1: PANA-L7-PEN
# This implements the "application ids MAY be encoded in a smaller number of bytes"
# Used in Application_Id56 choice statement
endian :big
uint32 :pen_id
uint16 :selector_id
end
class Application_Id56 < BinData::Primitive
endian :big
uint8 :classification_id
choice :selector_id, :selection => :classification_id do
# for classification engine id 20 we switch to Appid64PanaL7Pen to decode
appid56_pana_l7_pen 20
uint48 :default
end
def set(val)
unless val.nil?
self.classification_id=val.to_i<<48
if self.classification_id == 20
# classification engine id 20 (PANA_L7_PEN) contains a 4-byte PEN:
self.pen_id = val.to_i-((val.to_i>>48)<<48)>>16
self.selector_id = val.to_i-((val.to_i>>16)<<16)
else
self.selector_id = val.to_i-((val.to_i>>48)<<48)
end
end
end
def get
if self.classification_id == 20
self.classification_id.to_s + ".." + self.selector_id[:pen_id].to_s + ".." + self.selector_id[:selector_id].to_s
else
self.classification_id.to_s + ".." + self.selector_id.to_s
end
end
end
class Appid64PanaL7Pen < BinData::Record
# RFC6759 chapter 4.1: PANA-L7-PEN
# This implements the 3 bytes default selector id length
# Used in Application_Id64 choice statement
endian :big
uint32 :pen_id
uint24 :selector_id
end
class Application_Id64 < BinData::Primitive
endian :big
uint8 :classification_id
uint56 :selector_id
choice :selector_id, :selection => :classification_id do
# for classification engine id 20 we switch to Appid64PanaL7Pen to decode
appid64_pana_l7_pen 20
uint56 :default
end
def set(val)
unless val.nil?
self.classification_id=val.to_i<<56
self.selector_id = val.to_i-((val.to_i>>56)<<56)
if self.classification_id == 20
# classification engine id 20 (PANA_L7_PEN) contains a 4-byte PEN:
self.pen_id = val.to_i-((val.to_i>>56)<<56)>>24
self.selector_id = val.to_i-((val.to_i>>24)<<24)
else
self.selector_id = val.to_i-((val.to_i>>56)<<56)
end
end
end
def get
self.classification_id.to_s + ":" + self.selector_id.to_s
if self.classification_id == 20
self.classification_id.to_s + ".." + self.selector_id[:pen_id].to_s + ".." + self.selector_id[:selector_id].to_s
else
self.classification_id.to_s + ".." + self.selector_id.to_s
end
end
end
class Appid72PanaL7Pen < BinData::Record
# RFC6759 chapter 4.1: PANA-L7-PEN
# This implements the "application ids MAY be encoded with a larger length"
# Used in Application_Id72 choice statement
endian :big
uint32 :pen_id
uint32 :selector_id
end
class Application_Id72 < BinData::Primitive
endian :big
uint8 :classification_id
uint64 :selector_id
choice :selector_id, :selection => :classification_id do
# for classification engine id 20 we switch to Appid72PanaL7Pen to decode
appid72_pana_l7_pen 20
uint64 :default
end
def set(val)
unless val.nil?
self.classification_id=val.to_i<<64
self.selector_id = val.to_i-((val.to_i>>64)<<64)
self.classification_id = val.to_i<<64
if self.classification_id == 20
# classification engine id 20 (PANA_L7_PEN) contains a 4-byte PEN:
self.pen_id = val.to_i-((val.to_i>>64)<<64)>>32
self.selector_id = val.to_i-((val.to_i>>32)<<32)
else
self.selector_id = val.to_i-((val.to_i>>64)<<64)
end
end
end
def get
self.classification_id.to_s + ":" + self.selector_id.to_s
if self.classification_id == 20
self.classification_id.to_s + ".." + self.selector_id[:pen_id].to_s + ".." + self.selector_id[:selector_id].to_s
else
self.classification_id.to_s + ".." + self.selector_id.to_s
end
end
end
Expand Down
Binary file added spec/codecs/netflow9_test_fortigate_fortios_542_appid_data258_262.dat
View file
Binary file not shown.
Binary file added spec/codecs/netflow9_test_fortigate_fortios_542_appid_tpl258-269.dat
View file
Binary file not shown.
78 changes: 68 additions & 10 deletions spec/codecs/netflow_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1024,8 +1024,6 @@

end



context "Netflow 9 IE150 IE151" do
let(:data) do
packets = []
Expand Down Expand Up @@ -1103,6 +1101,66 @@

end


context "Netflow 9 Fortigate FortiOS 54x appid" do
let(:data) do
packets = []
packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_fortigate_fortios_542_appid_tpl258-269.dat"), :mode => "rb")
packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_fortigate_fortios_542_appid_data258_262.dat"), :mode => "rb")
end

let(:json_events) do
events = []
events << <<-END
{
"netflow": {
"output_snmp": 2,
"forwarding_status": {
"reason": 0,
"status": 1
},
"xlate_src_port": 45380,
"in_pkts": 6,
"ipv4_dst_addr": "182.50.136.239",
"first_switched": "2018-05-11T00:54:10.999Z",
"flowset_id": 262,
"l4_src_port": 45380,
"xlate_dst_port": 0,
"version": 9,
"application_id": "20..12356..36660",
"flow_seq_num": 350,
"ipv4_src_addr": "192.168.100.151",
"in_bytes": 748,
"protocol": 6,
"flow_end_reason": 3,
"last_switched": "2018-05-11T00:54:10.999Z",
"input_snmp": 8,
"out_pkts": 6,
"out_bytes": 748,
"xlate_src_addr_ipv4": "10.0.0.250",
"xlate_dst_addr_ipv4": "0.0.0.0",
"l4_dst_port": 80
},
"@timestamp": "2018-05-11T00:54:11.000Z",
"@version": "1"
}
END
events.map{|event| event.gsub(/\s+/, "")}
end

it "should decode raw data" do
expect(decode.size).to eq(17)
expect(decode[1].get("[netflow][application_id]")).to eq("20..12356..40568")
expect(decode[2].get("[netflow][application_id]")).to eq("20..12356..40568")
expect(decode[16].get("[netflow][application_id]")).to eq("20..12356..0")
end

it "should serialize to json" do
expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
end

end

context "IPFIX Nokia BRAS" do
let(:data) do
packets = []
Expand Down Expand Up @@ -1352,7 +1410,7 @@
"l4_src_port": 0,
"nprobe_proto_name": "\u0000\u00c1\u0000\u0000\u0001\u00ac\u0010\u0000d\u00e4O\u00ef\u00ff\u00ff\u00fa\u0007",
"version": 9,
"application_id": "0:82",
"application_id": "0..82",
"flow_seq_num": 2,
"ipv4_src_addr": "0.0.0.0",
"protocol": 0,
Expand All @@ -1372,7 +1430,7 @@
it "should decode raw data" do
expect(decode.size).to eq(1)
expect(decode[0].get("[netflow][nprobe_proto]")).to eq(82)
expect(decode[0].get("[netflow][application_id]")).to eq("0:82")
expect(decode[0].get("[netflow][application_id]")).to eq("0..82")
expect(decode[0].get("[netflow][in_bytes]")).to eq(82)
end

Expand Down Expand Up @@ -2291,7 +2349,7 @@
"application_description": "ARGUS",
"flowset_id": 260,
"version": 9,
"application_id": "1:13"
"application_id": "1..13"
},
"@timestamp": "2017-02-14T11:09:59.000Z",
"@version": "1"
Expand All @@ -2302,7 +2360,7 @@

it "should decode raw data" do
expect(decode.size).to eq(15)
expect(decode[14].get("[netflow][application_id]")).to eq("1:13")
expect(decode[14].get("[netflow][application_id]")).to eq("1..13")
expect(decode[14].get("[netflow][application_description]")).to eq("ARGUS")
end

Expand Down Expand Up @@ -2345,7 +2403,7 @@
"udp_dst_port": 161,
"src_mask": 0,
"version": 9,
"application_id": "5:38",
"application_id": "5..38",
"flow_seq_num": 1509134,
"ipv4_src_addr": "10.10.172.60",
"in_src_mac": "00:18:19:9e:6c:01",
Expand All @@ -2362,7 +2420,7 @@

it "should decode raw data" do
expect(decode.size).to eq(5)
expect(decode[4].get("[netflow][application_id]")).to eq("5:38")
expect(decode[4].get("[netflow][application_id]")).to eq("5..38")
end

it "should serialize to json" do
Expand All @@ -2388,7 +2446,7 @@
"staMacAddress": "34:02:86:75:c0:51",
"flowset_id": 261,
"version": 9,
"application_id": "13:431",
"application_id": "13..431",
"flow_seq_num": 78,
"in_bytes": 80973880,
"postIpDiffServCodePoint": 0,
Expand All @@ -2405,7 +2463,7 @@

it "should decode raw data" do
expect(decode.size).to eq(19)
expect(decode[18].get("[netflow][application_id]")).to eq("13:431")
expect(decode[18].get("[netflow][application_id]")).to eq("13..431")
end

it "should serialize to json" do
Expand Down

0 comments on commit 7a31f32

Please sign in to comment.