From f300114eae4a45627e2f96fa3bd9eeeadf57de76 Mon Sep 17 00:00:00 2001 From: Charly Koza Date: Wed, 14 Sep 2016 15:21:35 +0200 Subject: [PATCH 1/7] nested_objects implemented fixes https://github.com/logstash-plugins/logstash-input-gelf/issues/24 --- lib/logstash/inputs/gelf.rb | 69 +++++++++++++++++++++++++++++++++++++ spec/inputs/gelf_spec.rb | 49 ++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) diff --git a/lib/logstash/inputs/gelf.rb b/lib/logstash/inputs/gelf.rb index ec9443d..5ebae6d 100644 --- a/lib/logstash/inputs/gelf.rb +++ b/lib/logstash/inputs/gelf.rb @@ -51,6 +51,9 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base # config :strip_leading_underscore, :validate => :boolean, :default => true + # Whether or not to process dots in fields or leave them in place + config :nested_objects, :validate => :boolean, :default => false + RECONNECT_BACKOFF_SLEEP = 5 TIMESTAMP_GELF_FIELD = "timestamp".freeze SOURCE_HOST_FIELD = "source_host".freeze @@ -115,6 +118,7 @@ def udp_listener(output_queue) remap_gelf(event) if @remap strip_leading_underscore(event) if @strip_leading_underscore + nested_objects(event) if @nested_objects decorate(event) output_queue << event @@ -196,4 +200,69 @@ def strip_leading_underscore(event) event.remove(key) end end # deef removing_leading_underscores + + private + def nested_objects(event) + # process nested, create objects as needed, when key is 0, create an array. if object already exists and is an array push it. + base_target=event.to_hash + base_target.keys.each do |key| + next unless key.include? "." + value = event.get(key) + previous_key = nil + first_key=nil + target = base_target + + key.split(".").each do |subKey| + if previous_key.nil? + first_key=subKey + else#skip first subKey + if !container_has_element?(target, previous_key) + if subKey =~ /^\d+$/ + set_container_element(target, previous_key, Array.new) + else + set_container_element(target, previous_key, Hash.new) + end + end + target = get_container_element(target, previous_key) + end + previous_key = subKey + end + set_container_element(target, previous_key, value) + event.remove(key) + event.set(first_key, base_target[first_key]) + end + end + + private + def get_container_element(container, key) + if container.is_a?(Array) + container[Integer(key)] + elsif container.is_a?(Hash) + container[key] + else #Event + raise "not an array or hash" + end + end + + private + def set_container_element(container, key, value) + if container.is_a?(Array) + container[Integer(key)] = value + elsif container.is_a?(Hash) + container[key] = value + else #Event + raise "not an array or hash" + end + end + + private + def container_has_element?(container, key) + if container.is_a?(Array) + !container[Integer(key)].nil? + elsif container.is_a?(Hash) + container.key?(key) + else + raise "not an array or hash" + end + end end # class LogStash::Inputs::Gelf diff --git a/spec/inputs/gelf_spec.rb b/spec/inputs/gelf_spec.rb index 2eec09b..10d6a53 100644 --- a/spec/inputs/gelf_spec.rb +++ b/spec/inputs/gelf_spec.rb @@ -70,6 +70,55 @@ end end + it "reads nested gelf messages " do + port = 12210 + host = "127.0.0.1" + chunksize = 1420 + gelfclient = GELF::Notifier.new(host, port, chunksize) + + conf = <<-CONFIG + input { + gelf { + port => "#{port}" + host => "#{host}" + nested_objects => true + } + } + CONFIG + + result = input(conf) do |pipeline, queue| + # send a first message until plugin is up and receives it + while queue.size <= 0 + gelfclient.notify!("short_message" => "prime") + sleep(0.1) + end + gelfclient.notify!("short_message" => "start") + + e = queue.pop + while (e.get("message") != "start") + e = queue.pop + end + + gelfclient.notify!({ + "short_message" => "test nested", + "_toto.titi" => "objectValue", + "_foo.0" => "first", + "_foo.1" => "second", + "_ca.0.titi" => "1", + "_ca.1.titi" => "2", + }) + + queue.pop + end + + insist { result.get("message") } == "test nested" + insist { result.get("toto")["titi"] } == "objectValue" + insist { result.get("foo") } == ["first", "second"] + insist { result.get("ca")[0]["titi"] } == "1" + insist { result.get("ca")[1]["titi"] } == "2" + insist { result.get("host") } == Socket.gethostname + end + context "timestamp coercion" do # these test private methods, this is advisable for now until we roll out this coercion in the Timestamp class # and remove this From 5f487e2a93d2b14fe6dd71f422c406645c0b7b96 Mon Sep 17 00:00:00 2001 From: Charly Koza Date: Tue, 15 Nov 2016 17:41:28 +0100 Subject: [PATCH 2/7] style --- lib/logstash/inputs/gelf.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/logstash/inputs/gelf.rb b/lib/logstash/inputs/gelf.rb index 5ebae6d..edc0477 100644 --- a/lib/logstash/inputs/gelf.rb +++ b/lib/logstash/inputs/gelf.rb @@ -204,17 +204,17 @@ def strip_leading_underscore(event) private def nested_objects(event) # process nested, create objects as needed, when key is 0, create an array. if object already exists and is an array push it. - base_target=event.to_hash + base_target = event.to_hash base_target.keys.each do |key| next unless key.include? "." value = event.get(key) previous_key = nil - first_key=nil + first_key = nil target = base_target key.split(".").each do |subKey| if previous_key.nil? - first_key=subKey + first_key = subKey else#skip first subKey if !container_has_element?(target, previous_key) if subKey =~ /^\d+$/ From ec8cdc7f4c8d8234dc6e6d82e518ea40e0c9d3bb Mon Sep 17 00:00:00 2001 From: Charly Koza Date: Fri, 21 Oct 2016 15:51:57 +0200 Subject: [PATCH 3/7] test with empty string as key in object --- spec/inputs/gelf_spec.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/inputs/gelf_spec.rb b/spec/inputs/gelf_spec.rb index 10d6a53..99f59ca 100644 --- a/spec/inputs/gelf_spec.rb +++ b/spec/inputs/gelf_spec.rb @@ -106,6 +106,7 @@ "_foo.1" => "second", "_ca.0.titi" => "1", "_ca.1.titi" => "2", + "_empty." => "pouet", }) queue.pop @@ -116,6 +117,7 @@ insist { result.get("foo") } == ["first", "second"] insist { result.get("ca")[0]["titi"] } == "1" insist { result.get("ca")[1]["titi"] } == "2" + insist { result.get("empty")[""]} == "pouet" insist { result.get("host") } == Socket.gethostname end From d7e70b17c13d0e81c4d773978cfc8e773ea023d6 Mon Sep 17 00:00:00 2001 From: Charly Koza Date: Tue, 15 Nov 2016 17:49:15 +0100 Subject: [PATCH 4/7] handle dot at the end --- lib/logstash/inputs/gelf.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/logstash/inputs/gelf.rb b/lib/logstash/inputs/gelf.rb index edc0477..cf5a410 100644 --- a/lib/logstash/inputs/gelf.rb +++ b/lib/logstash/inputs/gelf.rb @@ -212,7 +212,12 @@ def nested_objects(event) first_key = nil target = base_target - key.split(".").each do |subKey| + keys = key.split(".") + if key =~ /\.$/ + keys.push(""); + end + + keys.each do |subKey| if previous_key.nil? first_key = subKey else#skip first subKey From 3a98c82f8a92b62e310a700311abe20f4fe690e7 Mon Sep 17 00:00:00 2001 From: Charly Koza Date: Tue, 15 Nov 2016 17:55:18 +0100 Subject: [PATCH 5/7] hash looking like an array test --- spec/inputs/gelf_spec.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spec/inputs/gelf_spec.rb b/spec/inputs/gelf_spec.rb index 99f59ca..eea10df 100644 --- a/spec/inputs/gelf_spec.rb +++ b/spec/inputs/gelf_spec.rb @@ -107,6 +107,9 @@ "_ca.0.titi" => "1", "_ca.1.titi" => "2", "_empty." => "pouet", + "_not_an_array.0" => "bob", + "_not_an_array.1" => "alice", + "_not_an_array.length" => "carol", }) queue.pop @@ -118,6 +121,9 @@ insist { result.get("ca")[0]["titi"] } == "1" insist { result.get("ca")[1]["titi"] } == "2" insist { result.get("empty")[""]} == "pouet" + insist { result.get("not_an_array")["0"]} == "bob" + insist { result.get("not_an_array")["1"]} == "alice" + insist { result.get("not_an_array")["length"]} == "carol" insist { result.get("host") } == Socket.gethostname end From f78ee1c6f603cd19871796fe83a8af6c9bd9230c Mon Sep 17 00:00:00 2001 From: Charly Koza Date: Tue, 15 Nov 2016 18:19:52 +0100 Subject: [PATCH 6/7] convert from array to hash if needed --- lib/logstash/inputs/gelf.rb | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/logstash/inputs/gelf.rb b/lib/logstash/inputs/gelf.rb index cf5a410..a3ac5fa 100644 --- a/lib/logstash/inputs/gelf.rb +++ b/lib/logstash/inputs/gelf.rb @@ -223,16 +223,16 @@ def nested_objects(event) else#skip first subKey if !container_has_element?(target, previous_key) if subKey =~ /^\d+$/ - set_container_element(target, previous_key, Array.new) + target = set_container_element(target, previous_key, Array.new) else - set_container_element(target, previous_key, Hash.new) + target = set_container_element(target, previous_key, Hash.new) end end target = get_container_element(target, previous_key) end previous_key = subKey end - set_container_element(target, previous_key, value) + target = set_container_element(target, previous_key, value) event.remove(key) event.set(first_key, base_target[first_key]) end @@ -252,12 +252,21 @@ def get_container_element(container, key) private def set_container_element(container, key, value) if container.is_a?(Array) - container[Integer(key)] = value + if !/\A\d+\z/.match(key) + #key is not an integer, so we need to convert array to hash + container = Hash[container.map.with_index { |x, i| [i, x] }] + container[key] = value + return container + else + container[Integer(key)] = value + end elsif container.is_a?(Hash) container[key] = value else #Event raise "not an array or hash" end + + return container end private From d57ce02f6edb64e7d291ef0817b70b29cf9fecdb Mon Sep 17 00:00:00 2001 From: Charly Koza Date: Wed, 16 Nov 2016 11:45:44 +0100 Subject: [PATCH 7/7] if key is not an integer, it cannot be in array --- lib/logstash/inputs/gelf.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/logstash/inputs/gelf.rb b/lib/logstash/inputs/gelf.rb index a3ac5fa..2b95815 100644 --- a/lib/logstash/inputs/gelf.rb +++ b/lib/logstash/inputs/gelf.rb @@ -272,7 +272,11 @@ def set_container_element(container, key, value) private def container_has_element?(container, key) if container.is_a?(Array) - !container[Integer(key)].nil? + if !/\A\d+\z/.match(key) + return false + else + !container[Integer(key)].nil? + end elsif container.is_a?(Hash) container.key?(key) else