Translate plugin: Option to not fill array with nil values when applied to array (using iterate_on)

[fkelbert]

When applying the translate plugin to an array, the destination array will contain nil values or the fallback value in case no match was found in the dictionary. There should exist an option to not write any nil values (or fallback values) to the resulting array. nil values or fallback values may not be desired, in which case they will need to be removed using another ruby filter.

Example:

input {
	generator {
		lines => [
			'{ "ip_array": [ "1.1.1.1", "3.3.3.3" ] }',
			'{ "ip_array": [ "3.3.3.3", "4.4.4.4" ] }',
			'{ "ip_array": [ "1.1.1.1", "2.2.2.2" ] }'
		]
		count => 1
	}
	
}

filter {
	json { source => "message"}

	translate {
		field => "[ip_array]"
		iterate_on => "[ip_array]"
		destination => "[intelligence]"
		dictionary => {
			"1.1.1.1" => "spam"
			"2.2.2.2" => "bot"
		}
	}
}

output { stdout {} }

Output:

{
    "intelligence" => [
        [0] "spam",
        [1] "bot"
    ],
            "host" => "fk",
      "@timestamp" => 2019-01-25T09:18:18.101Z,
        "@version" => "1",
        "ip_array" => [
        [0] "1.1.1.1",
        [1] "2.2.2.2"
    ],
         "message" => "{ \"ip_array\": [ \"1.1.1.1\", \"2.2.2.2\" ] }",
        "sequence" => 0
}
{
    "intelligence" => [
        [0] nil,
        [1] nil
    ],
            "host" => "fk",
      "@timestamp" => 2019-01-25T09:18:18.100Z,
        "@version" => "1",
        "ip_array" => [
        [0] "3.3.3.3",
        [1] "4.4.4.4"
    ],
         "message" => "{ \"ip_array\": [ \"3.3.3.3\", \"4.4.4.4\" ] }",
        "sequence" => 0
}
{
    "intelligence" => [
        [0] "spam",
        [1] nil
    ],
            "host" => "fk",
      "@timestamp" => 2019-01-25T09:18:18.050Z,
        "@version" => "1",
        "ip_array" => [
        [0] "1.1.1.1",
        [1] "3.3.3.3"
    ],
         "message" => "{ \"ip_array\": [ \"1.1.1.1\", \"3.3.3.3\" ] }",
        "sequence" => 0

Desired output:

{
    "intelligence" => [
        [0] "spam",
        [1] "bot"
    ],
            "host" => "fk",
      "@timestamp" => 2019-01-25T09:18:18.101Z,
        "@version" => "1",
        "ip_array" => [
        [0] "1.1.1.1",
        [1] "2.2.2.2"
    ],
         "message" => "{ \"ip_array\": [ \"1.1.1.1\", \"2.2.2.2\" ] }",
        "sequence" => 0
}
{
    "intelligence" => [],   # maybe do not even create the field
            "host" => "fk",
      "@timestamp" => 2019-01-25T09:18:18.100Z,
        "@version" => "1",
        "ip_array" => [
        [0] "3.3.3.3",
        [1] "4.4.4.4"
    ],
         "message" => "{ \"ip_array\": [ \"3.3.3.3\", \"4.4.4.4\" ] }",
        "sequence" => 0
}
{
    "intelligence" => [
        [0] "spam"
    ],
            "host" => "fk",
      "@timestamp" => 2019-01-25T09:18:18.050Z,
        "@version" => "1",
        "ip_array" => [
        [0] "1.1.1.1",
        [1] "3.3.3.3"
    ],
         "message" => "{ \"ip_array\": [ \"1.1.1.1\", \"3.3.3.3\" ] }",
        "sequence" => 0