Merge pull request #33 from fluent/fix/mp-utf8

set Data::MessagePack->utf8(1) by default.
fluent · Aug 16, 2024 · bb403d0 · bb403d0
2 parents 9da1f0d + bc68419
commit bb403d0
Show file tree

Hide file tree

Showing 4 changed files with 48 additions and 1 deletion.
diff --git a/README.md b/README.md
@@ -40,6 +40,7 @@ Fluent::Logger is a structured event logger for Fluentd.
         timeout                     => 'Num':  default is 3.0
         socket                      => 'Str':  default undef (e.g. "/var/run/fluent/fluent.sock")
         prefer_integer              => 'Bool': default 1 (set to Data::MessagePack->prefer_integer)
+        utf8                        => 'Bool': default 1 (set to Data::MessagePack->utf8)
         event_time                  => 'Bool': default 0 (timestamp includes nanoseconds, supported by fluentd >= 0.14.0)
         buffer_limit                => 'Int':  defualt 8388608 (8MB)
         buffer_overflow_handler     => 'Code': optional

diff --git a/lib/Fluent/Logger.pm b/lib/Fluent/Logger.pm
@@ -42,9 +42,11 @@ use Class::Tiny +{
     socket_io => sub {},
     errors => sub { [] },
     prefer_integer => sub { 1 },
+    utf8 => sub { 1 },
     packer => sub {
         my $self = shift;
         my $mp   = Data::MessagePack->new;
+        $mp->utf8( $self->utf8 );
         $mp->prefer_integer( $self->prefer_integer );
         $mp;
     },
@@ -402,6 +404,7 @@ create new logger instance.
     timeout                     => 'Num':  default is 3.0
     socket                      => 'Str':  default undef (e.g. "/var/run/fluent/fluent.sock")
     prefer_integer              => 'Bool': default 1 (set to Data::MessagePack->prefer_integer)
+    utf8                        => 'Bool': default 1 (set to Data::MessagePack->utf8)
     event_time                  => 'Bool': default 0 (timestamp includes nanoseconds, supported by fluentd >= 0.14.0)
     buffer_limit                => 'Int':  defualt 8388608 (8MB)
     buffer_overflow_handler     => 'Code': optional

diff --git a/t/07_str_bin.t b/t/07_str_bin.t
@@ -21,17 +21,20 @@ my $port = $server->port;
 use_ok "Fluent::Logger";
 
 subtest str_bin => sub {
-    my $logger = Fluent::Logger->new( port => $port );
+    my $logger = Fluent::Logger->new( port => $port, utf8 => 0 );
 
     isa_ok $logger, "Fluent::Logger";
+    is $logger->packer->get_utf8, 0, "packer utf8 is off";
     my $tag = "test.tcp";
     ok $logger->post( $tag, { "foo" => decode_utf8("内部文字列") }), "post str ok";
     ok $logger->post( $tag, { "bar" => "バイナリ列" }), "post bin ok";
+    ok $logger->post( $tag, { "broken" => "\xE0\x80\xAF" }), "post broken utf8 ok";
     sleep 1;
     my $log = slurp_log $dir;
     note $log;
     like $log => qr/"foo":"内部文字列","tag":"$tag"/, "match post str log";
     like $log => qr/"bar":"バイナリ列","tag":"$tag"/, "match post bin log";
+    like $log => qr/"broken":"\xE0\x80\xAF","tag":"$tag"/, "match post broken utf8 log";
 };
 
 done_testing;
diff --git a/t/07_str_utf8.t b/t/07_str_utf8.t
@@ -0,0 +1,40 @@
+use strict;
+use warnings;
+use FindBin;
+use lib "$FindBin::Bin/../";
+use Test::More;
+use Test::TCP;
+use Time::Piece;
+use Encode;
+use t::Util qw/ run_fluentd slurp_log /;
+use POSIX qw/ setlocale LC_ALL /;
+use Capture::Tiny qw/ capture /;
+
+use Config;
+if ( $Config{d_setlocale} ) {
+    setlocale(LC_ALL, "C");
+}
+
+my ($server, $dir) = run_fluentd();
+my $port = $server->port;
+
+use_ok "Fluent::Logger";
+
+subtest str_bin => sub {
+    my $logger = Fluent::Logger->new( port => $port, utf8 => 1 );
+
+    isa_ok $logger, "Fluent::Logger";
+    is $logger->packer->get_utf8, 1, "packer utf8 is on";
+    my $tag = "test.tcp";
+    ok $logger->post( $tag, { "foo" => decode_utf8("内部文字列") }), "post str ok";
+    ok $logger->post( $tag, { "bar" => "バイナリ列" }), "post bin ok";
+    ok $logger->post( $tag, { "broken" => "\xE0\x80\xAF" }), "post broken utf8 ok";
+    sleep 1;
+    my $log = slurp_log $dir;
+    note $log;
+    like $log => qr/"foo":"内部文字列","tag":"$tag"/, "match post str log";
+    like $log => qr/"bar":"バイナリ列","tag":"$tag"/, "match post bin log";
+    like $log => qr/"broken":"\xE0\x80\xAF","tag":"$tag"/, "match post broken utf8 log";
+};
+
+done_testing;