diff --git a/Makefile b/Makefile
index c75c329..46120d0 100644
--- a/Makefile
+++ b/Makefile
@@ -12,11 +12,11 @@
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
-.PHONY: test install streams firehose producer dummer hello $(wildcard test/test_*.rb) $(wildcard test/**/test_*.rb)
+.PHONY: test install benchmark benchmark-streams benchmark-producer hello $(wildcard test/test_*.rb) $(wildcard test/**/test_*.rb)
all:
bundle install
- bundle exec rake
+ bundle exec rake binaries
test:
bundle exec rake test
@@ -24,17 +24,13 @@ test:
install:
bundle exec rake install:local
-streams:
- bundle exec fluentd -c benchmark/streams.conf -vv
+benchmark: benchmark-streams benchmark-producer
-firehose:
- bundle exec fluentd -c benchmark/firehose.conf -vv
+benchmark-streams:
+ bundle exec rake benchmark TYPE=streams
-producer:
- bundle exec fluentd -c benchmark/producer.conf -vv
-
-dummer:
- bundle exec dummer -c benchmark/dummer.conf
+benchmark-producer:
+ bundle exec rake benchmark TYPE=producer
hello:
echo Hello World | bundle exec fluent-cat --none dummy
diff --git a/README.md b/README.md
index c83dd0e..d58b201 100644
--- a/README.md
+++ b/README.md
@@ -527,15 +527,9 @@ To launch `fluentd` process with this plugin for development, follow the steps b
git clone https://github.com/awslabs/aws-fluent-plugin-kinesis.git
cd aws-fluent-plugin-kinesis
make # will install gems and download KPL jar file and extract binaries
- make [stream/firehose/producer]
+ bundle exec fluentd -c /path/to/fluent.conf
-Then, in another terminal, run the command below. It will emit one record.
-
- make hello
-
-Also, you can test streaming log data by `dummer`:
-
- make dummer # keep writing to /tmp/dummy.log
+If you want to run benchmark, use `make benchmark`.
## Contributing
diff --git a/Rakefile b/Rakefile
index f65aa6f..39d94dd 100644
--- a/Rakefile
+++ b/Rakefile
@@ -24,6 +24,7 @@ Rake::TestTask.new do |test|
end
load 'kinesis_producer/tasks/binary.rake'
-
Rake::Task[:build].enhance [:binaries]
Rake::Task[:test].enhance [:binaries]
+
+load 'benchmark/task.rake'
diff --git a/benchmark/dummer.conf b/benchmark/dummer.conf
deleted file mode 100644
index 8ccc46a..0000000
--- a/benchmark/dummer.conf
+++ /dev/null
@@ -1,13 +0,0 @@
-configure 'sample' do
- output "/tmp/dummy.log"
- rate 5000
- delimiter "\t"
- labeled true
- field :id, type: :integer, countup: true, format: "%04d"
- field :time, type: :datetime, format: "[%Y-%m-%d %H:%M:%S]", random: false
- field :level, type: :string, any: %w[DEBUG INFO WARN ERROR]
- field :method, type: :string, any: %w[GET POST PUT]
- field :uri, type: :string, any: %w[/api/v1/people /api/v1/textdata /api/v1/messages]
- field :reqtime, type: :float, range: 0.1..5.0
- field :foobar, type: :string, length: 8
-end
diff --git a/benchmark/dummy.conf b/benchmark/dummy.conf
new file mode 100644
index 0000000..e69de29
diff --git a/benchmark/firehose.conf b/benchmark/firehose.conf
deleted file mode 100644
index cded37b..0000000
--- a/benchmark/firehose.conf
+++ /dev/null
@@ -1,24 +0,0 @@
-
- @type tail
- path /tmp/dummy.log
- format none
- tag dummy
-
-
-
- @type forward
-
-
-
- @type kinesis_firehose
- flush_interval 1
- buffer_chunk_limit 1m
- try_flush_interval 0.1
- queued_chunk_flush_interval 0.01
- num_threads 15
- detach_process 5
- log_level debug
-
- region us-west-2
- delivery_stream_name fluent-plugin-test
-
diff --git a/benchmark/producer.conf b/benchmark/producer.conf
deleted file mode 100644
index acdfe85..0000000
--- a/benchmark/producer.conf
+++ /dev/null
@@ -1,28 +0,0 @@
-
- @type tail
- path /tmp/dummy.log
- format none
- tag dummy
-
-
-
- @type forward
-
-
-
- @type kinesis_producer
- flush_interval 1
- buffer_chunk_limit 1m
- try_flush_interval 0.1
- queued_chunk_flush_interval 0.01
- num_threads 15
- detach_process 5
- log_level debug
-
- region ap-northeast-1
- stream_name fluent-plugin-test
- debug true
-
- record_max_buffered_time 1000
-
-
diff --git a/benchmark/streams.conf b/benchmark/streams.conf
deleted file mode 100644
index 961b9e0..0000000
--- a/benchmark/streams.conf
+++ /dev/null
@@ -1,24 +0,0 @@
-
- @type tail
- path /tmp/dummy.log
- format none
- tag dummy
-
-
-
- @type forward
-
-
-
- @type kinesis_streams
- flush_interval 1
- buffer_chunk_limit 1m
- try_flush_interval 0.1
- queued_chunk_flush_interval 0.01
- num_threads 15
- detach_process 5
- log_level debug
-
- region ap-northeast-1
- stream_name fluent-plugin-test
-
diff --git a/benchmark/task.rake b/benchmark/task.rake
new file mode 100644
index 0000000..0a1b485
--- /dev/null
+++ b/benchmark/task.rake
@@ -0,0 +1,72 @@
+#
+# Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Amazon Software License (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/asl/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+require_relative '../test/dummy_server'
+
+task :benchmark do
+ server = DummyServer.start
+ conf = profile_conf(ENV["TYPE"] || 'streams', ENV["RATE"] || 1000, server)
+ pid = spawn("bundle exec fluentd -i '#{conf}' -c benchmark/dummy.conf")
+ sleep 10
+ Process.kill("TERM", pid)
+ Process.wait
+ puts "Results: requets: #{server.requests.size}, raw_records: #{server.raw_records.size}, records: #{server.records.size}"
+ server.shutdown
+end
+
+def profile_conf(type, rate, server)
+ additional_conf = case type
+ when 'streams', 'firehose'
+ <<-EOS
+ endpoint https://localhost:#{server.port}
+ ssl_verify_peer false
+ EOS
+ when 'producer'
+ <<-EOS
+ debug true
+
+ custom_endpoint localhost
+ port #{server.port}
+ verify_certificate false
+ record_max_buffered_time 1000
+ log_level error
+
+ EOS
+ end
+
+ conf = <<-EOS
+
+ @type dummy
+ tag dummy
+ rate #{rate}
+
+
+
+ @type kinesis_#{type}
+ flush_interval 1
+ buffer_chunk_limit 1m
+ try_flush_interval 0.1
+ queued_chunk_flush_interval 0.01
+ @log_level debug
+
+ num_threads 100
+
+ region ap-northeast-1
+ stream_name fluent-plugin-test
+
+#{additional_conf}
+
+ EOS
+ conf
+end
diff --git a/fluent-plugin-kinesis.gemspec b/fluent-plugin-kinesis.gemspec
index 613566d..609d1e4 100644
--- a/fluent-plugin-kinesis.gemspec
+++ b/fluent-plugin-kinesis.gemspec
@@ -46,8 +46,6 @@ Gem::Specification.new do |spec|
spec.add_development_dependency "pry-byebug", ">= 3.3.0"
spec.add_development_dependency "pry-stack_explorer", ">= 0.4.9.2"
spec.add_development_dependency "net-empty_port", ">= 0.0.2"
- # Due to the version conflict of msgpack between fluentd-0.14 and fluent-logger-0.5.1
- # spec.add_development_dependency "dummer", ">= 0.4.0"
spec.add_development_dependency "rubyzip", ">= 1.0.0"
spec.add_development_dependency "mocha", ">= 1.1.0"
spec.add_development_dependency "webmock", ">= 1.24.2"
diff --git a/test/dummy_server.rb b/test/dummy_server.rb
index f9b205d..b4509b8 100644
--- a/test/dummy_server.rb
+++ b/test/dummy_server.rb
@@ -73,6 +73,10 @@ def requests
@requests
end
+ def raw_records
+ @accepted_records
+ end
+
def records
flatten_records(@accepted_records)
end