diff --git a/.gitignore b/.gitignore index 0cb6eeb..3789f49 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ /pkg/ /spec/reports/ /tmp/ +/vendor/ diff --git a/lib/fluent/plugin/out_gcs.rb b/lib/fluent/plugin/out_gcs.rb index 6a6334c..093f031 100644 --- a/lib/fluent/plugin/out_gcs.rb +++ b/lib/fluent/plugin/out_gcs.rb @@ -15,6 +15,7 @@ class GCSOutput < Output def initialize super require "google/cloud/storage" + Google::Apis.logger = log end config_param :project, :string, default: nil, @@ -130,7 +131,7 @@ def write(chunk) } opts.merge!(@encryption_opts) - log.debug { "out_gcs: upload chunk:#{chunk.key} to gcs://#{@bucket}/#{path} options: #{opts}" } + log.debug "out_gcs: upload chunk:#{chunk.key} to gcs://#{@bucket}/#{path} options: #{opts}" @gcs_bucket.upload_file(obj.path, path, opts) end end @@ -170,23 +171,30 @@ def generate_path(chunk, i = 0, prev = nil) "%{file_extension}" => @object_creator.file_extension, "%{hex_random}" => hex_random(chunk), "%{hostname}" => Socket.gethostname, - "%{index}" => i, "%{path}" => @path, "%{time_slice}" => time_slice, "%{uuid_flush}" => SecureRandom.uuid, } - path = @object_key_format.gsub(Regexp.union(tags.keys), tags) - path = extract_placeholders(path, chunk) - return path unless check_object_exists(path) - - if path == prev - if @overwrite - log.warn "object `#{path}` already exists but overwrites it" - return path + + until i < 0 do # Until overflow + tags["%{index}"] = i + path = @object_key_format.gsub(Regexp.union(tags.keys), tags) + path = extract_placeholders(path, chunk) + log.debug "checking if GCS path `#{path}` exists" + return path unless check_object_exists(path) + + if path == prev + if @overwrite + log.warn "object `#{path}` already exists but overwrites it" + return path + end + raise "object `#{path}` already exists" end - raise "object `#{path}` already exists" + + i += 1 end - generate_path(chunk, i + 1, path) + + raise "cannot find an unoccupied GCS path" end # This is stolen from Fluentd diff --git a/test/plugin/test_out_gcs.rb b/test/plugin/test_out_gcs.rb index fa42018..f46ac66 100644 --- a/test/plugin/test_out_gcs.rb +++ b/test/plugin/test_out_gcs.rb @@ -19,6 +19,9 @@ def setup @type memory timekey_use_utc true + + log_level debug + EOC def create_driver(conf = CONFIG)