From c5f2829e7ddb22ec1ce96207ad5b24a2eabd3621 Mon Sep 17 00:00:00 2001 From: Rollin Chen Date: Fri, 17 Apr 2015 15:25:03 +0800 Subject: [PATCH 1/2] make codec configurable move file to bucket in graceful shutdown --- lib/logstash/outputs/s3.rb | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/logstash/outputs/s3.rb b/lib/logstash/outputs/s3.rb index faafe019..2db90b52 100644 --- a/lib/logstash/outputs/s3.rb +++ b/lib/logstash/outputs/s3.rb @@ -58,7 +58,6 @@ # bucket => "boss_please_open_your_bucket" (required) # size_file => 2048 (optional) # time_file => 5 (optional) -# format => "plain" (optional) # canned_acl => "private" (optional. Options are "private", "public_read", "public_read_write", "authenticated_read". Defaults to "private" ) # } # @@ -69,7 +68,8 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base S3_INVALID_CHARACTERS = /[\^`><]/ config_name "s3" - default :codec, 'line' + config :codec, :validate => :codec, + :default => 'line' # S3 bucket config :bucket, :validate => :string @@ -123,7 +123,7 @@ def aws_s3_config def aws_service_endpoint(region) # Make the deprecated endpoint_region work # TODO: (ph) Remove this after deprecation. - + if @endpoint_region region_to_use = @endpoint_region else @@ -230,7 +230,7 @@ def test_s3_write File.delete(test_filename) end end - + public def restore_from_crashes @logger.debug("S3: is attempting to verify previous crashes...") @@ -308,10 +308,14 @@ def write_to_tempfile(event) public def teardown + #close temp file and ready to upload if needed + @tempfile.close + # upload current temporary file if needed + move_file_to_bucket(@tempfile.path) + shutdown_upload_workers @periodic_rotation_thread.stop! if @periodic_rotation_thread - @tempfile.close finished end @@ -333,7 +337,7 @@ def handle_event(encoded_event) else @logger.debug("S3: tempfile file size report.", :tempfile_size => @tempfile.size, :size_file => @size_file) end - end + end write_to_tempfile(encoded_event) end From 34299726d5852bcba4e0f04880f654c2e571518f Mon Sep 17 00:00:00 2001 From: Rollin Chen Date: Tue, 2 Jun 2015 17:25:05 +0800 Subject: [PATCH 2/2] remove duplicate codec in base add move_file_to_bucket flag to disable the move file in shutdown --- lib/logstash/outputs/s3.rb | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/logstash/outputs/s3.rb b/lib/logstash/outputs/s3.rb index 2db90b52..6b8dfa6e 100644 --- a/lib/logstash/outputs/s3.rb +++ b/lib/logstash/outputs/s3.rb @@ -68,8 +68,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base S3_INVALID_CHARACTERS = /[\^`><]/ config_name "s3" - config :codec, :validate => :codec, - :default => 'line' + default :codec, 'line' # S3 bucket config :bucket, :validate => :string @@ -110,6 +109,9 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base # Specify how many workers to use to upload the files to S3 config :upload_workers_count, :validate => :number, :default => 1 + # Flags for enable move file to s3 in tear down. It's useful if you need to automatic teminate some EC2 instances in aws. + config :move_file_in_tear_down, :validate => :boolean, :default => false + # Exposed attributes for testing purpose. attr_accessor :tempfile attr_reader :page_counter @@ -308,10 +310,13 @@ def write_to_tempfile(event) public def teardown - #close temp file and ready to upload if needed + #close temp file @tempfile.close - # upload current temporary file if needed - move_file_to_bucket(@tempfile.path) + + if @move_file_in_tear_down == true + # upload current temporary file if needed + move_file_to_bucket(@tempfile.path) + end shutdown_upload_workers @periodic_rotation_thread.stop! if @periodic_rotation_thread