From f5c932d87ee57481bbc16b08dbe1ef984724023a Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Tue, 16 Feb 2021 18:05:37 -0500 Subject: [PATCH 01/17] [Test] Testing with different architecture --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a50fc739..76744077 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,2 +1,6 @@ import: -- logstash-plugins/.ci:travis/travis.yml@1.x \ No newline at end of file +- logstash-plugins/.ci:travis/travis.yml@1.x + +arch: + - amd64 + - arm64 \ No newline at end of file From c931ddf6cf68e8a3e581311402bce8a2752298de Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Tue, 16 Feb 2021 18:11:24 -0500 Subject: [PATCH 02/17] Install jq --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 76744077..e3d56466 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ import: - logstash-plugins/.ci:travis/travis.yml@1.x +before_install: + - mkdir -p .ci && curl -sL https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards "*Dockerfile*" "*docker*" "*.sh" + - sudo apt-get install jq arch: - - amd64 - arm64 \ No newline at end of file From d4cf673b327d4752d71726a4d87a3eefd472ec41 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 17 Feb 2021 10:57:50 -0500 Subject: [PATCH 03/17] Add libseccomp --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e3d56466..6b4e95d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,5 +4,6 @@ import: before_install: - mkdir -p .ci && curl -sL https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards "*Dockerfile*" "*docker*" "*.sh" - sudo apt-get install jq + - sudo apt-get install libseccomp-dev arch: - arm64 \ No newline at end of file From 5177cdc36e1a18f0c7e523c60ce4808925240db3 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 17 Feb 2021 11:44:10 -0500 Subject: [PATCH 04/17] Add logging and skip to sincedb write in case of permissions snafu --- lib/filewatch/helper.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/filewatch/helper.rb b/lib/filewatch/helper.rb index 5da8c0fd..6bd32749 100644 --- a/lib/filewatch/helper.rb +++ b/lib/filewatch/helper.rb @@ -38,8 +38,13 @@ def write_atomically(file_name) return return_val if old_stat.nil? # Set correct uid/gid on new file - File.chown(old_stat.uid, old_stat.gid, file_name) if old_stat - + if old_stat + begin + File.chown(old_stat.uid, old_stat.gid, file_name) + rescue Errno::EPERM => e + logger.error("sincedb_write: #{file_name} error:", :old_stat => old_stat, :exception => e.class, :message => e.message) + end + end return_val end From 4095916f6c025614ebed59adae190a14f3aa8f2e Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 17 Feb 2021 12:08:34 -0500 Subject: [PATCH 05/17] Skipping hanging test --- spec/inputs/file_tail_spec.rb | 624 +++++++++++++++++----------------- 1 file changed, 312 insertions(+), 312 deletions(-) diff --git a/spec/inputs/file_tail_spec.rb b/spec/inputs/file_tail_spec.rb index 7aefc69c..c064891c 100644 --- a/spec/inputs/file_tail_spec.rb +++ b/spec/inputs/file_tail_spec.rb @@ -196,316 +196,316 @@ end - describe "testing with new, register, run and stop" do - let(:suffix) { "A" } - let(:conf) { Hash.new } - let(:mlconf) { Hash.new } - let(:events) { Array.new } - let(:mlcodec) { LogStash::Codecs::Multiline.new(mlconf) } - let(:tracer_codec) { FileInput::CodecTracer.new } - let(:tmpdir_path) { Stud::Temporary.directory } - let(:tmpfile_path) { ::File.join(tmpdir_path, "#{suffix}.txt") } - let(:path_path) { ::File.join(tmpdir_path, "*.txt") } - let(:sincedb_path) { ::File.join(tmpdir_path, "sincedb-#{suffix}") } - - after :each do - sleep(0.1) until subject.completely_stopped? - FileUtils.rm_rf(sincedb_path) - end - - context "when data exists and then more data is appended" do - subject { described_class.new(conf) } - - before do - File.open(tmpfile_path, "w") do |fd| - fd.puts("ignore me 1") - fd.puts("ignore me 2") - fd.fsync - end - mlconf.update("pattern" => "^\s", "what" => "previous") - conf.update("type" => "blah", - "path" => path_path, - "sincedb_path" => sincedb_path, - "stat_interval" => 0.1, - "codec" => mlcodec, - "delimiter" => TEST_FILE_DELIMITER) - end - - it "reads the appended data only" do - subject.register - actions = RSpec::Sequencing - .run_after(1, "append two lines after delay") do - File.open(tmpfile_path, "a") { |fd| fd.puts("hello"); fd.puts("world") } - end - .then("wait for one event") do - wait(0.75).for{events.size}.to eq(1) - end - .then("quit") do - subject.stop - end - .then("wait for flushed event") do - wait(0.75).for{events.size}.to eq(2) - end - - subject.run(events) - actions.assert_no_errors - - event1 = events[0] - expect(event1).not_to be_nil - expect(event1.get("path")).to eq tmpfile_path - expect(event1.get("[@metadata][path]")).to eq tmpfile_path - expect(event1.get("message")).to eq "hello" - - event2 = events[1] - expect(event2).not_to be_nil - expect(event2.get("path")).to eq tmpfile_path - expect(event2.get("[@metadata][path]")).to eq tmpfile_path - expect(event2.get("message")).to eq "world" - end - end - - context "when close_older config is specified" do - let(:line) { "line1.1-of-a" } - let(:suffix) { "X" } - subject { described_class.new(conf) } - - before do - conf.update( - "type" => "blah", - "path" => path_path, - "sincedb_path" => sincedb_path, - "stat_interval" => 0.02, - "codec" => tracer_codec, - "close_older" => "100 ms", - "start_position" => "beginning", - "delimiter" => TEST_FILE_DELIMITER) - - subject.register - end - - it "having timed_out, the codec is auto flushed" do - actions = RSpec::Sequencing - .run("create file") do - File.open(tmpfile_path, "wb") { |file| file.puts(line) } - end - .then_after(0.1, "identity is mapped") do - wait(0.75).for{subject.codec.identity_map[tmpfile_path]}.not_to be_nil, "identity is not mapped" - end - .then("wait for auto_flush") do - wait(0.75).for{subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush)}.to eq([true]), "autoflush didn't" - end - .then("quit") do - subject.stop - end - subject.run(events) - actions.assert_no_errors - expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq([true]) - end - end - - context "when ignore_older config is specified" do - let(:suffix) { "Y" } - before do - conf.update( - "type" => "blah", - "path" => path_path, - "sincedb_path" => sincedb_path, - "stat_interval" => 0.02, - "codec" => tracer_codec, - "ignore_older" => "500 ms", - "delimiter" => TEST_FILE_DELIMITER) - end - subject { described_class.new(conf) } - let(:line) { "line1.1-of-a" } - - it "the file is not read" do - subject.register - RSpec::Sequencing - .run("create file") do - File.open(tmp_dir_file, "a") do |fd| - fd.puts(line) - fd.fsync - end - FileInput.make_file_older(tmp_dir_file, 2) - end - .then_after(0.5, "stop") do - subject.stop - end - subject.run(events) - expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to be_falsey - end - end - - context "when wildcard path and a multiline codec is specified" do - subject { described_class.new(conf) } - let(:suffix) { "J" } - let(:tmpfile_path2) { ::File.join(tmpdir_path, "K.txt") } - before do - mlconf.update("pattern" => "^\s", "what" => "previous") - conf.update( - "type" => "blah", - "path" => path_path, - "start_position" => "beginning", - "sincedb_path" => sincedb_path, - "stat_interval" => 0.05, - "codec" => mlcodec, - "file_sort_by" => "path", - "delimiter" => TEST_FILE_DELIMITER) - - subject.register - end - - it "collects separate multiple line events from each file" do - subject - actions = RSpec::Sequencing - .run_after(0.1, "create files") do - File.open(tmpfile_path, "wb") do |fd| - fd.puts("line1.1-of-J") - fd.puts(" line1.2-of-J") - fd.puts(" line1.3-of-J") - end - File.open(tmpfile_path2, "wb") do |fd| - fd.puts("line1.1-of-K") - fd.puts(" line1.2-of-K") - fd.puts(" line1.3-of-K") - end - end - .then("assert both files are mapped as identities and stop") do - wait(2).for {subject.codec.identity_count}.to eq(2), "both files are not mapped as identities" - end - .then("stop") do - subject.stop - end - subject.run(events) - # wait for actions to complete - actions.assert_no_errors - expect(events.size).to eq(2) - e1, e2 = events - e1_message = e1.get("message") - e2_message = e2.get("message") - - expect(e1.get("path")).to match(/J.txt/) - expect(e2.get("path")).to match(/K.txt/) - expect(e1_message).to eq("line1.1-of-J#{TEST_FILE_DELIMITER} line1.2-of-J#{TEST_FILE_DELIMITER} line1.3-of-J") - expect(e2_message).to eq("line1.1-of-K#{TEST_FILE_DELIMITER} line1.2-of-K#{TEST_FILE_DELIMITER} line1.3-of-K") - end - - context "if auto_flush is enabled on the multiline codec" do - let(:mlconf) { { "auto_flush_interval" => 0.5 } } - let(:suffix) { "M" } - it "an event is generated via auto_flush" do - actions = RSpec::Sequencing - .run_after(0.1, "create files") do - File.open(tmpfile_path, "wb") do |fd| - fd.puts("line1.1-of-a") - fd.puts(" line1.2-of-a") - fd.puts(" line1.3-of-a") - end - end - .then("wait for auto_flush") do - wait(2).for{events.size}.to eq(1), "events size is not 1" - end - .then("stop") do - subject.stop - end - subject.run(events) - # wait for actions to complete - actions.assert_no_errors - e1 = events.first - e1_message = e1.get("message") - expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER} line1.2-of-a#{TEST_FILE_DELIMITER} line1.3-of-a") - expect(e1.get("path")).to match(/M.txt$/) - end - end - end - - describe "specifying max_open_files" do - let(:suffix) { "P" } - let(:tmpfile_path2) { ::File.join(tmpdir_path, "Q.txt") } - subject { described_class.new(conf) } - before do - File.open(tmpfile_path, "w") do |fd| - fd.puts("line1-of-P") - fd.puts("line2-of-P") - fd.fsync - end - File.open(tmpfile_path2, "w") do |fd| - fd.puts("line1-of-Q") - fd.puts("line2-of-Q") - fd.fsync - end - end - - context "when close_older is NOT specified" do - before do - conf.clear - conf.update( - "type" => "blah", - "path" => path_path, - "sincedb_path" => sincedb_path, - "stat_interval" => 0.1, - "max_open_files" => 1, - "start_position" => "beginning", - "file_sort_by" => "path", - "delimiter" => TEST_FILE_DELIMITER) - subject.register - end - it "collects line events from only one file" do - actions = RSpec::Sequencing - .run("assert one identity is mapped") do - wait(0.4).for{subject.codec.identity_count}.to be > 0, "no identity is mapped" - end - .then("stop") do - subject.stop - end - .then("stop flushes last event") do - wait(0.4).for{events.size}.to eq(2), "events size does not equal 2" - end - subject.run(events) - # wait for actions future value - actions.assert_no_errors - e1, e2 = events - expect(e1.get("message")).to eq("line1-of-P") - expect(e2.get("message")).to eq("line2-of-P") - end - end - - context "when close_older IS specified" do - before do - conf.update( - "type" => "blah", - "path" => path_path, - "sincedb_path" => sincedb_path, - "stat_interval" => 0.1, - "max_open_files" => 1, - "close_older" => 0.5, - "start_position" => "beginning", - "file_sort_by" => "path", - "delimiter" => TEST_FILE_DELIMITER) - subject.register - end - - it "collects line events from both files" do - actions = RSpec::Sequencing - .run("assert both identities are mapped and the first two events are built") do - wait(0.4).for{subject.codec.identity_count == 1 && events.size == 2}.to eq(true), "both identities are not mapped and the first two events are not built" - end - .then("wait for close to flush last event of each identity") do - wait(0.8).for{events.size}.to eq(4), "close does not flush last event of each identity" - end - .then_after(0.1, "stop") do - subject.stop - end - subject.run(events) - # wait for actions future value - actions.assert_no_errors - e1, e2, e3, e4 = events - expect(e1.get("message")).to eq("line1-of-P") - expect(e2.get("message")).to eq("line2-of-P") - expect(e3.get("message")).to eq("line1-of-Q") - expect(e4.get("message")).to eq("line2-of-Q") - end - end - end - end + # describe "testing with new, register, run and stop" do + # let(:suffix) { "A" } + # let(:conf) { Hash.new } + # let(:mlconf) { Hash.new } + # let(:events) { Array.new } + # let(:mlcodec) { LogStash::Codecs::Multiline.new(mlconf) } + # let(:tracer_codec) { FileInput::CodecTracer.new } + # let(:tmpdir_path) { Stud::Temporary.directory } + # let(:tmpfile_path) { ::File.join(tmpdir_path, "#{suffix}.txt") } + # let(:path_path) { ::File.join(tmpdir_path, "*.txt") } + # let(:sincedb_path) { ::File.join(tmpdir_path, "sincedb-#{suffix}") } + # + # after :each do + # sleep(0.1) until subject.completely_stopped? + # FileUtils.rm_rf(sincedb_path) + # end + # + # context "when data exists and then more data is appended" do + # subject { described_class.new(conf) } + # + # before do + # File.open(tmpfile_path, "w") do |fd| + # fd.puts("ignore me 1") + # fd.puts("ignore me 2") + # fd.fsync + # end + # mlconf.update("pattern" => "^\s", "what" => "previous") + # conf.update("type" => "blah", + # "path" => path_path, + # "sincedb_path" => sincedb_path, + # "stat_interval" => 0.1, + # "codec" => mlcodec, + # "delimiter" => TEST_FILE_DELIMITER) + # end + # + # it "reads the appended data only" do + # subject.register + # actions = RSpec::Sequencing + # .run_after(1, "append two lines after delay") do + # File.open(tmpfile_path, "a") { |fd| fd.puts("hello"); fd.puts("world") } + # end + # .then("wait for one event") do + # wait(0.75).for{events.size}.to eq(1) + # end + # .then("quit") do + # subject.stop + # end + # .then("wait for flushed event") do + # wait(0.75).for{events.size}.to eq(2) + # end + # + # subject.run(events) + # actions.assert_no_errors + # + # event1 = events[0] + # expect(event1).not_to be_nil + # expect(event1.get("path")).to eq tmpfile_path + # expect(event1.get("[@metadata][path]")).to eq tmpfile_path + # expect(event1.get("message")).to eq "hello" + # + # event2 = events[1] + # expect(event2).not_to be_nil + # expect(event2.get("path")).to eq tmpfile_path + # expect(event2.get("[@metadata][path]")).to eq tmpfile_path + # expect(event2.get("message")).to eq "world" + # end + # end + # + # context "when close_older config is specified" do + # let(:line) { "line1.1-of-a" } + # let(:suffix) { "X" } + # subject { described_class.new(conf) } + # + # before do + # conf.update( + # "type" => "blah", + # "path" => path_path, + # "sincedb_path" => sincedb_path, + # "stat_interval" => 0.02, + # "codec" => tracer_codec, + # "close_older" => "100 ms", + # "start_position" => "beginning", + # "delimiter" => TEST_FILE_DELIMITER) + # + # subject.register + # end + # + # it "having timed_out, the codec is auto flushed" do + # actions = RSpec::Sequencing + # .run("create file") do + # File.open(tmpfile_path, "wb") { |file| file.puts(line) } + # end + # .then_after(0.1, "identity is mapped") do + # wait(0.75).for{subject.codec.identity_map[tmpfile_path]}.not_to be_nil, "identity is not mapped" + # end + # .then("wait for auto_flush") do + # wait(0.75).for{subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush)}.to eq([true]), "autoflush didn't" + # end + # .then("quit") do + # subject.stop + # end + # subject.run(events) + # actions.assert_no_errors + # expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq([true]) + # end + # end + # + # context "when ignore_older config is specified" do + # let(:suffix) { "Y" } + # before do + # conf.update( + # "type" => "blah", + # "path" => path_path, + # "sincedb_path" => sincedb_path, + # "stat_interval" => 0.02, + # "codec" => tracer_codec, + # "ignore_older" => "500 ms", + # "delimiter" => TEST_FILE_DELIMITER) + # end + # subject { described_class.new(conf) } + # let(:line) { "line1.1-of-a" } + # + # it "the file is not read" do + # subject.register + # RSpec::Sequencing + # .run("create file") do + # File.open(tmp_dir_file, "a") do |fd| + # fd.puts(line) + # fd.fsync + # end + # FileInput.make_file_older(tmp_dir_file, 2) + # end + # .then_after(0.5, "stop") do + # subject.stop + # end + # subject.run(events) + # expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to be_falsey + # end + # end + # + # context "when wildcard path and a multiline codec is specified" do + # subject { described_class.new(conf) } + # let(:suffix) { "J" } + # let(:tmpfile_path2) { ::File.join(tmpdir_path, "K.txt") } + # before do + # mlconf.update("pattern" => "^\s", "what" => "previous") + # conf.update( + # "type" => "blah", + # "path" => path_path, + # "start_position" => "beginning", + # "sincedb_path" => sincedb_path, + # "stat_interval" => 0.05, + # "codec" => mlcodec, + # "file_sort_by" => "path", + # "delimiter" => TEST_FILE_DELIMITER) + # + # subject.register + # end + # + # it "collects separate multiple line events from each file" do + # subject + # actions = RSpec::Sequencing + # .run_after(0.1, "create files") do + # File.open(tmpfile_path, "wb") do |fd| + # fd.puts("line1.1-of-J") + # fd.puts(" line1.2-of-J") + # fd.puts(" line1.3-of-J") + # end + # File.open(tmpfile_path2, "wb") do |fd| + # fd.puts("line1.1-of-K") + # fd.puts(" line1.2-of-K") + # fd.puts(" line1.3-of-K") + # end + # end + # .then("assert both files are mapped as identities and stop") do + # wait(2).for {subject.codec.identity_count}.to eq(2), "both files are not mapped as identities" + # end + # .then("stop") do + # subject.stop + # end + # subject.run(events) + # # wait for actions to complete + # actions.assert_no_errors + # expect(events.size).to eq(2) + # e1, e2 = events + # e1_message = e1.get("message") + # e2_message = e2.get("message") + # + # expect(e1.get("path")).to match(/J.txt/) + # expect(e2.get("path")).to match(/K.txt/) + # expect(e1_message).to eq("line1.1-of-J#{TEST_FILE_DELIMITER} line1.2-of-J#{TEST_FILE_DELIMITER} line1.3-of-J") + # expect(e2_message).to eq("line1.1-of-K#{TEST_FILE_DELIMITER} line1.2-of-K#{TEST_FILE_DELIMITER} line1.3-of-K") + # end + # + # context "if auto_flush is enabled on the multiline codec" do + # let(:mlconf) { { "auto_flush_interval" => 0.5 } } + # let(:suffix) { "M" } + # it "an event is generated via auto_flush" do + # actions = RSpec::Sequencing + # .run_after(0.1, "create files") do + # File.open(tmpfile_path, "wb") do |fd| + # fd.puts("line1.1-of-a") + # fd.puts(" line1.2-of-a") + # fd.puts(" line1.3-of-a") + # end + # end + # .then("wait for auto_flush") do + # wait(2).for{events.size}.to eq(1), "events size is not 1" + # end + # .then("stop") do + # subject.stop + # end + # subject.run(events) + # # wait for actions to complete + # actions.assert_no_errors + # e1 = events.first + # e1_message = e1.get("message") + # expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER} line1.2-of-a#{TEST_FILE_DELIMITER} line1.3-of-a") + # expect(e1.get("path")).to match(/M.txt$/) + # end + # end + # end + # + # describe "specifying max_open_files" do + # let(:suffix) { "P" } + # let(:tmpfile_path2) { ::File.join(tmpdir_path, "Q.txt") } + # subject { described_class.new(conf) } + # before do + # File.open(tmpfile_path, "w") do |fd| + # fd.puts("line1-of-P") + # fd.puts("line2-of-P") + # fd.fsync + # end + # File.open(tmpfile_path2, "w") do |fd| + # fd.puts("line1-of-Q") + # fd.puts("line2-of-Q") + # fd.fsync + # end + # end + # + # context "when close_older is NOT specified" do + # before do + # conf.clear + # conf.update( + # "type" => "blah", + # "path" => path_path, + # "sincedb_path" => sincedb_path, + # "stat_interval" => 0.1, + # "max_open_files" => 1, + # "start_position" => "beginning", + # "file_sort_by" => "path", + # "delimiter" => TEST_FILE_DELIMITER) + # subject.register + # end + # it "collects line events from only one file" do + # actions = RSpec::Sequencing + # .run("assert one identity is mapped") do + # wait(0.4).for{subject.codec.identity_count}.to be > 0, "no identity is mapped" + # end + # .then("stop") do + # subject.stop + # end + # .then("stop flushes last event") do + # wait(0.4).for{events.size}.to eq(2), "events size does not equal 2" + # end + # subject.run(events) + # # wait for actions future value + # actions.assert_no_errors + # e1, e2 = events + # expect(e1.get("message")).to eq("line1-of-P") + # expect(e2.get("message")).to eq("line2-of-P") + # end + # end + # + # context "when close_older IS specified" do + # before do + # conf.update( + # "type" => "blah", + # "path" => path_path, + # "sincedb_path" => sincedb_path, + # "stat_interval" => 0.1, + # "max_open_files" => 1, + # "close_older" => 0.5, + # "start_position" => "beginning", + # "file_sort_by" => "path", + # "delimiter" => TEST_FILE_DELIMITER) + # subject.register + # end + # + # it "collects line events from both files" do + # actions = RSpec::Sequencing + # .run("assert both identities are mapped and the first two events are built") do + # wait(0.4).for{subject.codec.identity_count == 1 && events.size == 2}.to eq(true), "both identities are not mapped and the first two events are not built" + # end + # .then("wait for close to flush last event of each identity") do + # wait(0.8).for{events.size}.to eq(4), "close does not flush last event of each identity" + # end + # .then_after(0.1, "stop") do + # subject.stop + # end + # subject.run(events) + # # wait for actions future value + # actions.assert_no_errors + # e1, e2, e3, e4 = events + # expect(e1.get("message")).to eq("line1-of-P") + # expect(e2.get("message")).to eq("line2-of-P") + # expect(e3.get("message")).to eq("line1-of-Q") + # expect(e4.get("message")).to eq("line2-of-Q") + # end + # end + # end + # end end From 98b0fbd5f90d8767e3b27af10f190b09fa9b7568 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 17 Feb 2021 12:20:26 -0500 Subject: [PATCH 06/17] nclude loggable --- lib/filewatch/helper.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/filewatch/helper.rb b/lib/filewatch/helper.rb index 6bd32749..66fa8449 100644 --- a/lib/filewatch/helper.rb +++ b/lib/filewatch/helper.rb @@ -3,8 +3,10 @@ # https://raw.githubusercontent.com/rails/rails/v4.2.1/activesupport/lib/active_support/core_ext/file/atomic.rb # change method name to avoid borking active_support and vice versa require 'fileutils' +require "logstash/util/loggable" module FileHelper + include LogStash::Util::Loggable extend self # Write to a file atomically. Useful for situations where you don't # want other processes or threads to see half-written files. From df5a9bf7d52727ab00506cb1196ea97ef74c2333 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 17 Feb 2021 12:33:07 -0500 Subject: [PATCH 07/17] re-add removed test --- spec/inputs/file_tail_spec.rb | 624 +++++++++++++++++----------------- 1 file changed, 312 insertions(+), 312 deletions(-) diff --git a/spec/inputs/file_tail_spec.rb b/spec/inputs/file_tail_spec.rb index c064891c..7aefc69c 100644 --- a/spec/inputs/file_tail_spec.rb +++ b/spec/inputs/file_tail_spec.rb @@ -196,316 +196,316 @@ end - # describe "testing with new, register, run and stop" do - # let(:suffix) { "A" } - # let(:conf) { Hash.new } - # let(:mlconf) { Hash.new } - # let(:events) { Array.new } - # let(:mlcodec) { LogStash::Codecs::Multiline.new(mlconf) } - # let(:tracer_codec) { FileInput::CodecTracer.new } - # let(:tmpdir_path) { Stud::Temporary.directory } - # let(:tmpfile_path) { ::File.join(tmpdir_path, "#{suffix}.txt") } - # let(:path_path) { ::File.join(tmpdir_path, "*.txt") } - # let(:sincedb_path) { ::File.join(tmpdir_path, "sincedb-#{suffix}") } - # - # after :each do - # sleep(0.1) until subject.completely_stopped? - # FileUtils.rm_rf(sincedb_path) - # end - # - # context "when data exists and then more data is appended" do - # subject { described_class.new(conf) } - # - # before do - # File.open(tmpfile_path, "w") do |fd| - # fd.puts("ignore me 1") - # fd.puts("ignore me 2") - # fd.fsync - # end - # mlconf.update("pattern" => "^\s", "what" => "previous") - # conf.update("type" => "blah", - # "path" => path_path, - # "sincedb_path" => sincedb_path, - # "stat_interval" => 0.1, - # "codec" => mlcodec, - # "delimiter" => TEST_FILE_DELIMITER) - # end - # - # it "reads the appended data only" do - # subject.register - # actions = RSpec::Sequencing - # .run_after(1, "append two lines after delay") do - # File.open(tmpfile_path, "a") { |fd| fd.puts("hello"); fd.puts("world") } - # end - # .then("wait for one event") do - # wait(0.75).for{events.size}.to eq(1) - # end - # .then("quit") do - # subject.stop - # end - # .then("wait for flushed event") do - # wait(0.75).for{events.size}.to eq(2) - # end - # - # subject.run(events) - # actions.assert_no_errors - # - # event1 = events[0] - # expect(event1).not_to be_nil - # expect(event1.get("path")).to eq tmpfile_path - # expect(event1.get("[@metadata][path]")).to eq tmpfile_path - # expect(event1.get("message")).to eq "hello" - # - # event2 = events[1] - # expect(event2).not_to be_nil - # expect(event2.get("path")).to eq tmpfile_path - # expect(event2.get("[@metadata][path]")).to eq tmpfile_path - # expect(event2.get("message")).to eq "world" - # end - # end - # - # context "when close_older config is specified" do - # let(:line) { "line1.1-of-a" } - # let(:suffix) { "X" } - # subject { described_class.new(conf) } - # - # before do - # conf.update( - # "type" => "blah", - # "path" => path_path, - # "sincedb_path" => sincedb_path, - # "stat_interval" => 0.02, - # "codec" => tracer_codec, - # "close_older" => "100 ms", - # "start_position" => "beginning", - # "delimiter" => TEST_FILE_DELIMITER) - # - # subject.register - # end - # - # it "having timed_out, the codec is auto flushed" do - # actions = RSpec::Sequencing - # .run("create file") do - # File.open(tmpfile_path, "wb") { |file| file.puts(line) } - # end - # .then_after(0.1, "identity is mapped") do - # wait(0.75).for{subject.codec.identity_map[tmpfile_path]}.not_to be_nil, "identity is not mapped" - # end - # .then("wait for auto_flush") do - # wait(0.75).for{subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush)}.to eq([true]), "autoflush didn't" - # end - # .then("quit") do - # subject.stop - # end - # subject.run(events) - # actions.assert_no_errors - # expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq([true]) - # end - # end - # - # context "when ignore_older config is specified" do - # let(:suffix) { "Y" } - # before do - # conf.update( - # "type" => "blah", - # "path" => path_path, - # "sincedb_path" => sincedb_path, - # "stat_interval" => 0.02, - # "codec" => tracer_codec, - # "ignore_older" => "500 ms", - # "delimiter" => TEST_FILE_DELIMITER) - # end - # subject { described_class.new(conf) } - # let(:line) { "line1.1-of-a" } - # - # it "the file is not read" do - # subject.register - # RSpec::Sequencing - # .run("create file") do - # File.open(tmp_dir_file, "a") do |fd| - # fd.puts(line) - # fd.fsync - # end - # FileInput.make_file_older(tmp_dir_file, 2) - # end - # .then_after(0.5, "stop") do - # subject.stop - # end - # subject.run(events) - # expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to be_falsey - # end - # end - # - # context "when wildcard path and a multiline codec is specified" do - # subject { described_class.new(conf) } - # let(:suffix) { "J" } - # let(:tmpfile_path2) { ::File.join(tmpdir_path, "K.txt") } - # before do - # mlconf.update("pattern" => "^\s", "what" => "previous") - # conf.update( - # "type" => "blah", - # "path" => path_path, - # "start_position" => "beginning", - # "sincedb_path" => sincedb_path, - # "stat_interval" => 0.05, - # "codec" => mlcodec, - # "file_sort_by" => "path", - # "delimiter" => TEST_FILE_DELIMITER) - # - # subject.register - # end - # - # it "collects separate multiple line events from each file" do - # subject - # actions = RSpec::Sequencing - # .run_after(0.1, "create files") do - # File.open(tmpfile_path, "wb") do |fd| - # fd.puts("line1.1-of-J") - # fd.puts(" line1.2-of-J") - # fd.puts(" line1.3-of-J") - # end - # File.open(tmpfile_path2, "wb") do |fd| - # fd.puts("line1.1-of-K") - # fd.puts(" line1.2-of-K") - # fd.puts(" line1.3-of-K") - # end - # end - # .then("assert both files are mapped as identities and stop") do - # wait(2).for {subject.codec.identity_count}.to eq(2), "both files are not mapped as identities" - # end - # .then("stop") do - # subject.stop - # end - # subject.run(events) - # # wait for actions to complete - # actions.assert_no_errors - # expect(events.size).to eq(2) - # e1, e2 = events - # e1_message = e1.get("message") - # e2_message = e2.get("message") - # - # expect(e1.get("path")).to match(/J.txt/) - # expect(e2.get("path")).to match(/K.txt/) - # expect(e1_message).to eq("line1.1-of-J#{TEST_FILE_DELIMITER} line1.2-of-J#{TEST_FILE_DELIMITER} line1.3-of-J") - # expect(e2_message).to eq("line1.1-of-K#{TEST_FILE_DELIMITER} line1.2-of-K#{TEST_FILE_DELIMITER} line1.3-of-K") - # end - # - # context "if auto_flush is enabled on the multiline codec" do - # let(:mlconf) { { "auto_flush_interval" => 0.5 } } - # let(:suffix) { "M" } - # it "an event is generated via auto_flush" do - # actions = RSpec::Sequencing - # .run_after(0.1, "create files") do - # File.open(tmpfile_path, "wb") do |fd| - # fd.puts("line1.1-of-a") - # fd.puts(" line1.2-of-a") - # fd.puts(" line1.3-of-a") - # end - # end - # .then("wait for auto_flush") do - # wait(2).for{events.size}.to eq(1), "events size is not 1" - # end - # .then("stop") do - # subject.stop - # end - # subject.run(events) - # # wait for actions to complete - # actions.assert_no_errors - # e1 = events.first - # e1_message = e1.get("message") - # expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER} line1.2-of-a#{TEST_FILE_DELIMITER} line1.3-of-a") - # expect(e1.get("path")).to match(/M.txt$/) - # end - # end - # end - # - # describe "specifying max_open_files" do - # let(:suffix) { "P" } - # let(:tmpfile_path2) { ::File.join(tmpdir_path, "Q.txt") } - # subject { described_class.new(conf) } - # before do - # File.open(tmpfile_path, "w") do |fd| - # fd.puts("line1-of-P") - # fd.puts("line2-of-P") - # fd.fsync - # end - # File.open(tmpfile_path2, "w") do |fd| - # fd.puts("line1-of-Q") - # fd.puts("line2-of-Q") - # fd.fsync - # end - # end - # - # context "when close_older is NOT specified" do - # before do - # conf.clear - # conf.update( - # "type" => "blah", - # "path" => path_path, - # "sincedb_path" => sincedb_path, - # "stat_interval" => 0.1, - # "max_open_files" => 1, - # "start_position" => "beginning", - # "file_sort_by" => "path", - # "delimiter" => TEST_FILE_DELIMITER) - # subject.register - # end - # it "collects line events from only one file" do - # actions = RSpec::Sequencing - # .run("assert one identity is mapped") do - # wait(0.4).for{subject.codec.identity_count}.to be > 0, "no identity is mapped" - # end - # .then("stop") do - # subject.stop - # end - # .then("stop flushes last event") do - # wait(0.4).for{events.size}.to eq(2), "events size does not equal 2" - # end - # subject.run(events) - # # wait for actions future value - # actions.assert_no_errors - # e1, e2 = events - # expect(e1.get("message")).to eq("line1-of-P") - # expect(e2.get("message")).to eq("line2-of-P") - # end - # end - # - # context "when close_older IS specified" do - # before do - # conf.update( - # "type" => "blah", - # "path" => path_path, - # "sincedb_path" => sincedb_path, - # "stat_interval" => 0.1, - # "max_open_files" => 1, - # "close_older" => 0.5, - # "start_position" => "beginning", - # "file_sort_by" => "path", - # "delimiter" => TEST_FILE_DELIMITER) - # subject.register - # end - # - # it "collects line events from both files" do - # actions = RSpec::Sequencing - # .run("assert both identities are mapped and the first two events are built") do - # wait(0.4).for{subject.codec.identity_count == 1 && events.size == 2}.to eq(true), "both identities are not mapped and the first two events are not built" - # end - # .then("wait for close to flush last event of each identity") do - # wait(0.8).for{events.size}.to eq(4), "close does not flush last event of each identity" - # end - # .then_after(0.1, "stop") do - # subject.stop - # end - # subject.run(events) - # # wait for actions future value - # actions.assert_no_errors - # e1, e2, e3, e4 = events - # expect(e1.get("message")).to eq("line1-of-P") - # expect(e2.get("message")).to eq("line2-of-P") - # expect(e3.get("message")).to eq("line1-of-Q") - # expect(e4.get("message")).to eq("line2-of-Q") - # end - # end - # end - # end + describe "testing with new, register, run and stop" do + let(:suffix) { "A" } + let(:conf) { Hash.new } + let(:mlconf) { Hash.new } + let(:events) { Array.new } + let(:mlcodec) { LogStash::Codecs::Multiline.new(mlconf) } + let(:tracer_codec) { FileInput::CodecTracer.new } + let(:tmpdir_path) { Stud::Temporary.directory } + let(:tmpfile_path) { ::File.join(tmpdir_path, "#{suffix}.txt") } + let(:path_path) { ::File.join(tmpdir_path, "*.txt") } + let(:sincedb_path) { ::File.join(tmpdir_path, "sincedb-#{suffix}") } + + after :each do + sleep(0.1) until subject.completely_stopped? + FileUtils.rm_rf(sincedb_path) + end + + context "when data exists and then more data is appended" do + subject { described_class.new(conf) } + + before do + File.open(tmpfile_path, "w") do |fd| + fd.puts("ignore me 1") + fd.puts("ignore me 2") + fd.fsync + end + mlconf.update("pattern" => "^\s", "what" => "previous") + conf.update("type" => "blah", + "path" => path_path, + "sincedb_path" => sincedb_path, + "stat_interval" => 0.1, + "codec" => mlcodec, + "delimiter" => TEST_FILE_DELIMITER) + end + + it "reads the appended data only" do + subject.register + actions = RSpec::Sequencing + .run_after(1, "append two lines after delay") do + File.open(tmpfile_path, "a") { |fd| fd.puts("hello"); fd.puts("world") } + end + .then("wait for one event") do + wait(0.75).for{events.size}.to eq(1) + end + .then("quit") do + subject.stop + end + .then("wait for flushed event") do + wait(0.75).for{events.size}.to eq(2) + end + + subject.run(events) + actions.assert_no_errors + + event1 = events[0] + expect(event1).not_to be_nil + expect(event1.get("path")).to eq tmpfile_path + expect(event1.get("[@metadata][path]")).to eq tmpfile_path + expect(event1.get("message")).to eq "hello" + + event2 = events[1] + expect(event2).not_to be_nil + expect(event2.get("path")).to eq tmpfile_path + expect(event2.get("[@metadata][path]")).to eq tmpfile_path + expect(event2.get("message")).to eq "world" + end + end + + context "when close_older config is specified" do + let(:line) { "line1.1-of-a" } + let(:suffix) { "X" } + subject { described_class.new(conf) } + + before do + conf.update( + "type" => "blah", + "path" => path_path, + "sincedb_path" => sincedb_path, + "stat_interval" => 0.02, + "codec" => tracer_codec, + "close_older" => "100 ms", + "start_position" => "beginning", + "delimiter" => TEST_FILE_DELIMITER) + + subject.register + end + + it "having timed_out, the codec is auto flushed" do + actions = RSpec::Sequencing + .run("create file") do + File.open(tmpfile_path, "wb") { |file| file.puts(line) } + end + .then_after(0.1, "identity is mapped") do + wait(0.75).for{subject.codec.identity_map[tmpfile_path]}.not_to be_nil, "identity is not mapped" + end + .then("wait for auto_flush") do + wait(0.75).for{subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush)}.to eq([true]), "autoflush didn't" + end + .then("quit") do + subject.stop + end + subject.run(events) + actions.assert_no_errors + expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq([true]) + end + end + + context "when ignore_older config is specified" do + let(:suffix) { "Y" } + before do + conf.update( + "type" => "blah", + "path" => path_path, + "sincedb_path" => sincedb_path, + "stat_interval" => 0.02, + "codec" => tracer_codec, + "ignore_older" => "500 ms", + "delimiter" => TEST_FILE_DELIMITER) + end + subject { described_class.new(conf) } + let(:line) { "line1.1-of-a" } + + it "the file is not read" do + subject.register + RSpec::Sequencing + .run("create file") do + File.open(tmp_dir_file, "a") do |fd| + fd.puts(line) + fd.fsync + end + FileInput.make_file_older(tmp_dir_file, 2) + end + .then_after(0.5, "stop") do + subject.stop + end + subject.run(events) + expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to be_falsey + end + end + + context "when wildcard path and a multiline codec is specified" do + subject { described_class.new(conf) } + let(:suffix) { "J" } + let(:tmpfile_path2) { ::File.join(tmpdir_path, "K.txt") } + before do + mlconf.update("pattern" => "^\s", "what" => "previous") + conf.update( + "type" => "blah", + "path" => path_path, + "start_position" => "beginning", + "sincedb_path" => sincedb_path, + "stat_interval" => 0.05, + "codec" => mlcodec, + "file_sort_by" => "path", + "delimiter" => TEST_FILE_DELIMITER) + + subject.register + end + + it "collects separate multiple line events from each file" do + subject + actions = RSpec::Sequencing + .run_after(0.1, "create files") do + File.open(tmpfile_path, "wb") do |fd| + fd.puts("line1.1-of-J") + fd.puts(" line1.2-of-J") + fd.puts(" line1.3-of-J") + end + File.open(tmpfile_path2, "wb") do |fd| + fd.puts("line1.1-of-K") + fd.puts(" line1.2-of-K") + fd.puts(" line1.3-of-K") + end + end + .then("assert both files are mapped as identities and stop") do + wait(2).for {subject.codec.identity_count}.to eq(2), "both files are not mapped as identities" + end + .then("stop") do + subject.stop + end + subject.run(events) + # wait for actions to complete + actions.assert_no_errors + expect(events.size).to eq(2) + e1, e2 = events + e1_message = e1.get("message") + e2_message = e2.get("message") + + expect(e1.get("path")).to match(/J.txt/) + expect(e2.get("path")).to match(/K.txt/) + expect(e1_message).to eq("line1.1-of-J#{TEST_FILE_DELIMITER} line1.2-of-J#{TEST_FILE_DELIMITER} line1.3-of-J") + expect(e2_message).to eq("line1.1-of-K#{TEST_FILE_DELIMITER} line1.2-of-K#{TEST_FILE_DELIMITER} line1.3-of-K") + end + + context "if auto_flush is enabled on the multiline codec" do + let(:mlconf) { { "auto_flush_interval" => 0.5 } } + let(:suffix) { "M" } + it "an event is generated via auto_flush" do + actions = RSpec::Sequencing + .run_after(0.1, "create files") do + File.open(tmpfile_path, "wb") do |fd| + fd.puts("line1.1-of-a") + fd.puts(" line1.2-of-a") + fd.puts(" line1.3-of-a") + end + end + .then("wait for auto_flush") do + wait(2).for{events.size}.to eq(1), "events size is not 1" + end + .then("stop") do + subject.stop + end + subject.run(events) + # wait for actions to complete + actions.assert_no_errors + e1 = events.first + e1_message = e1.get("message") + expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER} line1.2-of-a#{TEST_FILE_DELIMITER} line1.3-of-a") + expect(e1.get("path")).to match(/M.txt$/) + end + end + end + + describe "specifying max_open_files" do + let(:suffix) { "P" } + let(:tmpfile_path2) { ::File.join(tmpdir_path, "Q.txt") } + subject { described_class.new(conf) } + before do + File.open(tmpfile_path, "w") do |fd| + fd.puts("line1-of-P") + fd.puts("line2-of-P") + fd.fsync + end + File.open(tmpfile_path2, "w") do |fd| + fd.puts("line1-of-Q") + fd.puts("line2-of-Q") + fd.fsync + end + end + + context "when close_older is NOT specified" do + before do + conf.clear + conf.update( + "type" => "blah", + "path" => path_path, + "sincedb_path" => sincedb_path, + "stat_interval" => 0.1, + "max_open_files" => 1, + "start_position" => "beginning", + "file_sort_by" => "path", + "delimiter" => TEST_FILE_DELIMITER) + subject.register + end + it "collects line events from only one file" do + actions = RSpec::Sequencing + .run("assert one identity is mapped") do + wait(0.4).for{subject.codec.identity_count}.to be > 0, "no identity is mapped" + end + .then("stop") do + subject.stop + end + .then("stop flushes last event") do + wait(0.4).for{events.size}.to eq(2), "events size does not equal 2" + end + subject.run(events) + # wait for actions future value + actions.assert_no_errors + e1, e2 = events + expect(e1.get("message")).to eq("line1-of-P") + expect(e2.get("message")).to eq("line2-of-P") + end + end + + context "when close_older IS specified" do + before do + conf.update( + "type" => "blah", + "path" => path_path, + "sincedb_path" => sincedb_path, + "stat_interval" => 0.1, + "max_open_files" => 1, + "close_older" => 0.5, + "start_position" => "beginning", + "file_sort_by" => "path", + "delimiter" => TEST_FILE_DELIMITER) + subject.register + end + + it "collects line events from both files" do + actions = RSpec::Sequencing + .run("assert both identities are mapped and the first two events are built") do + wait(0.4).for{subject.codec.identity_count == 1 && events.size == 2}.to eq(true), "both identities are not mapped and the first two events are not built" + end + .then("wait for close to flush last event of each identity") do + wait(0.8).for{events.size}.to eq(4), "close does not flush last event of each identity" + end + .then_after(0.1, "stop") do + subject.stop + end + subject.run(events) + # wait for actions future value + actions.assert_no_errors + e1, e2, e3, e4 = events + expect(e1.get("message")).to eq("line1-of-P") + expect(e2.get("message")).to eq("line2-of-P") + expect(e3.get("message")).to eq("line1-of-Q") + expect(e4.get("message")).to eq("line2-of-Q") + end + end + end + end end From d7d9dc63a091580012e8d8daa3bbf821b4b76370 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 17 Feb 2021 12:41:31 -0500 Subject: [PATCH 08/17] Set up appropriate env --- .travis.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6b4e95d1..96f9cc57 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,4 +6,10 @@ before_install: - sudo apt-get install jq - sudo apt-get install libseccomp-dev arch: - - arm64 \ No newline at end of file + - arm64 + +env: + jobs: + - ELASTIC_STACK_VERSION=7.x + - SNAPSHOT=true ELASTIC_STACK_VERSION=8.0.0-SNAPSHOT + - SNAPSHOT=true ELASTIC_STACK_VERSION=7.11.0-SNAPSHOT \ No newline at end of file From c00c00c12c891abddbf78ca58c3febc9b37a9345 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 17 Feb 2021 13:01:59 -0500 Subject: [PATCH 09/17] multi arch builds --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 96f9cc57..d518cba7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,10 @@ before_install: - sudo apt-get install libseccomp-dev arch: - arm64 + - amd64 env: jobs: - ELASTIC_STACK_VERSION=7.x - SNAPSHOT=true ELASTIC_STACK_VERSION=8.0.0-SNAPSHOT - - SNAPSHOT=true ELASTIC_STACK_VERSION=7.11.0-SNAPSHOT \ No newline at end of file + - SNAPSHOT=true ELASTIC_STACK_VERSION=7.11.0-SNAPSHOT From e277ee045272f0a2adb6b1fb3a1b33af8d834c77 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 17 Feb 2021 13:20:59 -0500 Subject: [PATCH 10/17] dbg --- spec/inputs/file_tail_spec.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spec/inputs/file_tail_spec.rb b/spec/inputs/file_tail_spec.rb index 7aefc69c..1b329c0c 100644 --- a/spec/inputs/file_tail_spec.rb +++ b/spec/inputs/file_tail_spec.rb @@ -201,7 +201,10 @@ let(:conf) { Hash.new } let(:mlconf) { Hash.new } let(:events) { Array.new } - let(:mlcodec) { LogStash::Codecs::Multiline.new(mlconf) } + let(:mlcodec) { + puts " The value of mlconf is #{mlconf}" + LogStash::Codecs::Multiline.new(mlconf) + } let(:tracer_codec) { FileInput::CodecTracer.new } let(:tmpdir_path) { Stud::Temporary.directory } let(:tmpfile_path) { ::File.join(tmpdir_path, "#{suffix}.txt") } From 915b14823fcf5e6337f3c552c27a1a6a3703d921 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 17 Feb 2021 14:17:15 -0500 Subject: [PATCH 11/17] tst --- spec/inputs/file_tail_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/inputs/file_tail_spec.rb b/spec/inputs/file_tail_spec.rb index 1b329c0c..4af7cd28 100644 --- a/spec/inputs/file_tail_spec.rb +++ b/spec/inputs/file_tail_spec.rb @@ -225,7 +225,7 @@ fd.puts("ignore me 2") fd.fsync end - mlconf.update("pattern" => "^\s", "what" => "previous") + mlconf.update("pattern" => "^\\s", "what" => "previous") conf.update("type" => "blah", "path" => path_path, "sincedb_path" => sincedb_path, @@ -344,7 +344,7 @@ let(:suffix) { "J" } let(:tmpfile_path2) { ::File.join(tmpdir_path, "K.txt") } before do - mlconf.update("pattern" => "^\s", "what" => "previous") + mlconf.update("pattern" => "^\\s", "what" => "previous") conf.update( "type" => "blah", "path" => path_path, From bd4e3c943683d43a20260fa710c609b82c0dc228 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 17 Feb 2021 15:04:20 -0500 Subject: [PATCH 12/17] Switch to virt: vm --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d518cba7..7c2a7e0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,13 +2,14 @@ import: - logstash-plugins/.ci:travis/travis.yml@1.x before_install: - - mkdir -p .ci && curl -sL https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards "*Dockerfile*" "*docker*" "*.sh" - sudo apt-get install jq - sudo apt-get install libseccomp-dev arch: - arm64 - amd64 +virt: vm + env: jobs: - ELASTIC_STACK_VERSION=7.x From 58089d489bc11f43657bdb4f3bc34c7c1e125bec Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 17 Feb 2021 15:24:01 -0500 Subject: [PATCH 13/17] tst --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7c2a7e0a..9d01d1c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ import: before_install: - sudo apt-get install jq - sudo apt-get install libseccomp-dev + - sudo locale-gen en_US.UTF-8 + - sudo dpkg-reconfigure locales arch: - arm64 - amd64 From 474f7df23f192c52092853fe654000ab3e5cf9ba Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 17 Feb 2021 16:24:34 -0500 Subject: [PATCH 14/17] Update dockerfile with locale --- .ci/Dockerfile | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .ci/Dockerfile diff --git a/.ci/Dockerfile b/.ci/Dockerfile new file mode 100644 index 00000000..1f8527fc --- /dev/null +++ b/.ci/Dockerfile @@ -0,0 +1,28 @@ +ARG ELASTIC_STACK_VERSION +ARG DISTRIBUTION_SUFFIX +FROM docker.elastic.co/logstash/logstash${DISTRIBUTION_SUFFIX}:${ELASTIC_STACK_VERSION} +RUN sudo localedef -i en_US -f UTF-8 en_US.UTF-8 +RUN locale +RUN locale -a +RUN localectl set-locale LANG=en_US.UTF-8 +USER logstash + +COPY --chown=logstash:logstash Gemfile /usr/share/plugins/plugin/Gemfile +COPY --chown=logstash:logstash *.gemspec VERSION* version* /usr/share/plugins/plugin/ +RUN cp /usr/share/logstash/logstash-core/versions-gem-copy.yml /usr/share/logstash/versions.yml +# NOTE: since 8.0 JDK is bundled as part of the LS distribution under $LS_HOME/jdk +ENV PATH="${PATH}:/usr/share/logstash/vendor/jruby/bin:/usr/share/logstash/jdk/bin" +ENV LOGSTASH_SOURCE="1" +ENV ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION +# DISTRIBUTION="default" (by default) or "oss" +ARG DISTRIBUTION +ENV DISTRIBUTION=$DISTRIBUTION +# INTEGRATION="true" while integration testing (false-y by default) +ARG INTEGRATION +ENV INTEGRATION=$INTEGRATION +RUN gem install bundler -v '< 2' +WORKDIR /usr/share/plugins/plugin +RUN bundle install --with test ci +COPY --chown=logstash:logstash . /usr/share/plugins/plugin +RUN bundle exec rake vendor +RUN .ci/setup.sh From e7f9fa8a38d394a965f7660ef47eb0203c313e29 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Mon, 15 Mar 2021 15:05:26 -0400 Subject: [PATCH 15/17] nxt --- .ci/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.ci/Dockerfile b/.ci/Dockerfile index 1f8527fc..9ccbde9f 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -1,10 +1,9 @@ ARG ELASTIC_STACK_VERSION ARG DISTRIBUTION_SUFFIX FROM docker.elastic.co/logstash/logstash${DISTRIBUTION_SUFFIX}:${ELASTIC_STACK_VERSION} -RUN sudo localedef -i en_US -f UTF-8 en_US.UTF-8 -RUN locale +USER root +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 RUN locale -a -RUN localectl set-locale LANG=en_US.UTF-8 USER logstash COPY --chown=logstash:logstash Gemfile /usr/share/plugins/plugin/Gemfile From 39999f4f70e8d28737131a8bd63610c00cfe7b07 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Mon, 15 Mar 2021 16:27:47 -0400 Subject: [PATCH 16/17] Add more retries to sincedb clean effort --- spec/inputs/file_read_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/inputs/file_read_spec.rb b/spec/inputs/file_read_spec.rb index d1533bd0..7dd17b33 100644 --- a/spec/inputs/file_read_spec.rb +++ b/spec/inputs/file_read_spec.rb @@ -338,7 +338,7 @@ sincedb_content = File.read(sincedb_path).strip expect( sincedb_content ).to_not be_empty - Stud.try(3.times) do + Stud.try(10.times) do sleep(1.5) # > sincedb_clean_after sincedb_content = File.read(sincedb_path).strip From 618f42b4d306e392e1b461fce4d8eca850d5f5c4 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Tue, 16 Mar 2021 11:06:19 -0400 Subject: [PATCH 17/17] dbg --- spec/inputs/file_read_spec.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/inputs/file_read_spec.rb b/spec/inputs/file_read_spec.rb index 7dd17b33..5c21030a 100644 --- a/spec/inputs/file_read_spec.rb +++ b/spec/inputs/file_read_spec.rb @@ -338,10 +338,11 @@ sincedb_content = File.read(sincedb_path).strip expect( sincedb_content ).to_not be_empty - Stud.try(10.times) do + Stud.try(10.times, RSpec::Expectations::ExpectationNotMetError) do sleep(1.5) # > sincedb_clean_after sincedb_content = File.read(sincedb_path).strip + puts "sincedb content is #{sincedb_content}" expect( sincedb_content ).to be_empty end end