Skip to content

Commit

Permalink
Updates for JRuby 9.4 (#125)
Browse files Browse the repository at this point in the history
This PR laid down adaptations to run this plugin under JRuby 9.4 (Ruby 3.1)

List of changes:

- in a method with var args the hash map has to be defined when passed as argument, d19fc58 Ruby 3.0
- Ruby 3.1 comes with Psych 4 which switched the alias of load from unsafe_load to safe_load reference. Created a wrapper method to behave consistently across different versions of JRuby.
- fixed a stubbing error in tests: 72a2af5
- covered ValueTracker with unit tests
  • Loading branch information
andsel authored Jun 15, 2023
1 parent 69786f4 commit c06e319
Show file tree
Hide file tree
Showing 6 changed files with 144 additions and 15 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
## 5.4.4
- Fix: adaptations for JRuby 9.4 [#125](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/125)

## 5.4.3
- Fix crash when metadata file can't be deleted after moving under path.data [#136](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/136)

Expand Down
15 changes: 14 additions & 1 deletion lib/logstash/plugin_mixins/jdbc/value_tracking.rb
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# encoding: utf-8
require "yaml" # persistence
require "date"
require "bigdecimal"

module LogStash module PluginMixins module Jdbc
class ValueTracking
Expand Down Expand Up @@ -31,6 +33,17 @@ def initialize(handler)
set_initial
end

if Psych::VERSION&.split('.')&.first.to_i >= 4
YAML_PERMITTED_CLASSES = [::DateTime, ::Time, ::BigDecimal].freeze
def self.load_yaml(source)
Psych::safe_load(source, permitted_classes: YAML_PERMITTED_CLASSES)
end
else
def self.load_yaml(source)
YAML::load(source)
end
end

def set_initial
# override in subclass
end
Expand Down Expand Up @@ -112,7 +125,7 @@ def clean

def read
return unless @exists
YAML.load(::File.read(@path))
ValueTracking.load_yaml(::File.read(@path))
end

def write(value)
Expand Down
2 changes: 1 addition & 1 deletion logstash-integration-jdbc.gemspec
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
Gem::Specification.new do |s|
s.name = 'logstash-integration-jdbc'
s.version = '5.4.3'
s.version = '5.4.4'
s.licenses = ['Apache License (2.0)']
s.summary = "Integration with JDBC - input and filter plugins"
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
Expand Down
14 changes: 7 additions & 7 deletions spec/filters/jdbc_streaming_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,7 @@ class TestJdbcStreaming < JdbcStreaming
CONFIG
end

sample("message" => "some text") do
sample({"message" => "some text"}) do
expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
end
end
Expand All @@ -277,7 +277,7 @@ class TestJdbcStreaming < JdbcStreaming
CONFIG
end

sample("message" => "some text") do
sample({"message" => "some text"}) do
expect(subject.get('new_field')).to eq([{"col_1" => 'from_database'}])
end
end
Expand All @@ -296,11 +296,11 @@ class TestJdbcStreaming < JdbcStreaming
CONFIG
end

sample("message" => "some text", "param_field" => "1") do
sample({"message" => "some text", "param_field" => "1"}) do
expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
end

sample("message" => "some text", "param_field" => "2") do
sample({"message" => "some text", "param_field" => "2"}) do
expect(subject.get('new_field').nil?)
end
end
Expand All @@ -319,11 +319,11 @@ class TestJdbcStreaming < JdbcStreaming
CONFIG
end

sample("message" => "some text", "param_field" => 1) do
sample({"message" => "some text", "param_field" => 1}) do
expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
end

sample("message" => "some text", "param_field" => "1") do
sample({"message" => "some text", "param_field" => "1"}) do
expect(subject.get('new_field').nil?)
end
end
Expand All @@ -342,7 +342,7 @@ class TestJdbcStreaming < JdbcStreaming
CONFIG
end

sample("message" => "some text") do
sample({"message" => "some text"}) do
expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
end
end
Expand Down
12 changes: 6 additions & 6 deletions spec/inputs/jdbc_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@
sleep 1
for i in 0..1
sleep 1
updated_last_run = YAML.load(File.read(settings["last_run_metadata_path"]))
updated_last_run = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))
expect(updated_last_run).to be > last_run_time
last_run_time = updated_last_run
end
Expand Down Expand Up @@ -547,7 +547,7 @@
expect(actual).to eq(expected)
plugin.stop
raw_last_run_value = File.read(settings["last_run_metadata_path"])
last_run_value = YAML.load(raw_last_run_value)
last_run_value = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(raw_last_run_value)
expect(last_run_value).to be_a(DateTime)
expect(last_run_value.strftime("%F %T.%N %Z")).to eq("2015-01-02 02:00:00.722000000 +00:00")

Expand All @@ -562,7 +562,7 @@
plugin.stop
expect(event.get("num")).to eq(12)
expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-02T03:00:00.811Z"))
last_run_value = YAML.load(File.read(settings["last_run_metadata_path"]))
last_run_value = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))
expect(last_run_value).to be_a(DateTime)
# verify that sub-seconds are recorded to the file
expect(last_run_value.strftime("%F %T.%N %Z")).to eq("2015-01-02 03:00:00.811000000 +00:00")
Expand Down Expand Up @@ -1169,7 +1169,7 @@
context "when a file exists" do
before do
# in a faked HOME folder save a valid previous last_run metadata file
allow(ENV).to receive(:[]).and_call_original
allow(ENV).to receive(:[]).with(anything).and_call_original
allow(ENV).to receive(:[]).with('HOME').and_return(fake_home)

File.open("#{fake_home}/.logstash_jdbc_last_run", 'w') do |file|
Expand Down Expand Up @@ -1722,7 +1722,7 @@
plugin.run(queue)

expect(queue.size).to eq(expected_queue_size)
expect(YAML.load(File.read(settings["last_run_metadata_path"]))).to eq(expected_queue_size)
expect(LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))).to eq(expected_queue_size)
end
end

Expand All @@ -1747,7 +1747,7 @@
plugin.run(queue)

expect(queue.size).to eq(expected_queue_size)
expect(YAML.load(File.read(settings["last_run_metadata_path"]))).to eq(last_run_value + expected_queue_size)
expect(LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))).to eq(last_run_value + expected_queue_size)
end
end
end
Expand Down
113 changes: 113 additions & 0 deletions spec/plugin_mixins/jdbc/value_tracking_spec.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
# encoding: utf-8
require "logstash/plugin_mixins/jdbc/value_tracking"
require "tempfile"

module LogStash module PluginMixins module Jdbc
describe ValueTracking do
context "#load_yaml" do

context "with date string" do
let(:yaml_date_source) { "--- !ruby/object:DateTime '2023-06-15 09:59:30.558000000 +02:00'\n" }

it "should be loaded" do
parsed_date = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(yaml_date_source)
expect(parsed_date.class).to eq DateTime
expect(parsed_date.year).to eq 2023
expect(parsed_date.month).to eq 6
expect(parsed_date.day).to eq 15
end
end

context "with time string" do
let(:yaml_time_source) { "--- 2023-06-15 15:28:15.227874000 +02:00\n" }

it "should be loaded" do
parsed_time = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(yaml_time_source)
expect(parsed_time.class).to eq Time
expect(parsed_time.year).to eq 2023
expect(parsed_time.month).to eq 6
expect(parsed_time.day).to eq 15
expect(parsed_time.hour).to eq 15
expect(parsed_time.min).to eq 28
expect(parsed_time.sec).to eq 15
end
end

context "with date string" do
let(:yaml_bigdecimal_source) { "--- !ruby/object:BigDecimal '0:0.1e1'\n" }

it "should be loaded" do
parsed_bigdecimal = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(yaml_bigdecimal_source)
expect(parsed_bigdecimal.class).to eq BigDecimal
expect(parsed_bigdecimal.to_i).to eq 1
end
end
end

context "#build_last_value_tracker" do

let(:plugin) { double("fake plugin") }
let(:temp_file) { Tempfile.new('last_run_tracker') }

before(:each) do
allow(plugin).to receive(:record_last_run).and_return(true)
allow(plugin).to receive(:clean_run).and_return(false)
allow(plugin).to receive(:last_run_metadata_file_path).and_return(temp_file.path)
end

context "create numerical tracker" do
before(:each) do
allow(plugin).to receive(:use_column_value).and_return(true)
allow(plugin).to receive(:tracking_column_type).and_return("numeric")
end

it "should write correctly" do
tracker = ValueTracking.build_last_value_tracker(plugin)
tracker.set_value(1)
tracker.write

temp_file.rewind
v = ValueTracking.load_yaml(::File.read(temp_file.path))
expect(v).to eq 1
end
end

context "create date time tracker" do
before(:each) do
allow(plugin).to receive(:use_column_value).and_return(false)
allow(plugin).to receive(:jdbc_default_timezone).and_return(:something_not_nil)
end

it "should write correctly" do
tracker = ValueTracking.build_last_value_tracker(plugin)
tracker.set_value("2023-06-15T15:28:15+02:00")
tracker.write

temp_file.rewind
v = ValueTracking.load_yaml(::File.read(temp_file.path))
expect(v.class).to eq DateTime
expect(v.year).to eq 2023
end
end

context "create time tracker" do
before(:each) do
allow(plugin).to receive(:use_column_value).and_return(false)
allow(plugin).to receive(:jdbc_default_timezone).and_return(nil)
end

it "should write correctly" do
tracker = ValueTracking.build_last_value_tracker(plugin)
tracker.set_value("2023-06-15T15:28:15+02:00")
tracker.write

temp_file.rewind
v = ValueTracking.load_yaml(::File.read(temp_file.path))
expect(v.class).to eq Time
expect(v.min).to eq 28
end
end

end
end
end end end

0 comments on commit c06e319

Please sign in to comment.