From 5ecbee3b462411fc71df032de8bfacb8a45ffefb Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Mon, 19 Sep 2022 18:41:24 +0300 Subject: [PATCH 01/59] Optimize CI execution on PR and Push, to not run them twice (#858) --- .github/workflows/ruby.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 2acb40f31..5269cf449 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -1,6 +1,13 @@ name: CI -on: [push, pull_request] +on: + push: + branches: [master] + pull_request: + types: [ + synchronize, # PR was updated + opened # PR was open + ] jobs: ruby-2: From 73b2fb17ed832e6ac79712f13a291cd98e83d6e4 Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Mon, 10 Oct 2022 14:56:13 +0300 Subject: [PATCH 02/59] Allow async options for `delete_all` method (#857) That enables aynchronous journal clean up. Co-Authored-By: Bartek Bulat Co-authored-by: Alex Popov Co-authored-by: Bartek Bulat Co-authored-by: Alex Popov --- CHANGELOG.md | 2 + README.md | 13 ++++++ lib/chewy/journal.rb | 8 +++- lib/chewy/rake_helper.rb | 43 ++++++++++++++++--- lib/chewy/search/request.rb | 18 ++++++-- lib/chewy/stash.rb | 6 +-- lib/tasks/chewy.rake | 8 +++- spec/chewy/rake_helper_spec.rb | 68 +++++++++++++++++++++++++++++++ spec/chewy/search/request_spec.rb | 25 ++++++++++++ 9 files changed, 177 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74c3a5573..51db56ca0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### New Features +* [#857](https://github.com/toptal/chewy/pull/857): Allow passing `wait_for_completion`, `request_per_second` and `scroll_size` options to `chewy:journal:clean` rake task and `delete_all` query builder method. ([@konalegi][])([@barthez][]) + ### Changes ### Bugs Fixed diff --git a/README.md b/README.md index 38c1431fc..a84d0c4df 100644 --- a/README.md +++ b/README.md @@ -677,6 +677,8 @@ You may be wondering why do you need it? The answer is simple: not to lose the d Imagine that you reset your index in a zero-downtime manner (to separate index), and at the meantime somebody keeps updating the data frequently (to old index). So all these actions will be written to the journal index and you'll be able to apply them after index reset using the `Chewy::Journal` interface. +When enabled, journal can grow to enormous size, consider setting up cron job that would clean it occasionally using [`chewy:journal:clean` rake task](#chewyjournal). + ### Index manipulation ```ruby @@ -1144,6 +1146,17 @@ rake chewy:journal:apply["$(date -v-1H -u +%FT%TZ)"] # apply journaled changes f rake chewy:journal:apply["$(date -v-1H -u +%FT%TZ)",users] # apply journaled changes for the past hour on UsersIndex only ``` +When the size of the journal becomes very large, the classical way of deletion would be obstructive and resource consuming. Fortunately, Chewy internally uses [delete-by-query](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/docs-delete-by-query.html#docs-delete-by-query-task-api) ES function which supports async execution with batching and [throttling](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html#docs-delete-by-query-throttle). + +The available options, which can be set by ENV variables, are listed below: +* `WAIT_FOR_COMPLETION` - a boolean flag. It controls async execution. It waits by default. When set to `false` (`0`, `f`, `false` or `off` in any case spelling is accepted as `false`), Elasticsearch performs some preflight checks, launches the request, and returns a task reference you can use to cancel the task or get its status. +* `REQUESTS_PER_SECOND` - float. The throttle for this request in sub-requests per second. No throttling is enforced by default. +* `SCROLL_SIZE` - integer. The number of documents to be deleted in single sub-request. The default batch size is 1000. + +```bash +rake chewy:journal:clean WAIT_FOR_COMPLETION=false REQUESTS_PER_SECOND=10 SCROLL_SIZE=5000 +``` + ### RSpec integration Just add `require 'chewy/rspec'` to your spec_helper.rb and you will get additional features: diff --git a/lib/chewy/journal.rb b/lib/chewy/journal.rb index d1a887140..056601465 100644 --- a/lib/chewy/journal.rb +++ b/lib/chewy/journal.rb @@ -43,8 +43,12 @@ def apply(since_time, fetch_limit: 10, **import_options) # # @param until_time [Time, DateTime] time to clean up until it # @return [Hash] delete_by_query ES API call result - def clean(until_time = nil) - Chewy::Stash::Journal.clean(until_time, only: @only) + def clean(until_time = nil, delete_by_query_options: {}) + Chewy::Stash::Journal.clean( + until_time, + only: @only, + delete_by_query_options: delete_by_query_options.merge(refresh: false) + ) end private diff --git a/lib/chewy/rake_helper.rb b/lib/chewy/rake_helper.rb index 8e0de510d..c9150b8e8 100644 --- a/lib/chewy/rake_helper.rb +++ b/lib/chewy/rake_helper.rb @@ -19,6 +19,9 @@ module RakeHelper output.puts " Applying journal to #{targets}, #{count} entries, stage #{payload[:stage]}" end + DELETE_BY_QUERY_OPTIONS = %w[WAIT_FOR_COMPLETION REQUESTS_PER_SECOND SCROLL_SIZE].freeze + FALSE_VALUES = %w[0 f false off].freeze + class << self # Performs zero-downtime reindexing of all documents for the specified indexes # @@ -162,7 +165,7 @@ def journal_apply(time: nil, only: nil, except: nil, output: $stdout) subscribed_task_stats(output) do output.puts "Applying journal entries created after #{time}" - count = Chewy::Journal.new(indexes_from(only: only, except: except)).apply(time) + count = Chewy::Journal.new(journal_indexes_from(only: only, except: except)).apply(time) output.puts 'No journal entries were created after the specified time' if count.zero? end end @@ -181,12 +184,16 @@ def journal_apply(time: nil, only: nil, except: nil, output: $stdout) # @param except [Array, Chewy::Index, String] indexes to exclude from processing # @param output [IO] output io for logging # @return [Array] indexes that were actually updated - def journal_clean(time: nil, only: nil, except: nil, output: $stdout) + def journal_clean(time: nil, only: nil, except: nil, delete_by_query_options: {}, output: $stdout) subscribed_task_stats(output) do output.puts "Cleaning journal entries created before #{time}" if time - response = Chewy::Journal.new(indexes_from(only: only, except: except)).clean(time) - count = response['deleted'] || response['_indices']['_all']['deleted'] - output.puts "Cleaned up #{count} journal entries" + response = Chewy::Journal.new(journal_indexes_from(only: only, except: except)).clean(time, delete_by_query_options: delete_by_query_options) + if response.key?('task') + output.puts "Task to cleanup the journal has been created, #{response['task']}" + else + count = response['deleted'] || response['_indices']['_all']['deleted'] + output.puts "Cleaned up #{count} journal entries" + end end end @@ -228,6 +235,26 @@ def update_mapping(name:, output: $stdout) end end + # Reads options that are required to run journal cleanup asynchronously from ENV hash + # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html + # + # @example + # Chewy::RakeHelper.delete_by_query_options_from_env({'WAIT_FOR_COMPLETION' => 'false','REQUESTS_PER_SECOND' => '10','SCROLL_SIZE' => '5000'}) + # # => { wait_for_completion: false, requests_per_second: 10.0, scroll_size: 5000 } + # + def delete_by_query_options_from_env(env) + env + .slice(*DELETE_BY_QUERY_OPTIONS) + .transform_keys { |k| k.downcase.to_sym } + .to_h do |key, value| + case key + when :wait_for_completion then [key, !FALSE_VALUES.include?(value.downcase)] + when :requests_per_second then [key, value.to_f] + when :scroll_size then [key, value.to_i] + end + end + end + def normalize_indexes(*identifiers) identifiers.flatten(1).map { |identifier| normalize_index(identifier) } end @@ -248,6 +275,12 @@ def subscribed_task_stats(output = $stdout, &block) private + def journal_indexes_from(only: nil, except: nil) + return if Array.wrap(only).empty? && Array.wrap(except).empty? + + indexes_from(only: only, except: except) + end + def indexes_from(only: nil, except: nil) indexes = if only.present? normalize_indexes(Array.wrap(only)) diff --git a/lib/chewy/search/request.rb b/lib/chewy/search/request.rb index 687e8a093..6c7a62156 100644 --- a/lib/chewy/search/request.rb +++ b/lib/chewy/search/request.rb @@ -962,10 +962,22 @@ def pluck(*fields) # # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html # @note The result hash is different for different API used. - # @param refresh [true, false] field names + # @param refresh [true, false] Refreshes all shards involved in the delete by query + # @param wait_for_completion [true, false] wait for request completion or run it asynchronously + # and return task reference at `.tasks/task/${taskId}`. + # @param requests_per_second [Float] The throttle for this request in sub-requests per second + # @param scroll_size [Integer] Size of the scroll request that powers the operation + # @return [Hash] the result of query execution - def delete_all(refresh: true) - request_body = only(WHERE_STORAGES).render.merge(refresh: refresh) + def delete_all(refresh: true, wait_for_completion: nil, requests_per_second: nil, scroll_size: nil) + request_body = only(WHERE_STORAGES).render.merge( + { + refresh: refresh, + wait_for_completion: wait_for_completion, + requests_per_second: requests_per_second, + scroll_size: scroll_size + }.compact + ) ActiveSupport::Notifications.instrument 'delete_query.chewy', notification_payload(request: request_body) do request_body[:body] = {query: {match_all: {}}} if request_body[:body].empty? Chewy.client.delete_by_query(request_body) diff --git a/lib/chewy/stash.rb b/lib/chewy/stash.rb index 2181ee492..d49957d50 100644 --- a/lib/chewy/stash.rb +++ b/lib/chewy/stash.rb @@ -28,12 +28,12 @@ def self.entries(since_time, only: []) # Cleans up all the journal entries until the specified time. If nothing is # specified - cleans up everything. # - # @param since_time [Time, DateTime] the time top boundary + # @param until_time [Time, DateTime] Clean everything before that date # @param only [Chewy::Index, Array] indexes to clean up journal entries for - def self.clean(until_time = nil, only: []) + def self.clean(until_time = nil, only: [], delete_by_query_options: {}) scope = self.for(only) scope = scope.filter(range: {created_at: {lte: until_time}}) if until_time - scope.delete_all + scope.delete_all(**delete_by_query_options) end # Selects all the journal entries for the specified indices. diff --git a/lib/tasks/chewy.rake b/lib/tasks/chewy.rake index 5cf4ef17c..6071ccdca 100644 --- a/lib/tasks/chewy.rake +++ b/lib/tasks/chewy.rake @@ -94,7 +94,13 @@ namespace :chewy do desc 'Removes journal records created before the specified timestamp for the specified indexes/types or all of them' task clean: :environment do |_task, args| - Chewy::RakeHelper.journal_clean(**parse_journal_args(args.extras)) + delete_options = Chewy::RakeHelper.delete_by_query_options_from_env(ENV) + Chewy::RakeHelper.journal_clean( + [ + parse_journal_args(args.extras), + {delete_by_query_options: delete_options} + ].reduce({}, :merge) + ) end end end diff --git a/spec/chewy/rake_helper_spec.rb b/spec/chewy/rake_helper_spec.rb index d9754ef68..d0f84b15c 100644 --- a/spec/chewy/rake_helper_spec.rb +++ b/spec/chewy/rake_helper_spec.rb @@ -426,6 +426,33 @@ described_class.journal_clean(except: CitiesIndex, output: output) expect(output.string).to match(Regexp.new(<<-OUTPUT, Regexp::MULTILINE)) \\ACleaned up 1 journal entries +Total: \\d+s\\Z + OUTPUT + end + + it 'executes asynchronously' do + output = StringIO.new + expect(Chewy.client).to receive(:delete_by_query).with( + { + body: {query: {match_all: {}}}, + index: ['chewy_journal'], + refresh: false, + requests_per_second: 10.0, + scroll_size: 200, + wait_for_completion: false + } + ).and_call_original + described_class.journal_clean( + output: output, + delete_by_query_options: { + wait_for_completion: false, + requests_per_second: 10.0, + scroll_size: 200 + } + ) + + expect(output.string).to match(Regexp.new(<<-OUTPUT, Regexp::MULTILINE)) +\\ATask to cleanup the journal has been created, [^\\n]* Total: \\d+s\\Z OUTPUT end @@ -502,4 +529,45 @@ end end end + + describe '.delete_by_query_options_from_env' do + subject(:options) { described_class.delete_by_query_options_from_env(env) } + let(:env) do + { + 'WAIT_FOR_COMPLETION' => 'false', + 'REQUESTS_PER_SECOND' => '10', + 'SCROLL_SIZE' => '5000' + } + end + + it 'parses the options' do + expect(options).to eq( + wait_for_completion: false, + requests_per_second: 10.0, + scroll_size: 5000 + ) + end + + context 'with different boolean values' do + it 'parses the option correctly' do + %w[1 t true TRUE on ON].each do |v| + expect(described_class.delete_by_query_options_from_env({'WAIT_FOR_COMPLETION' => v})) + .to eq(wait_for_completion: true) + end + + %w[0 f false FALSE off OFF].each do |v| + expect(described_class.delete_by_query_options_from_env({'WAIT_FOR_COMPLETION' => v})) + .to eq(wait_for_completion: false) + end + end + end + + context 'with other env' do + let(:env) { {'SOME_ENV' => '123', 'REQUESTS_PER_SECOND' => '15'} } + + it 'parses only the options' do + expect(options).to eq(requests_per_second: 15.0) + end + end + end end diff --git a/spec/chewy/search/request_spec.rb b/spec/chewy/search/request_spec.rb index f8ea8d1ea..4aede10f3 100644 --- a/spec/chewy/search/request_spec.rb +++ b/spec/chewy/search/request_spec.rb @@ -817,6 +817,31 @@ request: {index: ['products'], body: {query: {match: {name: 'name3'}}}, refresh: false} ) end + + it 'delete records asynchronously' do + outer_payload = nil + ActiveSupport::Notifications.subscribe('delete_query.chewy') do |_name, _start, _finish, _id, payload| + outer_payload = payload + end + subject.query(match: {name: 'name3'}).delete_all( + refresh: false, + wait_for_completion: false, + requests_per_second: 10.0, + scroll_size: 2000 + ) + expect(outer_payload).to eq( + index: ProductsIndex, + indexes: [ProductsIndex], + request: { + index: ['products'], + body: {query: {match: {name: 'name3'}}}, + refresh: false, + wait_for_completion: false, + requests_per_second: 10.0, + scroll_size: 2000 + } + ) + end end describe '#response=' do From 4c067e2421109689087aa74483090bb49a209d88 Mon Sep 17 00:00:00 2001 From: Viktor Date: Tue, 15 Nov 2022 15:14:15 +0200 Subject: [PATCH 03/59] Fix redundant crutch call with :update_fields opt (#863) --- CHANGELOG.md | 2 ++ lib/chewy/index/import/bulk_builder.rb | 9 ++++----- spec/chewy/index/import/bulk_builder_spec.rb | 4 ++++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51db56ca0..ef2ae2757 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ ### Bugs Fixed +* [#863](https://github.com/toptal/chewy/pull/863): Fix `crutches` call doesn't respect `update_fields` option. ([@skcc321][]) + ## 7.2.6 (2022-06-13) ### New Features diff --git a/lib/chewy/index/import/bulk_builder.rb b/lib/chewy/index/import/bulk_builder.rb index c39a30cfd..60917b511 100644 --- a/lib/chewy/index/import/bulk_builder.rb +++ b/lib/chewy/index/import/bulk_builder.rb @@ -48,12 +48,11 @@ def crutches_for_index def index_entry(object) entry = {} entry[:_id] = index_object_ids[object] if index_object_ids[object] + entry[:routing] = routing(object) if join_field? - data = data_for(object) parent = cache(entry[:_id]) - - entry[:routing] = routing(object) if join_field? - if parent_changed?(data, parent) + data = data_for(object) if parent.present? + if parent.present? && parent_changed?(data, parent) reindex_entries(object, data) + reindex_descendants(object) elsif @fields.present? return [] unless entry[:_id] @@ -61,7 +60,7 @@ def index_entry(object) entry[:data] = {doc: data_for(object, fields: @fields)} [{update: entry}] else - entry[:data] = data + entry[:data] = data || data_for(object) [{index: entry}] end end diff --git a/spec/chewy/index/import/bulk_builder_spec.rb b/spec/chewy/index/import/bulk_builder_spec.rb index c4b21ae22..dd63c442c 100644 --- a/spec/chewy/index/import/bulk_builder_spec.rb +++ b/spec/chewy/index/import/bulk_builder_spec.rb @@ -62,6 +62,8 @@ def derived let(:to_index) { cities.first(2) } let(:delete) { [cities.last] } specify do + expect(subject).to receive(:data_for).with(cities.first).and_call_original + expect(subject).to receive(:data_for).with(cities.second).and_call_original expect(subject.bulk_body).to eq([ {index: {_id: 1, data: {'name' => 'City17', 'rating' => 42}}}, {index: {_id: 2, data: {'name' => 'City18', 'rating' => 42}}}, @@ -72,6 +74,8 @@ def derived context ':fields' do let(:fields) { %w[name] } specify do + expect(subject).to receive(:data_for).with(cities.first, fields: [:name]).and_call_original + expect(subject).to receive(:data_for).with(cities.second, fields: [:name]).and_call_original expect(subject.bulk_body).to eq([ {update: {_id: 1, data: {doc: {'name' => 'City17'}}}}, {update: {_id: 2, data: {doc: {'name' => 'City18'}}}}, From e6fe79252a59da6bffa4e6f78f2b61a65a3b5868 Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Tue, 15 Nov 2022 16:49:55 +0300 Subject: [PATCH 04/59] Prepare 7.2.7 release (#865) --- CHANGELOG.md | 8 ++++++++ lib/chewy/version.rb | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef2ae2757..16dae1344 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ ### New Features +### Changes + +### Bugs Fixed + +## 7.2.7 (2022-11-15) + +### New Features + * [#857](https://github.com/toptal/chewy/pull/857): Allow passing `wait_for_completion`, `request_per_second` and `scroll_size` options to `chewy:journal:clean` rake task and `delete_all` query builder method. ([@konalegi][])([@barthez][]) ### Changes diff --git a/lib/chewy/version.rb b/lib/chewy/version.rb index 5b896f45c..39af809b9 100644 --- a/lib/chewy/version.rb +++ b/lib/chewy/version.rb @@ -1,3 +1,3 @@ module Chewy - VERSION = '7.2.6'.freeze + VERSION = '7.2.7'.freeze end From 7869c3b20123e84d667140cb25a927ef746c456c Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Fri, 23 Dec 2022 10:06:54 +0300 Subject: [PATCH 05/59] Fix github CI --- .github/workflows/ruby.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 5269cf449..297549ec1 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -6,7 +6,8 @@ on: pull_request: types: [ synchronize, # PR was updated - opened # PR was open + opened, # PR was open + reopened # PR was reopened ] jobs: From 0ad19abc4cdde1eb1ae83c08e46bd9b610708a87 Mon Sep 17 00:00:00 2001 From: Fabio Maia Date: Fri, 23 Dec 2022 11:22:43 -0500 Subject: [PATCH 06/59] Fix return value for subscribed_task_stats (#856) --- CHANGELOG.md | 2 ++ lib/chewy/rake_helper.rb | 1 + spec/chewy/rake_helper_spec.rb | 7 +++++++ 3 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16dae1344..0823534ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### Bugs Fixed +* [#856](https://github.com/toptal/chewy/pull/856): Fix return value of subscribed_task_stats used in rake tasks. ([@fabiormoura][]) + ## 7.2.7 (2022-11-15) ### New Features diff --git a/lib/chewy/rake_helper.rb b/lib/chewy/rake_helper.rb index c9150b8e8..8814b4b04 100644 --- a/lib/chewy/rake_helper.rb +++ b/lib/chewy/rake_helper.rb @@ -270,6 +270,7 @@ def subscribed_task_stats(output = $stdout, &block) ActiveSupport::Notifications.subscribed(JOURNAL_CALLBACK.curry[output], 'apply_journal.chewy') do ActiveSupport::Notifications.subscribed(IMPORT_CALLBACK.curry[output], 'import_objects.chewy', &block) end + ensure output.puts "Total: #{human_duration(Time.now - start)}" end diff --git a/spec/chewy/rake_helper_spec.rb b/spec/chewy/rake_helper_spec.rb index d0f84b15c..dee34afe9 100644 --- a/spec/chewy/rake_helper_spec.rb +++ b/spec/chewy/rake_helper_spec.rb @@ -570,4 +570,11 @@ end end end + + describe '.subscribed_task_stats' do + specify do + block_output = described_class.subscribed_task_stats(StringIO.new) { 'expected output' } + expect(block_output).to eq('expected output') + end + end end From fed4a21120c1fb80cfa4b455c90aeecade82ef3e Mon Sep 17 00:00:00 2001 From: Agnieszka Stec <51296866+Ygnys@users.noreply.github.com> Date: Fri, 13 Jan 2023 10:32:21 +0100 Subject: [PATCH 07/59] fixed a few typos in the Readme file (#867) --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a84d0c4df..3ac60cd06 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ # Chewy -Chewy is an ODM (Object Document Mapper), built on top of the [the official Elasticsearch client](https://github.com/elastic/elasticsearch-ruby). +Chewy is an ODM (Object Document Mapper), built on top of [the official Elasticsearch client](https://github.com/elastic/elasticsearch-ruby). ## Why Chewy? @@ -458,7 +458,7 @@ field :hierarchy_link, type: :join, relations: {question: %i[answer comment], an ``` assuming you have `comment_type` and `commented_id` fields in your model. -Note that when you reindex a parent, it's children and grandchildren will be reindexed as well. +Note that when you reindex a parent, its children and grandchildren will be reindexed as well. This may require additional queries to the primary database and to elastisearch. Also note that the join field doesn't support crutches (it should be a field directly defined on the model). @@ -525,7 +525,7 @@ So Chewy Crutches™ technology is able to increase your indexing performance in ### Witchcraft™ technology -One more experimental technology to increase import performance. As far as you know, chewy defines value proc for every imported field in mapping, so at the import time each of this procs is executed on imported object to extract result document to import. It would be great for performance to use one huge whole-document-returning proc instead. So basically the idea or Witchcraft™ technology is to compile a single document-returning proc from the index definition. +One more experimental technology to increase import performance. As far as you know, chewy defines value proc for every imported field in mapping, so at the import time each of these procs is executed on imported object to extract result document to import. It would be great for performance to use one huge whole-document-returning proc instead. So basically the idea or Witchcraft™ technology is to compile a single document-returning proc from the index definition. ```ruby index_scope Product @@ -569,7 +569,7 @@ Obviously not every type of definition might be compiled. There are some restric end ``` -However, it is quite possible that your index definition will be supported by Witchcraft™ technology out of the box in the most of the cases. +However, it is quite possible that your index definition will be supported by Witchcraft™ technology out of the box in most of the cases. ### Raw Import @@ -675,7 +675,7 @@ end You may be wondering why do you need it? The answer is simple: not to lose the data. -Imagine that you reset your index in a zero-downtime manner (to separate index), and at the meantime somebody keeps updating the data frequently (to old index). So all these actions will be written to the journal index and you'll be able to apply them after index reset using the `Chewy::Journal` interface. +Imagine that you reset your index in a zero-downtime manner (to separate index), and in the meantime somebody keeps updating the data frequently (to old index). So all these actions will be written to the journal index and you'll be able to apply them after index reset using the `Chewy::Journal` interface. When enabled, journal can grow to enormous size, consider setting up cron job that would clean it occasionally using [`chewy:journal:clean` rake task](#chewyjournal). @@ -888,7 +888,7 @@ Chewy has notifying the following events: {index: 30, delete: 5} ``` - * `payload[:errors]`: might not exists. Contains grouped errors with objects ids list: + * `payload[:errors]`: might not exist. Contains grouped errors with objects ids list: ```ruby {index: { @@ -1020,7 +1020,7 @@ Request DSL also provides additional scope actions, like `delete_all`, `exists?` #### Pagination -The request DSL supports pagination with `Kaminari`. An extension is enabled on initializtion if `Kaminari` is available. See [Chewy::Search](lib/chewy/search.rb) and [Chewy::Search::Pagination::Kaminari](lib/chewy/search/pagination/kaminari.rb) for details. +The request DSL supports pagination with `Kaminari`. An extension is enabled on initialization if `Kaminari` is available. See [Chewy::Search](lib/chewy/search.rb) and [Chewy::Search::Pagination::Kaminari](lib/chewy/search/pagination/kaminari.rb) for details. #### Named scopes From 7ee400557402fcfe90881a158297b89c835e628f Mon Sep 17 00:00:00 2001 From: Alex Popov Date: Tue, 17 Jan 2023 15:37:44 +0200 Subject: [PATCH 08/59] Rename Rogue One to Platform SRE (#870) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2ccdef619..4968d730e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -.github/workflows @toptal/rogue-one +.github/workflows @toptal/platform-sre From 8ea2cfe5694d05ed34c32918f3465defcb9b73ae Mon Sep 17 00:00:00 2001 From: Viktor Date: Fri, 31 Mar 2023 18:47:24 +0300 Subject: [PATCH 09/59] [feature] delayed_sidekiq strategy (#869) --- CHANGELOG.md | 2 + README.md | 74 +++++++ chewy.gemspec | 1 + lib/chewy/index.rb | 25 +++ lib/chewy/index/import.rb | 31 ++- lib/chewy/strategy.rb | 1 + lib/chewy/strategy/delayed_sidekiq.rb | 17 ++ .../strategy/delayed_sidekiq/scheduler.rb | 148 ++++++++++++++ lib/chewy/strategy/delayed_sidekiq/worker.rb | 52 +++++ spec/chewy/strategy/delayed_sidekiq_spec.rb | 190 ++++++++++++++++++ 10 files changed, 539 insertions(+), 2 deletions(-) create mode 100644 lib/chewy/strategy/delayed_sidekiq.rb create mode 100644 lib/chewy/strategy/delayed_sidekiq/scheduler.rb create mode 100644 lib/chewy/strategy/delayed_sidekiq/worker.rb create mode 100644 spec/chewy/strategy/delayed_sidekiq_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 0823534ba..2da701b2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### New Features +* [#869](https://github.com/toptal/chewy/pull/869): New strategy - `delayed_sidekiq`. Allow passing `strategy: :delayed_sidekiq` option to `SomeIndex.import([1, ...], strategy: :delayed_sidekiq)`. The strategy is compatible with `update_fields` option as well. ([@skcc321][]) + ### Changes ### Bugs Fixed diff --git a/README.md b/README.md index 3ac60cd06..149e8ae54 100644 --- a/README.md +++ b/README.md @@ -774,6 +774,80 @@ The default queue name is `chewy`, you can customize it in settings: `sidekiq.qu Chewy.settings[:sidekiq] = {queue: :low} ``` +#### `:delayed_sidekiq` + +It accumulates ids of records to be reindexed during the latency window in redis and then does the reindexing of all accumulated records at once. +The strategy is very useful in case of frequently mutated records. +It supports `update_fields` option, so it will try to select just enough data from the DB + +There are three options that can be defined in the index: +```ruby +class CitiesIndex... + strategy_config delayed_sidekiq: { + latency: 3, + margin: 2, + ttl: 60 * 60 * 24, + reindex_wrapper: ->(&reindex) { + ActiveRecord::Base.connected_to(role: :reading) { reindex.call } + } + # latency - will prevent scheduling identical jobs + # margin - main purpose is to cover db replication lag by the margin + # ttl - a chunk expiration time (in seconds) + # reindex_wrapper - lambda that accepts block to wrap that reindex process AR connection block. + } + + ... +end +``` + +Also you can define defaults in the `initializers/chewy.rb` +```ruby +Chewy.settings = { + strategy_config: { + delayed_sidekiq: { + latency: 3, + margin: 2, + ttl: 60 * 60 * 24, + reindex_wrapper: ->(&reindex) { + ActiveRecord::Base.connected_to(role: :reading) { reindex.call } + } + } + } +} + +``` +or in `config/chewy.yml` +```ruby + strategy_config: + delayed_sidekiq: + latency: 3 + margin: 2 + ttl: <%= 60 * 60 * 24 %> + # reindex_wrapper setting is not possible here!!! use the initializer instead +``` + +You can use the strategy identically to other strategies +```ruby +Chewy.strategy(:delayed_sidekiq) do + City.popular.map(&:do_some_update_action!) +end +``` + +The default queue name is `chewy`, you can customize it in settings: `sidekiq.queue_name` +``` +Chewy.settings[:sidekiq] = {queue: :low} +``` + +Explicit call of the reindex using `:delayed_sidekiq strategy` +```ruby +CitiesIndex.import([1, 2, 3], strategy: :delayed_sidekiq) +``` + +Explicit call of the reindex using `:delayed_sidekiq` strategy with `:update_fields` support +```ruby +CitiesIndex.import([1, 2, 3], update_fields: [:name], strategy: :delayed_sidekiq) +``` + #### `:active_job` This does the same thing as `:atomic`, but using ActiveJob. This will inherit the ActiveJob configuration settings including the `active_job.queue_adapter` setting for the environment. Patch `Chewy::Strategy::ActiveJob::Worker` for index updates improving. diff --git a/chewy.gemspec b/chewy.gemspec index e85d12b84..0e542ab56 100644 --- a/chewy.gemspec +++ b/chewy.gemspec @@ -19,6 +19,7 @@ Gem::Specification.new do |spec| # rubocop:disable Metrics/BlockLength spec.add_development_dependency 'database_cleaner' spec.add_development_dependency 'elasticsearch-extensions' + spec.add_development_dependency 'mock_redis' spec.add_development_dependency 'rake' spec.add_development_dependency 'rspec', '>= 3.7.0' spec.add_development_dependency 'rspec-collection_matchers' diff --git a/lib/chewy/index.rb b/lib/chewy/index.rb index cdb3fa055..4c63cd8e4 100644 --- a/lib/chewy/index.rb +++ b/lib/chewy/index.rb @@ -20,6 +20,10 @@ class Index pipeline raw_import refresh replication ].freeze + STRATEGY_OPTIONS = { + delayed_sidekiq: %i[latency margin ttl reindex_wrapper] + }.freeze + include Search include Actions include Aliases @@ -221,6 +225,27 @@ def default_import_options(params) params.assert_valid_keys(IMPORT_OPTIONS_KEYS) self._default_import_options = _default_import_options.merge(params) end + + def strategy_config(params = {}) + @strategy_config ||= begin + config_struct = Struct.new(*STRATEGY_OPTIONS.keys).new + + STRATEGY_OPTIONS.each_with_object(config_struct) do |(strategy, options), res| + res[strategy] = case strategy + when :delayed_sidekiq + Struct.new(*STRATEGY_OPTIONS[strategy]).new.tap do |config| + options.each do |option| + config[option] = params.dig(strategy, option) || Chewy.configuration.dig(:strategy_config, strategy, option) + end + + config[:reindex_wrapper] ||= ->(&reindex) { reindex.call } # default wrapper + end + else + raise NotImplementedError, "Unsupported strategy: '#{strategy}'" + end + end + end + end end end end diff --git a/lib/chewy/index/import.rb b/lib/chewy/index/import.rb index d9a23aaee..cc50a4056 100644 --- a/lib/chewy/index/import.rb +++ b/lib/chewy/index/import.rb @@ -73,7 +73,7 @@ module ClassMethods # @option options [true, Integer, Hash] parallel enables parallel import processing with the Parallel gem, accepts the number of workers or any Parallel gem acceptable options # @return [true, false] false in case of errors ruby2_keywords def import(*args) - import_routine(*args).blank? + intercept_import_using_strategy(*args).blank? end # @!method import!(*collection, **options) @@ -84,7 +84,8 @@ module ClassMethods # # @raise [Chewy::ImportFailed] in case of errors ruby2_keywords def import!(*args) - errors = import_routine(*args) + errors = intercept_import_using_strategy(*args) + raise Chewy::ImportFailed.new(self, errors) if errors.present? true @@ -126,6 +127,32 @@ def compose(object, crutches = nil, fields: []) private + def intercept_import_using_strategy(*args) + args_clone = args.deep_dup + options = args_clone.extract_options! + strategy = options.delete(:strategy) + + return import_routine(*args) if strategy.blank? + + ids = args_clone.flatten + return {} if ids.blank? + return {argument: {"#{strategy} supports ids only!" => ids}} unless ids.all? do |id| + id.respond_to?(:to_i) + end + + case strategy + when :delayed_sidekiq + begin + Chewy::Strategy::DelayedSidekiq::Scheduler.new(self, ids, options).postpone + {} # success. errors handling convention + rescue StandardError => e + {scheduler: {e.message => ids}} + end + else + {argument: {"unsupported strategy: '#{strategy}'" => ids}} + end + end + def import_routine(*args) return if !args.first.nil? && empty_objects_or_scope?(args.first) diff --git a/lib/chewy/strategy.rb b/lib/chewy/strategy.rb index 83baadb24..a8c6c5dfa 100644 --- a/lib/chewy/strategy.rb +++ b/lib/chewy/strategy.rb @@ -8,6 +8,7 @@ require 'sidekiq' require 'chewy/strategy/sidekiq' require 'chewy/strategy/lazy_sidekiq' + require 'chewy/strategy/delayed_sidekiq' rescue LoadError nil end diff --git a/lib/chewy/strategy/delayed_sidekiq.rb b/lib/chewy/strategy/delayed_sidekiq.rb new file mode 100644 index 000000000..2c694071c --- /dev/null +++ b/lib/chewy/strategy/delayed_sidekiq.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Chewy + class Strategy + class DelayedSidekiq < Sidekiq + require_relative 'delayed_sidekiq/scheduler' + + def leave + @stash.each do |type, ids| + next if ids.empty? + + DelayedSidekiq::Scheduler.new(type, ids).postpone + end + end + end + end +end diff --git a/lib/chewy/strategy/delayed_sidekiq/scheduler.rb b/lib/chewy/strategy/delayed_sidekiq/scheduler.rb new file mode 100644 index 000000000..9f4bf386e --- /dev/null +++ b/lib/chewy/strategy/delayed_sidekiq/scheduler.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +require_relative '../../index' + +# The class is responsible for accumulating in redis [type, ids] +# that were requested to be reindexed during `latency` seconds. +# The reindex job is going to be scheduled after a `latency` seconds. +# that job is going to read accumulated [type, ids] from the redis +# and reindex all them at once. +module Chewy + class Strategy + class DelayedSidekiq + require_relative 'worker' + + class Scheduler + DEFAULT_TTL = 60 * 60 * 24 # in seconds + DEFAULT_LATENCY = 10 + DEFAULT_MARGIN = 2 + DEFAULT_QUEUE = 'chewy' + KEY_PREFIX = 'chewy:delayed_sidekiq' + FALLBACK_FIELDS = 'all' + FIELDS_IDS_SEPARATOR = ';' + IDS_SEPARATOR = ',' + + def initialize(type, ids, options = {}) + @type = type + @ids = ids + @options = options + end + + # the diagram: + # + # inputs: + # latency == 2 + # reindex_time = Time.current + # + # Parallel OR Sequential triggers of reindex: | What is going on in reindex store (Redis): + # -------------------------------------------------------------------------------------------------- + # | + # process 1 (reindex_time): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1] + # Schedule.new(CitiesIndex, [1]).postpone | chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}] + # | & schedule a DelayedSidekiq::Worker at 1679347869 (at + 3) + # | it will zpop chewy:delayed_sidekiq:timechunks up to 1679347866 score and reindex all ids with zpoped keys + # | chewy:delayed_sidekiq:CitiesIndex:1679347866 + # | + # | + # process 2 (reindex_time): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2] + # Schedule.new(CitiesIndex, [2]).postpone | chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}] + # | & do not schedule a new worker + # | + # | + # process 1 (reindex_time + (latency - 1).seconds): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3] + # Schedule.new(CitiesIndex, [3]).postpone | chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}] + # | & do not schedule a new worker + # | + # | + # process 2 (reindex_time + (latency + 1).seconds): | chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3] + # Schedule.new(CitiesIndex, [4]).postpone | chewy:delayed_sidekiq:CitiesIndex:1679347868 = [4] + # | chewy:delayed_sidekiq:timechunks = [ + # | { score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"} + # | { score: 1679347868, "chewy:delayed_sidekiq:CitiesIndex:1679347868"} + # | ] + # | & schedule a DelayedSidekiq::Worker at 1679347871 (at + 3) + # | it will zpop chewy:delayed_sidekiq:timechunks up to 1679347868 score and reindex all ids with zpoped keys + # | chewy:delayed_sidekiq:CitiesIndex:1679347866 (in case of failed previous reindex), + # | chewy:delayed_sidekiq:CitiesIndex:1679347868 + def postpone + ::Sidekiq.redis do |redis| + # warning: Redis#sadd will always return an Integer in Redis 5.0.0. Use Redis#sadd? instead + if redis.respond_to?(:sadd?) + redis.sadd?(timechunk_key, serialize_data) + else + redis.sadd(timechunk_key, serialize_data) + end + + redis.expire(timechunk_key, ttl) + + unless redis.zrank(timechunks_key, timechunk_key) + redis.zadd(timechunks_key, at, timechunk_key) + redis.expire(timechunks_key, ttl) + + ::Sidekiq::Client.push( + 'queue' => sidekiq_queue, + 'at' => at + margin, + 'class' => Chewy::Strategy::DelayedSidekiq::Worker, + 'args' => [type_name, at] + ) + end + end + end + + private + + attr_reader :type, :ids, :options + + # this method returns predictable value that jumps by latency value + # another words each latency seconds it return the same value + def at + @at ||= begin + schedule_at = latency.seconds.from_now.to_f + + (schedule_at - (schedule_at % latency)).to_i + end + end + + def fields + options[:update_fields].presence || [FALLBACK_FIELDS] + end + + def timechunks_key + "#{KEY_PREFIX}:#{type_name}:timechunks" + end + + def timechunk_key + "#{KEY_PREFIX}:#{type_name}:#{at}" + end + + def serialize_data + [ids.join(IDS_SEPARATOR), fields.join(IDS_SEPARATOR)].join(FIELDS_IDS_SEPARATOR) + end + + def type_name + type.name + end + + def latency + strategy_config.latency || DEFAULT_LATENCY + end + + def margin + strategy_config.margin || DEFAULT_MARGIN + end + + def ttl + strategy_config.ttl || DEFAULT_TTL + end + + def sidekiq_queue + Chewy.settings.dig(:sidekiq, :queue) || DEFAULT_QUEUE + end + + def strategy_config + type.strategy_config.delayed_sidekiq + end + end + end + end +end diff --git a/lib/chewy/strategy/delayed_sidekiq/worker.rb b/lib/chewy/strategy/delayed_sidekiq/worker.rb new file mode 100644 index 000000000..4d17a4cd1 --- /dev/null +++ b/lib/chewy/strategy/delayed_sidekiq/worker.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Chewy + class Strategy + class DelayedSidekiq + class Worker + include ::Sidekiq::Worker + + def perform(type, score, options = {}) + options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async + + ::Sidekiq.redis do |redis| + timechunks_key = "#{Scheduler::KEY_PREFIX}:#{type}:timechunks" + timechunk_keys = redis.zrangebyscore(timechunks_key, -1, score) + members = timechunk_keys.flat_map { |timechunk_key| redis.smembers(timechunk_key) }.compact + + # extract ids and fields & do the reset of records + ids, fields = extract_ids_and_fields(members) + options[:update_fields] = fields if fields + + index = type.constantize + index.strategy_config.delayed_sidekiq.reindex_wrapper.call do + options.any? ? index.import!(ids, **options) : index.import!(ids) + end + + redis.del(timechunk_keys) + redis.zremrangebyscore(timechunks_key, -1, score) + end + end + + private + + def extract_ids_and_fields(members) + ids = [] + fields = [] + + members.each do |member| + member_ids, member_fields = member.split(Scheduler::FIELDS_IDS_SEPARATOR).map do |v| + v.split(Scheduler::IDS_SEPARATOR) + end + ids |= member_ids + fields |= member_fields + end + + fields = nil if fields.include?(Scheduler::FALLBACK_FIELDS) + + [ids.map(&:to_i), fields] + end + end + end + end +end diff --git a/spec/chewy/strategy/delayed_sidekiq_spec.rb b/spec/chewy/strategy/delayed_sidekiq_spec.rb new file mode 100644 index 000000000..ac9f792c7 --- /dev/null +++ b/spec/chewy/strategy/delayed_sidekiq_spec.rb @@ -0,0 +1,190 @@ +require 'spec_helper' + +if defined?(Sidekiq) + require 'sidekiq/testing' + require 'mock_redis' + + describe Chewy::Strategy::DelayedSidekiq do + around do |example| + Chewy.strategy(:bypass) { example.run } + end + + before do + redis = MockRedis.new + allow(Sidekiq).to receive(:redis).and_yield(redis) + Sidekiq::Worker.clear_all + end + + before do + stub_model(:city) do + update_index('cities') { self } + end + + stub_index(:cities) do + index_scope City + end + end + + let(:city) { City.create!(name: 'hello') } + let(:other_city) { City.create!(name: 'world') } + + it 'does not trigger immediate reindex due to it`s async nature' do + expect { [city, other_city].map(&:save!) } + .not_to update_index(CitiesIndex, strategy: :delayed_sidekiq) + end + + it "respects 'refresh: false' options" do + allow(Chewy).to receive(:disable_refresh_async).and_return(true) + expect(CitiesIndex).to receive(:import!).with([city.id, other_city.id], refresh: false) + scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id, other_city.id]) + scheduler.postpone + Chewy::Strategy::DelayedSidekiq::Worker.drain + end + + context 'with default config' do + it 'does schedule a job that triggers reindex with default options' do + Timecop.freeze do + expect(Sidekiq::Client).to receive(:push).with( + hash_including( + 'queue' => 'chewy', + 'at' => (Time.current.to_i.ceil(-1) + 2.seconds).to_i, + 'class' => Chewy::Strategy::DelayedSidekiq::Worker, + 'args' => ['CitiesIndex', an_instance_of(Integer)] + ) + ).and_call_original + + expect($stdout).not_to receive(:puts) + + Sidekiq::Testing.inline! do + expect { [city, other_city].map(&:save!) } + .to update_index(CitiesIndex, strategy: :delayed_sidekiq) + .and_reindex(city, other_city).only + end + end + end + end + + context 'with custom config' do + before do + CitiesIndex.strategy_config( + delayed_sidekiq: { + reindex_wrapper: lambda { |&reindex| + puts 'hello' + reindex.call + }, + margin: 5, + latency: 60 + } + ) + end + + it 'respects :strategy_config options' do + Timecop.freeze do + expect(Sidekiq::Client).to receive(:push).with( + hash_including( + 'queue' => 'chewy', + 'at' => (60.seconds.from_now.change(sec: 0) + 5.seconds).to_i, + 'class' => Chewy::Strategy::DelayedSidekiq::Worker, + 'args' => ['CitiesIndex', an_instance_of(Integer)] + ) + ).and_call_original + + expect($stdout).to receive(:puts).with('hello') # check that reindex_wrapper works + + Sidekiq::Testing.inline! do + expect { [city, other_city].map(&:save!) } + .to update_index(CitiesIndex, strategy: :delayed_sidekiq) + .and_reindex(city, other_city).only + end + end + end + end + + context 'two reindex call within the timewindow' do + it 'accumulates all ids does the reindex one time' do + Timecop.freeze do + expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id]).once + scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id]) + scheduler.postpone + scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id]) + scheduler.postpone + Chewy::Strategy::DelayedSidekiq::Worker.drain + end + end + + context 'one call with update_fields another one without update_fields' do + it 'does reindex of all fields' do + Timecop.freeze do + expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id]).once + scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name']) + scheduler.postpone + scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id]) + scheduler.postpone + Chewy::Strategy::DelayedSidekiq::Worker.drain + end + end + end + + context 'both calls with different update fields' do + it 'deos reindex with union of fields' do + Timecop.freeze do + expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id], update_fields: %w[description name]).once + scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name']) + scheduler.postpone + scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id], update_fields: ['description']) + scheduler.postpone + Chewy::Strategy::DelayedSidekiq::Worker.drain + end + end + end + end + + context 'two calls within different timewindows' do + it 'does two separate reindexes' do + Timecop.freeze do + expect(CitiesIndex).to receive(:import!).with([city.id]).once + expect(CitiesIndex).to receive(:import!).with([other_city.id]).once + Timecop.travel(20.seconds.ago) do + scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id]) + scheduler.postpone + end + scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id]) + scheduler.postpone + Chewy::Strategy::DelayedSidekiq::Worker.drain + end + end + end + + context 'first call has update_fields' do + it 'does first reindex with the expected update_fields and second without update_fields' do + Timecop.freeze do + expect(CitiesIndex).to receive(:import!).with([city.id], update_fields: ['name']).once + expect(CitiesIndex).to receive(:import!).with([other_city.id]).once + Timecop.travel(20.seconds.ago) do + scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name']) + scheduler.postpone + end + scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id]) + scheduler.postpone + Chewy::Strategy::DelayedSidekiq::Worker.drain + end + end + end + + context 'both calls have update_fields option' do + it 'does both reindexes with their expected update_fields option' do + Timecop.freeze do + expect(CitiesIndex).to receive(:import!).with([city.id], update_fields: ['name']).once + expect(CitiesIndex).to receive(:import!).with([other_city.id], update_fields: ['description']).once + Timecop.travel(20.seconds.ago) do + scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name']) + scheduler.postpone + end + scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id], update_fields: ['description']) + scheduler.postpone + Chewy::Strategy::DelayedSidekiq::Worker.drain + end + end + end + end +end From 7566f1dcfe98d536c6381c851ecfb7b98ae4af18 Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Mon, 3 Apr 2023 11:00:49 +0300 Subject: [PATCH 10/59] Configure CI to check for ruby 3.2 compatibility (#879) --- .github/workflows/ruby.yml | 2 +- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 297549ec1..fd6b02b44 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -42,7 +42,7 @@ jobs: strategy: fail-fast: false matrix: - ruby: [ '3.0', 3.1 ] + ruby: [ '3.0', '3.1', '3.2' ] gemfile: [ rails.6.1.activerecord, rails.7.0.activerecord ] name: ${{ matrix.ruby }}-${{ matrix.gemfile }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 2da701b2e..6f8cfe2f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### New Features * [#869](https://github.com/toptal/chewy/pull/869): New strategy - `delayed_sidekiq`. Allow passing `strategy: :delayed_sidekiq` option to `SomeIndex.import([1, ...], strategy: :delayed_sidekiq)`. The strategy is compatible with `update_fields` option as well. ([@skcc321][]) +* [#879](https://github.com/toptal/chewy/pull/879): Configure CI to check for ruby 3.2 compatibility. ([@konalegi][]) ### Changes From 8b886d7587d7d3cb877a773fb283f6cf97796840 Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Mon, 3 Apr 2023 12:07:22 +0300 Subject: [PATCH 11/59] Prepare 7.3.0 release (#880) --- CHANGELOG.md | 8 ++++++++ lib/chewy/version.rb | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f8cfe2f1..fb8f20a25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ ### New Features +### Changes + +### Bugs Fixed + +## 7.3.0 (2023-04-03) + +### New Features + * [#869](https://github.com/toptal/chewy/pull/869): New strategy - `delayed_sidekiq`. Allow passing `strategy: :delayed_sidekiq` option to `SomeIndex.import([1, ...], strategy: :delayed_sidekiq)`. The strategy is compatible with `update_fields` option as well. ([@skcc321][]) * [#879](https://github.com/toptal/chewy/pull/879): Configure CI to check for ruby 3.2 compatibility. ([@konalegi][]) diff --git a/lib/chewy/version.rb b/lib/chewy/version.rb index 39af809b9..b1eb15dad 100644 --- a/lib/chewy/version.rb +++ b/lib/chewy/version.rb @@ -1,3 +1,3 @@ module Chewy - VERSION = '7.2.7'.freeze + VERSION = '7.3.0'.freeze end From 8e3fa9a8a0e4a0e57cbe37a51f958542b8a6f69e Mon Sep 17 00:00:00 2001 From: Kenta Mukai Date: Wed, 5 Apr 2023 16:14:22 +0900 Subject: [PATCH 12/59] Fix `chewy:journal:clean` task for ruby 3.x (#874) * with double splat operator * add test case for rake task `chewy:journal:clean` * :gear: fix Style/StringLiterals * update CHANGELOG --- CHANGELOG.md | 2 ++ lib/tasks/chewy.rake | 2 +- spec/chewy/rake_helper_spec.rb | 12 ++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb8f20a25..eb20db44a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### Bugs Fixed +* [#874](https://github.com/toptal/chewy/pull/874): Fix `chewy:journal:clean` task for ruby 3.x. ([@muk-ai](https://github.com/muk-ai)) + ## 7.3.0 (2023-04-03) ### New Features diff --git a/lib/tasks/chewy.rake b/lib/tasks/chewy.rake index 6071ccdca..f8a99560c 100644 --- a/lib/tasks/chewy.rake +++ b/lib/tasks/chewy.rake @@ -96,7 +96,7 @@ namespace :chewy do task clean: :environment do |_task, args| delete_options = Chewy::RakeHelper.delete_by_query_options_from_env(ENV) Chewy::RakeHelper.journal_clean( - [ + **[ parse_journal_args(args.extras), {delete_by_query_options: delete_options} ].reduce({}, :merge) diff --git a/spec/chewy/rake_helper_spec.rb b/spec/chewy/rake_helper_spec.rb index dee34afe9..f563b03d0 100644 --- a/spec/chewy/rake_helper_spec.rb +++ b/spec/chewy/rake_helper_spec.rb @@ -1,4 +1,5 @@ require 'spec_helper' +require 'rake' describe Chewy::RakeHelper, :orm do before { Chewy.massacre } @@ -456,6 +457,17 @@ Total: \\d+s\\Z OUTPUT end + + context 'execute "chewy:journal:clean" rake task' do + subject(:task) { Rake.application['chewy:journal:clean'] } + before do + Rake::DefaultLoader.new.load('lib/tasks/chewy.rake') + Rake::Task.define_task(:environment) + end + it 'does not raise error' do + expect { task.invoke }.to_not raise_error + end + end end describe '.reindex' do From 1e9a1e0b565f2b4c011e76ee5ba97cbdcda2ca19 Mon Sep 17 00:00:00 2001 From: Rob Nichols <119297020+RobNicholsGDS@users.noreply.github.com> Date: Thu, 6 Apr 2023 14:59:14 +0100 Subject: [PATCH 13/59] Replace bypass description with one that better describes behaviour (#876) --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 149e8ae54..deb94da81 100644 --- a/README.md +++ b/README.md @@ -884,7 +884,9 @@ It is convenient for use in e.g. the Rails console with non-block notation: #### `:bypass` -The bypass strategy simply silences index updates. +When the bypass strategy is active the index will not be automatically updated on object save. + +For example, on `City.first.save!` the cities index would not be updated. #### Nesting From 827591d6378a2b2b373d495931d8494346b766d1 Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Thu, 20 Apr 2023 17:29:31 +0300 Subject: [PATCH 14/59] Fix memory leak for ruby 3.2 (#882) * Fix memory leak for ruby 3.2 * Add changelog * Update documentation --- CHANGELOG.md | 1 + README.md | 2 +- lib/chewy/index/crutch.rb | 22 +++++++++++++++------- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb20db44a..1381025c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Bugs Fixed * [#874](https://github.com/toptal/chewy/pull/874): Fix `chewy:journal:clean` task for ruby 3.x. ([@muk-ai](https://github.com/muk-ai)) +* [#882](https://github.com/toptal/chewy/pull/882): Fix memory leak during `chewy:reset` for ruby 3.2 ([@konalegi](https://github.com/konalegi)) ## 7.3.0 (2023-04-03) diff --git a/README.md b/README.md index deb94da81..d610ea73e 100644 --- a/README.md +++ b/README.md @@ -503,7 +503,7 @@ class ProductsIndex < Chewy::Index field :name # simply use crutch-fetched data as a value: - field :category_names, value: ->(product, crutches) { crutches.categories[product.id] } + field :category_names, value: ->(product, crutches) { crutches[:categories][product.id] } end ``` diff --git a/lib/chewy/index/crutch.rb b/lib/chewy/index/crutch.rb index a36fc4ef8..4377187af 100644 --- a/lib/chewy/index/crutch.rb +++ b/lib/chewy/index/crutch.rb @@ -12,13 +12,21 @@ class Crutches def initialize(index, collection) @index = index @collection = collection - @index._crutches.each_key do |name| - singleton_class.class_eval <<-METHOD, __FILE__, __LINE__ + 1 - def #{name} - @#{name} ||= @index._crutches[:#{name}].call @collection - end - METHOD - end + @crutches_instances = {} + end + + def method_missing(name, *, **) + return self[name] if @index._crutches.key?(name) + + super + end + + def respond_to_missing?(name, include_private = false) + @index._crutches.key?(name) || super + end + + def [](name) + @crutches_instances[name] ||= @index._crutches[:"#{name}"].call(@collection) end end From 7fb0b50957e7a8260a32ab62ff67ff20f6057124 Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Thu, 20 Apr 2023 17:33:26 +0300 Subject: [PATCH 15/59] Prepare 7.3.1 release (#883) --- CHANGELOG.md | 4 ++++ lib/chewy/version.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1381025c8..9c1a60d56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ ### Bugs Fixed +## 7.3.1 (2023-04-20) + +### Bugs Fixed + * [#874](https://github.com/toptal/chewy/pull/874): Fix `chewy:journal:clean` task for ruby 3.x. ([@muk-ai](https://github.com/muk-ai)) * [#882](https://github.com/toptal/chewy/pull/882): Fix memory leak during `chewy:reset` for ruby 3.2 ([@konalegi](https://github.com/konalegi)) diff --git a/lib/chewy/version.rb b/lib/chewy/version.rb index b1eb15dad..a8e9d893d 100644 --- a/lib/chewy/version.rb +++ b/lib/chewy/version.rb @@ -1,3 +1,3 @@ module Chewy - VERSION = '7.3.0'.freeze + VERSION = '7.3.1'.freeze end From c20d1128d6447ff202239cd5121c67e9c70c9ed1 Mon Sep 17 00:00:00 2001 From: Martijn Lafeber Date: Thu, 20 Apr 2023 17:41:40 +0200 Subject: [PATCH 16/59] Fix in mock_elasticsearch_response_sources (#861) * Fix for incremental id in mock_elasticsearch_response_sources * Modifies test so it passes when id matches source id. * Adds changelog entry --- CHANGELOG.md | 2 ++ lib/chewy/minitest/helpers.rb | 2 +- spec/chewy/minitest/helpers_spec.rb | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c1a60d56..aff74b82b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ * [#874](https://github.com/toptal/chewy/pull/874): Fix `chewy:journal:clean` task for ruby 3.x. ([@muk-ai](https://github.com/muk-ai)) * [#882](https://github.com/toptal/chewy/pull/882): Fix memory leak during `chewy:reset` for ruby 3.2 ([@konalegi](https://github.com/konalegi)) +* [#861](https://github.com/toptal/chewy/pull/861): Fix bug in mock_elasticsearch_response_sources ([@lafeber](https://github.com/lafeber)) + ## 7.3.0 (2023-04-03) ### New Features diff --git a/lib/chewy/minitest/helpers.rb b/lib/chewy/minitest/helpers.rb index 26ff95e3f..e9fbd1362 100644 --- a/lib/chewy/minitest/helpers.rb +++ b/lib/chewy/minitest/helpers.rb @@ -97,7 +97,7 @@ def mock_elasticsearch_response_sources(index, hits, &block) { '_index' => index.index_name, '_type' => '_doc', - '_id' => (i + 1).to_s, + '_id' => hit[:id] || (i + 1).to_s, '_score' => 3.14, '_source' => hit } diff --git a/spec/chewy/minitest/helpers_spec.rb b/spec/chewy/minitest/helpers_spec.rb index 98fd0f08e..41353b928 100644 --- a/spec/chewy/minitest/helpers_spec.rb +++ b/spec/chewy/minitest/helpers_spec.rb @@ -32,14 +32,14 @@ def assert_equal(expected, actual, message) { '_index' => 'dummies', '_type' => '_doc', - '_id' => '1', + '_id' => '2', '_score' => 3.14, '_source' => source } ] end - let(:source) { {'name' => 'some_name'} } + let(:source) { {'name' => 'some_name', id: '2'} } let(:sources) { [source] } context 'mocks by raw response' do From 8fe3d0f27d9813853a4a71ac7478b440ddb56a20 Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Fri, 21 Apr 2023 10:07:34 +0300 Subject: [PATCH 17/59] Prepare 7.3.2 release (#884) --- CHANGELOG.md | 12 ++++++++++-- lib/chewy/version.rb | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aff74b82b..61c34bbc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,16 @@ ### Bugs Fixed +## 7.3.2 (2023-04-20) + +### New Features + +### Changes + +### Bugs Fixed + +* [#861](https://github.com/toptal/chewy/pull/861): Fix bug in mock_elasticsearch_response_sources ([@lafeber](https://github.com/lafeber)) + ## 7.3.1 (2023-04-20) ### Bugs Fixed @@ -15,8 +25,6 @@ * [#874](https://github.com/toptal/chewy/pull/874): Fix `chewy:journal:clean` task for ruby 3.x. ([@muk-ai](https://github.com/muk-ai)) * [#882](https://github.com/toptal/chewy/pull/882): Fix memory leak during `chewy:reset` for ruby 3.2 ([@konalegi](https://github.com/konalegi)) -* [#861](https://github.com/toptal/chewy/pull/861): Fix bug in mock_elasticsearch_response_sources ([@lafeber](https://github.com/lafeber)) - ## 7.3.0 (2023-04-03) ### New Features diff --git a/lib/chewy/version.rb b/lib/chewy/version.rb index a8e9d893d..e72574f17 100644 --- a/lib/chewy/version.rb +++ b/lib/chewy/version.rb @@ -1,3 +1,3 @@ module Chewy - VERSION = '7.3.1'.freeze + VERSION = '7.3.2'.freeze end From f77e9dc9867df8ad152f0243295bcab26c63a338 Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Wed, 5 Jul 2023 19:12:53 +0300 Subject: [PATCH 18/59] Skip journal creation on import (#888) * Skip journal creation on import * Update changelog --- CHANGELOG.md | 2 ++ lib/chewy/index/import/routine.rb | 2 +- lib/chewy/rake_helper.rb | 13 +++++++++++++ lib/tasks/chewy.rake | 5 +++++ spec/chewy/index/import_spec.rb | 13 +++++++++++++ spec/chewy/rake_helper_spec.rb | 11 +++++++++++ 6 files changed, 45 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61c34bbc6..c36ef560c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### New Features +* [#888](https://github.com/toptal/chewy/pull/888/files): Skip journal creation on import ([@konalegi](https://github.com/konalegi)) + ### Changes ### Bugs Fixed diff --git a/lib/chewy/index/import/routine.rb b/lib/chewy/index/import/routine.rb index 556510572..61004955a 100644 --- a/lib/chewy/index/import/routine.rb +++ b/lib/chewy/index/import/routine.rb @@ -64,7 +64,7 @@ def initialize(index, **options) # Creates the journal index and the corresponding index if necessary. # @return [Object] whatever def create_indexes! - Chewy::Stash::Journal.create if @options[:journal] + Chewy::Stash::Journal.create if @options[:journal] && !Chewy.configuration[:skip_journal_creation_on_import] return if Chewy.configuration[:skip_index_creation_on_import] @index.create!(**@bulk_options.slice(:suffix)) unless @index.exists? diff --git a/lib/chewy/rake_helper.rb b/lib/chewy/rake_helper.rb index 8814b4b04..d55aa3d65 100644 --- a/lib/chewy/rake_helper.rb +++ b/lib/chewy/rake_helper.rb @@ -197,6 +197,19 @@ def journal_clean(time: nil, only: nil, except: nil, delete_by_query_options: {} end end + # Creates journal index. + # + # @example + # Chewy::RakeHelper.journal_create # creates journal + # + # @param output [IO] output io for logging + # @return Chewy::Index Returns instance of chewy index + def journal_create(output: $stdout) + subscribed_task_stats(output) do + Chewy::Stash::Journal.create! + end + end + # Eager loads and returns all the indexes defined in the application # except Chewy::Stash::Specification and Chewy::Stash::Journal. # diff --git a/lib/tasks/chewy.rake b/lib/tasks/chewy.rake index f8a99560c..29e0cef6e 100644 --- a/lib/tasks/chewy.rake +++ b/lib/tasks/chewy.rake @@ -87,6 +87,11 @@ namespace :chewy do end namespace :journal do + desc 'Create manually journal, useful when `skip_journal_creation_on_import` is used' + task create: :environment do |_task, _args| + Chewy::RakeHelper.journal_create + end + desc 'Applies changes that were done after the specified time for the specified indexes/types or all of them' task apply: :environment do |_task, args| Chewy::RakeHelper.journal_apply(**parse_journal_args(args.extras)) diff --git a/spec/chewy/index/import_spec.rb b/spec/chewy/index/import_spec.rb index ddd0cd76b..abc0152f3 100644 --- a/spec/chewy/index/import_spec.rb +++ b/spec/chewy/index/import_spec.rb @@ -60,6 +60,19 @@ def subscribe_notification CitiesIndex.import(dummy_city) end end + + context 'skip journal creation on import' do + before do + Chewy::Stash::Journal.create! + Chewy.config.settings[:skip_journal_creation_on_import] = true + end + after { Chewy.config.settings[:skip_journal_creation_on_import] = nil } + + specify do + expect(Chewy::Stash::Journal).not_to receive(:create!) + CitiesIndex.import(dummy_city, journal: true) + end + end end shared_examples 'importing' do diff --git a/spec/chewy/rake_helper_spec.rb b/spec/chewy/rake_helper_spec.rb index f563b03d0..3e80e83ec 100644 --- a/spec/chewy/rake_helper_spec.rb +++ b/spec/chewy/rake_helper_spec.rb @@ -470,6 +470,17 @@ end end + describe '.journal_create' do + specify do + output = StringIO.new + described_class.journal_create(output: output) + expect(Chewy::Stash::Journal.exists?).to be_truthy + expect(output.string).to match(Regexp.new(<<-OUTPUT, Regexp::MULTILINE)) +Total: \\d+s\\Z + OUTPUT + end + end + describe '.reindex' do before do journal From 94d8e11b99cceb9611e4605e7f701b06648b7a4a Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Fri, 7 Jul 2023 14:04:41 +0300 Subject: [PATCH 19/59] Prepare 7.3.3 release (#889) --- CHANGELOG.md | 8 ++++++++ lib/chewy/version.rb | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c36ef560c..2ac304365 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ ### New Features +### Changes + +### Bugs Fixed + +## 7.3.3 (2023-07-07) + +### New Features + * [#888](https://github.com/toptal/chewy/pull/888/files): Skip journal creation on import ([@konalegi](https://github.com/konalegi)) ### Changes diff --git a/lib/chewy/version.rb b/lib/chewy/version.rb index e72574f17..27c719e01 100644 --- a/lib/chewy/version.rb +++ b/lib/chewy/version.rb @@ -1,3 +1,3 @@ module Chewy - VERSION = '7.3.2'.freeze + VERSION = '7.3.3'.freeze end From 541b6969adafec6ce8cf6899a36387d571b6815c Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Tue, 29 Aug 2023 16:29:05 +0300 Subject: [PATCH 20/59] Add task to create all indexes (#892) * Add task to create all indexes * Update changelog and readme * Address review comments * Control verbosity of the index creation --- CHANGELOG.md | 2 ++ README.md | 4 +++ lib/chewy/rake_helper.rb | 18 ++++++++++++ lib/tasks/chewy.rake | 5 ++++ spec/chewy/rake_helper_spec.rb | 53 ++++++++++++++++++++++++++++++++++ 5 files changed, 82 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ac304365..c94b36f3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### New Features +* [#888](https://github.com/toptal/chewy/pull/892): Rake task to create missing indexes ([@konalegi](https://github.com/konalegi)) + ### Changes ### Bugs Fixed diff --git a/README.md b/README.md index d610ea73e..b029c9461 100644 --- a/README.md +++ b/README.md @@ -1197,6 +1197,10 @@ Right now the approach is that if some data had been updated, but index definiti Also, there is always full reset alternative with `rake chewy:reset`. +#### `chewy:create_missing_indexes` + +This rake task creates newly defined indexes in ElasticSearch and skips existing ones. Useful for production-like environments. + #### Parallelizing rake tasks Every task described above has its own parallel version. Every parallel rake task takes the number for processes for execution as the first argument and the rest of the arguments are exactly the same as for the non-parallel task version. diff --git a/lib/chewy/rake_helper.rb b/lib/chewy/rake_helper.rb index d55aa3d65..0e5cb706d 100644 --- a/lib/chewy/rake_helper.rb +++ b/lib/chewy/rake_helper.rb @@ -268,6 +268,24 @@ def delete_by_query_options_from_env(env) end end + def create_missing_indexes!(output: $stdout, env: ENV) + subscribed_task_stats(output) do + Chewy.eager_load! + all_indexes = Chewy::Index.descendants + all_indexes -= [Chewy::Stash::Journal] unless Chewy.configuration[:journal] + all_indexes.each do |index| + if index.exists? + output.puts "#{index.name} already exists, skipping" if env['VERBOSE'] + next + end + + index.create! + + output.puts "#{index.name} index successfully created" + end + end + end + def normalize_indexes(*identifiers) identifiers.flatten(1).map { |identifier| normalize_index(identifier) } end diff --git a/lib/tasks/chewy.rake b/lib/tasks/chewy.rake index 29e0cef6e..636ffb871 100644 --- a/lib/tasks/chewy.rake +++ b/lib/tasks/chewy.rake @@ -57,6 +57,11 @@ namespace :chewy do Chewy::RakeHelper.update_mapping(name: args[:index_name]) end + desc 'Creates missing indexes' + task create_missing_indexes: :environment do + Chewy::RakeHelper.create_missing_indexes! + end + namespace :parallel do desc 'Parallel version of `rake chewy:reset`' task reset: :environment do |_task, args| diff --git a/spec/chewy/rake_helper_spec.rb b/spec/chewy/rake_helper_spec.rb index 3e80e83ec..510e85f1a 100644 --- a/spec/chewy/rake_helper_spec.rb +++ b/spec/chewy/rake_helper_spec.rb @@ -470,6 +470,59 @@ end end + describe '.create_missing_indexes!' do + before do + [CountriesIndex, Chewy::Stash::Specification].map(&:create!) + + # To avoid flaky issues when previous specs were run + expect(Chewy::Index).to receive(:descendants).and_return( + [ + UsersIndex, + CountriesIndex, + CitiesIndex, + Chewy::Stash::Specification, + Chewy::Stash::Journal + ] + ) + end + + specify do + output = StringIO.new + described_class.create_missing_indexes!(output: output) + expect(CitiesIndex.exists?).to be_truthy + expect(UsersIndex.exists?).to be_truthy + expect(Chewy::Stash::Journal.exists?).to be_falsey + expect(output.string).to match(Regexp.new(<<-OUTPUT, Regexp::MULTILINE)) +UsersIndex index successfully created +CitiesIndex index successfully created +Total: \\d+s\\Z + OUTPUT + end + + context 'when verbose' do + specify do + output = StringIO.new + described_class.create_missing_indexes!(output: output, env: {'VERBOSE' => '1'}) + expect(output.string).to match(Regexp.new(<<-OUTPUT, Regexp::MULTILINE)) +UsersIndex index successfully created +CountriesIndex already exists, skipping +CitiesIndex index successfully created +Chewy::Stash::Specification already exists, skipping +Total: \\d+s\\Z + OUTPUT + end + end + + context 'when journaling is enabled' do + before { Chewy.config.settings[:journal] = true } + after { Chewy.config.settings.delete(:journal) } + specify do + described_class.create_missing_indexes!(output: StringIO.new) + expect(Chewy::Stash::Journal.exists?).to be_truthy + end + end + end + describe '.journal_create' do specify do output = StringIO.new From e10c65531845d0b1d177ac03ea1ffb85eedce01f Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Tue, 29 Aug 2023 16:42:30 +0300 Subject: [PATCH 21/59] Bump version to 7.3.4 (#893) --- CHANGELOG.md | 8 ++++++++ lib/chewy/version.rb | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c94b36f3a..fc63274c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ ### New Features +### Changes + +### Bugs Fixed + +## 7.3.4 (2023-08-29) + +### New Features + * [#888](https://github.com/toptal/chewy/pull/892): Rake task to create missing indexes ([@konalegi](https://github.com/konalegi)) ### Changes diff --git a/lib/chewy/version.rb b/lib/chewy/version.rb index 27c719e01..0e30a437d 100644 --- a/lib/chewy/version.rb +++ b/lib/chewy/version.rb @@ -1,3 +1,3 @@ module Chewy - VERSION = '7.3.3'.freeze + VERSION = '7.3.4'.freeze end From 3eba90467f8ff7f869fb16d9568985dfc709af2f Mon Sep 17 00:00:00 2001 From: Ebeagu Samuel Date: Fri, 6 Oct 2023 12:32:25 +0100 Subject: [PATCH 22/59] Add dependabot configuation (#896) --- .github/dependabot.yml | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..a675b1060 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,42 @@ +version: 2 +registries: + toptal-github: + type: "git" + url: "https://github.com" + username: "x-access-token" + password: "${{secrets.DEPENDABOT_GITHUB_TOKEN}}" + +updates: + - package-ecosystem: bundler + directory: "/" + schedule: + interval: "weekly" + day: "wednesday" + time: "07:00" + pull-request-branch-name: + separator: "-" + labels: + - "no-jira" + - "ruby" + - "dependencies" + reviewers: + - "toptal/devx" + registries: + - toptal-github + insecure-external-code-execution: allow + open-pull-requests-limit: 3 + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "wednesday" + time: "07:00" + pull-request-branch-name: + separator: "-" + labels: + - "no-jira" + - "dependencies" + - "gha" + reviewers: + - "toptal/devx" + open-pull-requests-limit: 3 From 22ead9a21b0908de856e0452dc7273db8c6725de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Oct 2023 00:05:28 +0200 Subject: [PATCH 23/59] Bump actions/checkout from 2 to 4 (#897) Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ruby.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index fd6b02b44..636110841 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -24,7 +24,7 @@ jobs: BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby }} @@ -50,7 +50,7 @@ jobs: BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby }} @@ -66,7 +66,7 @@ jobs: rubocop: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: ruby-version: 2.7 From 317dd938dd15fb84a764b66983852012f7651001 Mon Sep 17 00:00:00 2001 From: Alejandro Perea Date: Wed, 6 Dec 2023 18:05:54 +0100 Subject: [PATCH 24/59] [Fix #903] Fix deprecation warning in LogSubscriber when updating to Rails 7.1 (#907) --- .github/workflows/ruby.yml | 2 +- CHANGELOG.md | 2 ++ gemfiles/rails.7.1.activerecord.gemfile | 13 +++++++++++++ lib/chewy.rb | 1 + lib/chewy/log_subscriber.rb | 6 +++++- 5 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 gemfiles/rails.7.1.activerecord.gemfile diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 636110841..9c518a8fa 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -43,7 +43,7 @@ jobs: fail-fast: false matrix: ruby: [ '3.0', '3.1', '3.2' ] - gemfile: [ rails.6.1.activerecord, rails.7.0.activerecord ] + gemfile: [ rails.6.1.activerecord, rails.7.0.activerecord, rails.7.1.activerecord ] name: ${{ matrix.ruby }}-${{ matrix.gemfile }} env: diff --git a/CHANGELOG.md b/CHANGELOG.md index fc63274c0..5dc336724 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### New Features +* [#907](https://github.com/toptal/chewy/pull/907): Fix deprecation warning in LogSubscriber for Rails 7.1 ([@alejandroperea](https://github.com/alejandroperea)) + ### Changes ### Bugs Fixed diff --git a/gemfiles/rails.7.1.activerecord.gemfile b/gemfiles/rails.7.1.activerecord.gemfile new file mode 100644 index 000000000..af8ec812e --- /dev/null +++ b/gemfiles/rails.7.1.activerecord.gemfile @@ -0,0 +1,13 @@ +source 'https://rubygems.org' + +gem 'activejob', '~> 7.1.0' +gem 'activerecord', '~> 7.1.0' +gem 'activesupport', '~> 7.1.0' +gem 'kaminari-core', '~> 1.1.0', require: false +gem 'parallel', require: false +gem 'rspec_junit_formatter', '~> 0.4.1' +gem 'sidekiq', require: false + +gem 'rexml' if RUBY_VERSION >= '3.0.0' + +gemspec path: '../' diff --git a/lib/chewy.rb b/lib/chewy.rb index a6239be31..e95f25c75 100644 --- a/lib/chewy.rb +++ b/lib/chewy.rb @@ -1,3 +1,4 @@ +require 'active_support' require 'active_support/version' require 'active_support/concern' require 'active_support/deprecation' diff --git a/lib/chewy/log_subscriber.rb b/lib/chewy/log_subscriber.rb index c35d63fbf..368c0813b 100644 --- a/lib/chewy/log_subscriber.rb +++ b/lib/chewy/log_subscriber.rb @@ -24,7 +24,11 @@ def render_action(action, event) subject = payload[:type].presence || payload[:index] action = "#{subject} #{action} (#{event.duration.round(1)}ms)" - action = color(action, GREEN, true) + action = if ActiveSupport.version >= Gem::Version.new('7.1') + color(action, GREEN, bold: true) + else + color(action, GREEN, true) + end debug(" #{action} #{description}") end From 1a5594fdc976574b78fd1b7fc155aa86bfaa1d52 Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Wed, 6 Dec 2023 20:23:32 +0300 Subject: [PATCH 25/59] Bump version to 7.3.5 (#908) --- CHANGELOG.md | 8 ++++++++ lib/chewy/version.rb | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dc336724..45d26f7a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ ### New Features +### Changes + +### Bugs Fixed + +## 7.3.5 (2023-12-06) + +### New Features + * [#907](https://github.com/toptal/chewy/pull/907): Fix deprecation warning in LogSubscriber for Rails 7.1 ([@alejandroperea](https://github.com/alejandroperea)) ### Changes diff --git a/lib/chewy/version.rb b/lib/chewy/version.rb index 0e30a437d..14d2dc97b 100644 --- a/lib/chewy/version.rb +++ b/lib/chewy/version.rb @@ -1,3 +1,3 @@ module Chewy - VERSION = '7.3.4'.freeze + VERSION = '7.3.5'.freeze end From a3f50264274891021a1803b1ce599198b7f208e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juraj=20Kostolansk=C3=BD?= Date: Tue, 12 Dec 2023 12:29:10 +0100 Subject: [PATCH 26/59] Add KNN (#890) * Add KNN * Fix PR number in Changelog * Fix test for collapse and knn --- CHANGELOG.md | 2 ++ lib/chewy/search/parameters/knn.rb | 16 ++++++++++++++++ lib/chewy/search/request.rb | 17 ++++++++++++++--- spec/chewy/search/parameters/knn_spec.rb | 5 +++++ spec/chewy/search/request_spec.rb | 16 +++++++++------- 5 files changed, 46 insertions(+), 10 deletions(-) create mode 100644 lib/chewy/search/parameters/knn.rb create mode 100644 spec/chewy/search/parameters/knn_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 45d26f7a0..85bf21866 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### New Features +* [#890](https://github.com/toptal/chewy/pull/890): Add the [`knn`](https://www.elastic.co/guide/en/elasticsearch/reference/current/knn-search.html) option to the request. ([@jkostolansky][]) + ### Changes ### Bugs Fixed diff --git a/lib/chewy/search/parameters/knn.rb b/lib/chewy/search/parameters/knn.rb new file mode 100644 index 000000000..fa3de772b --- /dev/null +++ b/lib/chewy/search/parameters/knn.rb @@ -0,0 +1,16 @@ +require 'chewy/search/parameters/storage' + +module Chewy + module Search + class Parameters + # Just a standard hash storage. Nothing to see here. + # + # @see Chewy::Search::Parameters::HashStorage + # @see Chewy::Search::Request#knn + # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/knn-search.html + class Knn < Storage + include HashStorage + end + end + end +end diff --git a/lib/chewy/search/request.rb b/lib/chewy/search/request.rb index 6c7a62156..f3b1031b4 100644 --- a/lib/chewy/search/request.rb +++ b/lib/chewy/search/request.rb @@ -20,7 +20,7 @@ class Request UNDEFINED = Class.new.freeze EVERFIELDS = %w[_index _type _id _parent _routing].freeze DELEGATED_METHODS = %i[ - query filter post_filter order reorder docvalue_fields + query filter post_filter knn order reorder docvalue_fields track_scores track_total_hits request_cache explain version profile search_type preference limit offset terminate_after timeout min_score source stored_fields search_after @@ -41,7 +41,7 @@ class Request EXTRA_STORAGES = %i[aggs suggest].freeze # An array of storage names that are changing the returned hist collection in any way. WHERE_STORAGES = %i[ - query filter post_filter none min_score rescore indices_boost collapse + query filter post_filter knn none min_score rescore indices_boost collapse ].freeze delegate :hits, :wrappers, :objects, :records, :documents, @@ -520,7 +520,18 @@ def reorder(value, *values) # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/collapse-search-results.html # @param value [Hash] # @return [Chewy::Search::Request] - %i[request_cache search_type preference timeout limit offset terminate_after min_score ignore_unavailable collapse].each do |name| + # + # @!method knn(value) + # Replaces the value of the `knn` request part. + # + # @example + # PlacesIndex.knn(field: :vector, query_vector: [4, 2], k: 5, num_candidates: 50) + # # => {:knn=>{"field"=>:vector, "query_vector"=>[4, 2], "k"=>5, "num_candidates"=>50}}}> + # @see Chewy::Search::Parameters::Knn + # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/knn-search.html + # @param value [Hash] + # @return [Chewy::Search::Request] + %i[request_cache search_type preference timeout limit offset terminate_after min_score ignore_unavailable collapse knn].each do |name| define_method name do |value| modify(name) { replace!(value) } end diff --git a/spec/chewy/search/parameters/knn_spec.rb b/spec/chewy/search/parameters/knn_spec.rb new file mode 100644 index 000000000..cc4f45f70 --- /dev/null +++ b/spec/chewy/search/parameters/knn_spec.rb @@ -0,0 +1,5 @@ +require 'chewy/search/parameters/hash_storage_examples' + +describe Chewy::Search::Parameters::Knn do + it_behaves_like :hash_storage, :knn +end diff --git a/spec/chewy/search/request_spec.rb b/spec/chewy/search/request_spec.rb index 4aede10f3..414de49a7 100644 --- a/spec/chewy/search/request_spec.rb +++ b/spec/chewy/search/request_spec.rb @@ -314,14 +314,16 @@ end end - describe '#collapse' do - specify { expect(subject.collapse(foo: {bar: 42}).render[:body]).to include(collapse: {'foo' => {bar: 42}}) } - specify do - expect(subject.collapse(foo: {bar: 42}).collapse(moo: {baz: 43}).render[:body]) - .to include(collapse: {'moo' => {baz: 43}}) + %i[collapse knn].each do |name| + describe "##{name}" do + specify { expect(subject.send(name, foo: {bar: 42}).render[:body]).to include(name => {'foo' => {bar: 42}}) } + specify do + expect(subject.send(name, foo: {bar: 42}).send(name, moo: {baz: 43}).render[:body]) + .to include(name => {'moo' => {baz: 43}}) + end + specify { expect(subject.send(name, foo: {bar: 42}).send(name, nil).render[:body]).to be_blank } + specify { expect { subject.send(name, foo: {bar: 42}) }.not_to change { subject.render } } end - specify { expect(subject.collapse(foo: {bar: 42}).collapse(nil).render[:body]).to be_blank } - specify { expect { subject.collapse(foo: {bar: 42}) }.not_to change { subject.render } } end describe '#docvalue_fields' do From 6addcb31f086a9e4da4167ca3e91a2a6dfd9848e Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Wed, 13 Dec 2023 18:12:41 +0300 Subject: [PATCH 27/59] Bump version and update changelog (#913) --- CHANGELOG.md | 8 ++++++++ lib/chewy/version.rb | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85bf21866..5cfceae25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ ### New Features +### Changes + +### Bugs Fixed + +## 7.3.5 (2023-12-13) + +### New Features + * [#890](https://github.com/toptal/chewy/pull/890): Add the [`knn`](https://www.elastic.co/guide/en/elasticsearch/reference/current/knn-search.html) option to the request. ([@jkostolansky][]) ### Changes diff --git a/lib/chewy/version.rb b/lib/chewy/version.rb index 14d2dc97b..b22c523e3 100644 --- a/lib/chewy/version.rb +++ b/lib/chewy/version.rb @@ -1,3 +1,3 @@ module Chewy - VERSION = '7.3.5'.freeze + VERSION = '7.3.6'.freeze end From f9681e45542dc23ad0c011395df518ba73737d85 Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Wed, 13 Dec 2023 18:37:52 +0300 Subject: [PATCH 28/59] Remove ruby 2.x (#911) * Remove ruby 2.x * Fix rubocop --- .github/workflows/ruby.yml | 30 ++----------------- .rubocop.yml | 2 +- CHANGELOG.md | 1 + Gemfile | 8 ++--- README.md | 2 +- chewy.gemspec | 20 ++----------- gemfiles/base.gemfile | 12 ++++++++ gemfiles/rails.5.2.activerecord.gemfile | 11 ------- gemfiles/rails.6.0.activerecord.gemfile | 11 ------- gemfiles/rails.6.1.activerecord.gemfile | 3 +- gemfiles/rails.7.0.activerecord.gemfile | 3 +- gemfiles/rails.7.1.activerecord.gemfile | 3 +- lib/chewy.rb | 2 +- lib/chewy/config.rb | 22 +++++++------- lib/chewy/fields/root.rb | 2 +- lib/chewy/index/adapter/active_record.rb | 2 +- lib/chewy/index/adapter/object.rb | 6 ++-- lib/chewy/index/adapter/orm.rb | 4 +-- lib/chewy/index/import.rb | 4 +-- lib/chewy/index/import/bulk_builder.rb | 4 +-- .../index/observe/active_record_methods.rb | 2 +- lib/chewy/index/syncer.rb | 2 +- lib/chewy/minitest/search_index_receiver.rb | 4 ++- lib/chewy/rake_helper.rb | 6 ++-- lib/chewy/rspec/update_index.rb | 6 ++-- lib/chewy/runtime/version.rb | 2 +- lib/chewy/search.rb | 10 ++++--- lib/chewy/search/parameters.rb | 6 ++-- lib/chewy/search/parameters/indices.rb | 2 +- lib/chewy/search/parameters/storage.rb | 2 +- spec/chewy/index/actions_spec.rb | 8 ++--- spec/chewy/index/import/bulk_builder_spec.rb | 6 ++-- spec/chewy/index/import_spec.rb | 6 ++-- spec/chewy/minitest/helpers_spec.rb | 2 +- .../minitest/search_index_receiver_spec.rb | 10 ++++--- spec/chewy/rake_helper_spec.rb | 8 ++--- spec/chewy/rspec/helpers_spec.rb | 2 +- .../search/pagination/kaminari_examples.rb | 2 +- spec/chewy/search/pagination/kaminari_spec.rb | 2 +- spec/chewy/strategy/active_job_spec.rb | 16 +++++----- spec/chewy/strategy/lazy_sidekiq_spec.rb | 20 ++++++------- spec/chewy/strategy/sidekiq_spec.rb | 8 ++--- spec/chewy_spec.rb | 2 +- spec/spec_helper.rb | 2 +- 44 files changed, 125 insertions(+), 163 deletions(-) create mode 100644 gemfiles/base.gemfile delete mode 100644 gemfiles/rails.5.2.activerecord.gemfile delete mode 100644 gemfiles/rails.6.0.activerecord.gemfile diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 9c518a8fa..979dadf97 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -11,39 +11,13 @@ on: ] jobs: - ruby-2: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - ruby: [2.6, 2.7] - gemfile: [rails.5.2.activerecord, rails.6.0.activerecord, rails.6.1.activerecord] - name: ${{ matrix.ruby }}-${{ matrix.gemfile }} - - env: - BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile - - steps: - - uses: actions/checkout@v4 - - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby }} - bundler-cache: true - - name: Run Elasticsearch - uses: elastic/elastic-github-actions/elasticsearch@9de0f78f306e4ebc0838f057e6b754364685e759 - with: - stack-version: 7.10.1 - port: 9250 - - name: Tests - run: bundle exec rspec - ruby-3: runs-on: ubuntu-latest strategy: fail-fast: false matrix: ruby: [ '3.0', '3.1', '3.2' ] - gemfile: [ rails.6.1.activerecord, rails.7.0.activerecord, rails.7.1.activerecord ] + gemfile: [rails.6.1.activerecord, rails.7.0.activerecord, rails.7.1.activerecord] name: ${{ matrix.ruby }}-${{ matrix.gemfile }} env: @@ -69,6 +43,6 @@ jobs: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: - ruby-version: 2.7 + ruby-version: 3.0 bundler-cache: true - run: bundle exec rubocop --format simple diff --git a/.rubocop.yml b/.rubocop.yml index ec0426c0e..e6c827173 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -2,7 +2,7 @@ inherit_from: .rubocop_todo.yml AllCops: NewCops: enable - TargetRubyVersion: 2.6 + TargetRubyVersion: 3.0 Layout/AccessModifierIndentation: EnforcedStyle: outdent diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cfceae25..fd938af9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ ### New Features * [#890](https://github.com/toptal/chewy/pull/890): Add the [`knn`](https://www.elastic.co/guide/en/elasticsearch/reference/current/knn-search.html) option to the request. ([@jkostolansky][]) +* [#911](https://github.com/toptal/chewy/pull/911): Remove ruby 2.x. ([@konalegi][]) ### Changes diff --git a/Gemfile b/Gemfile index 969f51ad2..e838bbb9d 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,5 @@ source 'https://rubygems.org' -gemspec - gem 'activerecord' gem 'activejob', require: false @@ -18,5 +16,7 @@ gem 'guard-rspec' gem 'redcarpet' gem 'yard' -gem 'rexml' if RUBY_VERSION >= '3.0.0' -gem 'ruby2_keywords' if RUBY_VERSION < '2.7' +gem 'rexml' + +eval_gemfile 'gemfiles/base.gemfile' +gemspec diff --git a/README.md b/README.md index b029c9461..bc08c32a8 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Or install it yourself as: ### Ruby -Chewy is compatible with MRI 2.6-3.0¹. +Chewy is compatible with MRI 3.0-3.2¹. > ¹ Ruby 3 is only supported with Rails 6.1 diff --git a/chewy.gemspec b/chewy.gemspec index 0e542ab56..866ed3c36 100644 --- a/chewy.gemspec +++ b/chewy.gemspec @@ -2,7 +2,7 @@ lib = File.expand_path('lib', __dir__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) require 'chewy/version' -Gem::Specification.new do |spec| # rubocop:disable Metrics/BlockLength +Gem::Specification.new do |spec| spec.name = 'chewy' spec.version = Chewy::VERSION spec.authors = ['Toptal, LLC', 'pyromaniac'] @@ -14,24 +14,10 @@ Gem::Specification.new do |spec| # rubocop:disable Metrics/BlockLength spec.files = `git ls-files`.split($RS) spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } - spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) spec.require_paths = ['lib'] - spec.add_development_dependency 'database_cleaner' - spec.add_development_dependency 'elasticsearch-extensions' - spec.add_development_dependency 'mock_redis' - spec.add_development_dependency 'rake' - spec.add_development_dependency 'rspec', '>= 3.7.0' - spec.add_development_dependency 'rspec-collection_matchers' - spec.add_development_dependency 'rspec-its' - spec.add_development_dependency 'rubocop', '1.11' - spec.add_development_dependency 'sqlite3' - spec.add_development_dependency 'timecop' - - spec.add_development_dependency 'method_source' - spec.add_development_dependency 'unparser' - - spec.add_dependency 'activesupport', '>= 5.2' + spec.add_dependency 'activesupport', '>= 5.2' # Remove with major version bump, 8.x spec.add_dependency 'elasticsearch', '>= 7.12.0', '< 7.14.0' spec.add_dependency 'elasticsearch-dsl' + spec.metadata['rubygems_mfa_required'] = 'true' end diff --git a/gemfiles/base.gemfile b/gemfiles/base.gemfile new file mode 100644 index 000000000..656b9395e --- /dev/null +++ b/gemfiles/base.gemfile @@ -0,0 +1,12 @@ +gem 'database_cleaner' +gem 'elasticsearch-extensions' +gem 'method_source' +gem 'mock_redis' +gem 'rake' +gem 'rspec', '>= 3.7.0' +gem 'rspec-collection_matchers' +gem 'rspec-its' +gem 'rubocop', '1.48' +gem 'sqlite3' +gem 'timecop' +gem 'unparser' diff --git a/gemfiles/rails.5.2.activerecord.gemfile b/gemfiles/rails.5.2.activerecord.gemfile deleted file mode 100644 index 5838db590..000000000 --- a/gemfiles/rails.5.2.activerecord.gemfile +++ /dev/null @@ -1,11 +0,0 @@ -source 'https://rubygems.org' - -gem 'activejob', '~> 5.2.0' -gem 'activerecord', '~> 5.2.0' -gem 'activesupport', '~> 5.2.0' -gem 'kaminari-core', '~> 1.1.0', require: false -gem 'parallel', require: false -gem 'rspec_junit_formatter', '~> 0.4.1' -gem 'sidekiq', require: false - -gemspec path: '../' diff --git a/gemfiles/rails.6.0.activerecord.gemfile b/gemfiles/rails.6.0.activerecord.gemfile deleted file mode 100644 index 1f0696be9..000000000 --- a/gemfiles/rails.6.0.activerecord.gemfile +++ /dev/null @@ -1,11 +0,0 @@ -source 'https://rubygems.org' - -gem 'activejob', '~> 6.0.0' -gem 'activerecord', '~> 6.0.0' -gem 'activesupport', '~> 6.0.0' -gem 'kaminari-core', '~> 1.1.0', require: false -gem 'parallel', require: false -gem 'rspec_junit_formatter', '~> 0.4.1' -gem 'sidekiq', require: false - -gemspec path: '../' diff --git a/gemfiles/rails.6.1.activerecord.gemfile b/gemfiles/rails.6.1.activerecord.gemfile index cdc7b82cc..526db972f 100644 --- a/gemfiles/rails.6.1.activerecord.gemfile +++ b/gemfiles/rails.6.1.activerecord.gemfile @@ -8,6 +8,7 @@ gem 'parallel', require: false gem 'rspec_junit_formatter', '~> 0.4.1' gem 'sidekiq', require: false -gem 'rexml' if RUBY_VERSION >= '3.0.0' +gem 'rexml' gemspec path: '../' +eval_gemfile 'base.gemfile' diff --git a/gemfiles/rails.7.0.activerecord.gemfile b/gemfiles/rails.7.0.activerecord.gemfile index e90b18cab..1176622ac 100644 --- a/gemfiles/rails.7.0.activerecord.gemfile +++ b/gemfiles/rails.7.0.activerecord.gemfile @@ -8,6 +8,7 @@ gem 'parallel', require: false gem 'rspec_junit_formatter', '~> 0.4.1' gem 'sidekiq', require: false -gem 'rexml' if RUBY_VERSION >= '3.0.0' +gem 'rexml' gemspec path: '../' +eval_gemfile 'base.gemfile' diff --git a/gemfiles/rails.7.1.activerecord.gemfile b/gemfiles/rails.7.1.activerecord.gemfile index af8ec812e..eb39c6ee2 100644 --- a/gemfiles/rails.7.1.activerecord.gemfile +++ b/gemfiles/rails.7.1.activerecord.gemfile @@ -8,6 +8,7 @@ gem 'parallel', require: false gem 'rspec_junit_formatter', '~> 0.4.1' gem 'sidekiq', require: false -gem 'rexml' if RUBY_VERSION >= '3.0.0' +gem 'rexml' gemspec path: '../' +eval_gemfile 'base.gemfile' diff --git a/lib/chewy.rb b/lib/chewy.rb index e95f25c75..cac447724 100644 --- a/lib/chewy.rb +++ b/lib/chewy.rb @@ -48,7 +48,7 @@ def try_require(path) require 'chewy/fields/base' require 'chewy/fields/root' require 'chewy/journal' -require 'chewy/railtie' if defined?(::Rails::Railtie) +require 'chewy/railtie' if defined?(Rails::Railtie) ActiveSupport.on_load(:active_record) do include Chewy::Index::Observe::ActiveRecordMethods diff --git a/lib/chewy/config.rb b/lib/chewy/config.rb index 29f71749b..4d7a695f7 100644 --- a/lib/chewy/config.rb +++ b/lib/chewy/config.rb @@ -127,17 +127,19 @@ def configuration private def yaml_settings - @yaml_settings ||= begin - if defined?(Rails::VERSION) - file = Rails.root.join('config', 'chewy.yml') + @yaml_settings ||= build_yaml_settings || {} + end - if File.exist?(file) - yaml = ERB.new(File.read(file)).result - hash = YAML.respond_to?(:unsafe_load) ? YAML.unsafe_load(yaml) : YAML.load(yaml) # rubocop:disable Security/YAMLLoad - hash[Rails.env].try(:deep_symbolize_keys) if hash - end - end || {} - end + def build_yaml_settings + return unless defined?(Rails::VERSION) + + file = Rails.root.join('config', 'chewy.yml') + + return unless File.exist?(file) + + yaml = ERB.new(File.read(file)).result + hash = YAML.unsafe_load(yaml) + hash[Rails.env].try(:deep_symbolize_keys) if hash end def build_search_class(base) diff --git a/lib/chewy/fields/root.rb b/lib/chewy/fields/root.rb index c37044bf3..d6e125ba8 100644 --- a/lib/chewy/fields/root.rb +++ b/lib/chewy/fields/root.rb @@ -27,7 +27,7 @@ def mappings_hash mappings[name] end - ruby2_keywords def dynamic_template(*args) + def dynamic_template(*args) options = args.extract_options!.deep_symbolize_keys if args.first template_name = :"template_#{dynamic_templates.count.next}" diff --git a/lib/chewy/index/adapter/active_record.rb b/lib/chewy/index/adapter/active_record.rb index b69e11b9a..05b029aa7 100644 --- a/lib/chewy/index/adapter/active_record.rb +++ b/lib/chewy/index/adapter/active_record.rb @@ -6,7 +6,7 @@ module Adapter class ActiveRecord < Orm def self.accepts?(target) defined?(::ActiveRecord::Base) && ( - target.is_a?(Class) && target < ::ActiveRecord::Base || + (target.is_a?(Class) && target < ::ActiveRecord::Base) || target.is_a?(::ActiveRecord::Relation)) end diff --git a/lib/chewy/index/adapter/object.rb b/lib/chewy/index/adapter/object.rb index f8df7995c..156a10232 100644 --- a/lib/chewy/index/adapter/object.rb +++ b/lib/chewy/index/adapter/object.rb @@ -85,7 +85,7 @@ def identify(collection) # @param args [Array<#to_json>] # @option options [Integer] :batch_size import processing batch size # @return [true, false] - ruby2_keywords def import(*args, &block) + def import(*args, &block) collection, options = import_args(*args) import_objects(collection, options, &block) end @@ -113,7 +113,7 @@ def identify(collection) # end # # @see Chewy::Index::Adapter::Base#import_fields - ruby2_keywords def import_fields(*args, &block) + def import_fields(*args, &block) return enum_for(:import_fields, *args) unless block_given? options = args.extract_options! @@ -139,7 +139,7 @@ def identify(collection) # For the Object adapter returns the objects themselves in batches. # # @see Chewy::Index::Adapter::Base#import_references - ruby2_keywords def import_references(*args, &block) + def import_references(*args, &block) return enum_for(:import_references, *args) unless block_given? collection, options = import_args(*args) diff --git a/lib/chewy/index/adapter/orm.rb b/lib/chewy/index/adapter/orm.rb index 57e53e82b..c86b6d06b 100644 --- a/lib/chewy/index/adapter/orm.rb +++ b/lib/chewy/index/adapter/orm.rb @@ -72,7 +72,7 @@ def identify(collection) # # or # UsersIndex.import users.map(&:id) # user ids will be deleted from index # - ruby2_keywords def import(*args, &block) + def import(*args, &block) collection, options = import_args(*args) if !collection.is_a?(relation_class) || options[:direct_import] @@ -82,7 +82,7 @@ def identify(collection) end end - ruby2_keywords def import_fields(*args, &block) + def import_fields(*args, &block) return enum_for(:import_fields, *args) unless block_given? collection, options = import_args(*args) diff --git a/lib/chewy/index/import.rb b/lib/chewy/index/import.rb index cc50a4056..8de6a877a 100644 --- a/lib/chewy/index/import.rb +++ b/lib/chewy/index/import.rb @@ -72,7 +72,7 @@ module ClassMethods # @option options [true, false] update_failover enables full objects reimport in cases of partial update errors, `true` by default # @option options [true, Integer, Hash] parallel enables parallel import processing with the Parallel gem, accepts the number of workers or any Parallel gem acceptable options # @return [true, false] false in case of errors - ruby2_keywords def import(*args) + def import(*args) intercept_import_using_strategy(*args).blank? end @@ -83,7 +83,7 @@ module ClassMethods # in case of any import errors. # # @raise [Chewy::ImportFailed] in case of errors - ruby2_keywords def import!(*args) + def import!(*args) errors = intercept_import_using_strategy(*args) raise Chewy::ImportFailed.new(self, errors) if errors.present? diff --git a/lib/chewy/index/import/bulk_builder.rb b/lib/chewy/index/import/bulk_builder.rb index 60917b511..52e3fdcfd 100644 --- a/lib/chewy/index/import/bulk_builder.rb +++ b/lib/chewy/index/import/bulk_builder.rb @@ -162,12 +162,12 @@ def load_cache .filter(ids: {values: ids_for_cache}) .order('_doc') .pluck(:_id, :_routing, join_field) - .map do |id, routing, join| + .to_h do |id, routing, join| [ id, {routing: routing, parent_id: join['parent']} ] - end.to_h + end end def existing_routing(id) diff --git a/lib/chewy/index/observe/active_record_methods.rb b/lib/chewy/index/observe/active_record_methods.rb index ca5834efc..7f4897705 100644 --- a/lib/chewy/index/observe/active_record_methods.rb +++ b/lib/chewy/index/observe/active_record_methods.rb @@ -71,7 +71,7 @@ def initialize_chewy_callbacks end end - ruby2_keywords def update_index(type_name, *args, &block) + def update_index(type_name, *args, &block) callback_options = Observe.extract_callback_options!(args) update_proc = Observe.update_proc(type_name, *args, &block) callback = Chewy::Index::Observe::Callback.new(update_proc, callback_options) diff --git a/lib/chewy/index/syncer.rb b/lib/chewy/index/syncer.rb index 4016989da..b8365bb50 100644 --- a/lib/chewy/index/syncer.rb +++ b/lib/chewy/index/syncer.rb @@ -27,7 +27,7 @@ class Index # @see Chewy::Index::Actions::ClassMethods#sync class Syncer DEFAULT_SYNC_BATCH_SIZE = 20_000 - ISO_DATETIME = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?\z/.freeze + ISO_DATETIME = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?\z/ OUTDATED_IDS_WORKER = lambda do |outdated_sync_field_type, source_data_hash, index, total, index_data| ::Process.setproctitle("chewy [#{index}]: sync outdated calculation (#{::Parallel.worker_number + 1}/#{total})") if index index_data.each_with_object([]) do |(id, index_sync_value), result| diff --git a/lib/chewy/minitest/search_index_receiver.rb b/lib/chewy/minitest/search_index_receiver.rb index d8f0bd4f8..69c95b55e 100644 --- a/lib/chewy/minitest/search_index_receiver.rb +++ b/lib/chewy/minitest/search_index_receiver.rb @@ -6,6 +6,8 @@ # The class will capture the data from the *param on the Chewy::Index.bulk method and # aggregate the data for test analysis. class SearchIndexReceiver + MUTATION_FOR_CLASS = Struct.new(:indexes, :deletes, keyword_init: true) + def initialize @mutations = {} end @@ -71,6 +73,6 @@ def updated_indexes # @param index [Chewy::Index] the index to fetch. # @return [#indexes, #deletes] an object with a list of indexes and a list of deletes. def mutation_for(index) - @mutations[index] ||= OpenStruct.new(indexes: [], deletes: []) + @mutations[index] ||= MUTATION_FOR_CLASS.new(indexes: [], deletes: []) end end diff --git a/lib/chewy/rake_helper.rb b/lib/chewy/rake_helper.rb index 0e5cb706d..a59fa02a3 100644 --- a/lib/chewy/rake_helper.rb +++ b/lib/chewy/rake_helper.rb @@ -347,9 +347,9 @@ def warn_missing_index(output) return if journal_exists? output.puts "############################################################\n" \ - "WARN: You are risking to lose some changes during the reset.\n" \ - " Please consider enabling journaling.\n" \ - " See https://github.com/toptal/chewy#journaling\n" \ + "WARN: You are risking to lose some changes during the reset.\n " \ + "Please consider enabling journaling.\n " \ + "See https://github.com/toptal/chewy#journaling\n" \ '############################################################' end diff --git a/lib/chewy/rspec/update_index.rb b/lib/chewy/rspec/update_index.rb index bd6b2a3f2..3a16ece2a 100644 --- a/lib/chewy/rspec/update_index.rb +++ b/lib/chewy/rspec/update_index.rb @@ -108,7 +108,7 @@ def supports_block_expectations? params_matcher = @no_refresh ? has_entry(refresh: false) : any_parameters Chewy::Index::Import::BulkRequest.stubs(:new).with(index, params_matcher).returns(mock_bulk_request) else - mocked_already = ::RSpec::Mocks.space.proxy_for(Chewy::Index::Import::BulkRequest).method_double_if_exists_for_message(:new) + mocked_already = RSpec::Mocks.space.proxy_for(Chewy::Index::Import::BulkRequest).method_double_if_exists_for_message(:new) allow(Chewy::Index::Import::BulkRequest).to receive(:new).and_call_original unless mocked_already params_matcher = @no_refresh ? hash_including(refresh: false) : any_args allow(Chewy::Index::Import::BulkRequest).to receive(:new).with(index, params_matcher).and_return(mock_bulk_request) @@ -220,7 +220,7 @@ def extract_documents(*args) expected_count = options[:times] || options[:count] expected_attributes = (options[:with] || options[:attributes] || {}).deep_symbolize_keys - args.flatten.map do |document| + args.flatten.to_h do |document| id = document.respond_to?(:id) ? document.id.to_s : document.to_s [id, { document: document, @@ -229,7 +229,7 @@ def extract_documents(*args) real_count: 0, real_attributes: {} }] - end.to_h + end end def compare_attributes(expected, real) diff --git a/lib/chewy/runtime/version.rb b/lib/chewy/runtime/version.rb index 1154ec0dc..a96880544 100644 --- a/lib/chewy/runtime/version.rb +++ b/lib/chewy/runtime/version.rb @@ -5,7 +5,7 @@ class Version attr_reader :major, :minor, :patch def initialize(version) - @major, @minor, @patch = *(version.to_s.split('.', 3) + [0] * 3).first(3).map(&:to_i) + @major, @minor, @patch = *(version.to_s.split('.', 3) + ([0] * 3)).first(3).map(&:to_i) end def to_s diff --git a/lib/chewy/search.rb b/lib/chewy/search.rb index 78dc59a1a..be2ba29a3 100644 --- a/lib/chewy/search.rb +++ b/lib/chewy/search.rb @@ -56,7 +56,7 @@ def search_string(query, options = {}) # # @example # PlacesIndex.query(match: {name: 'Moscow'}) - ruby2_keywords def method_missing(name, *args, &block) + def method_missing(name, *args, &block) if search_class::DELEGATED_METHODS.include?(name) all.send(name, *args, &block) else @@ -84,10 +84,12 @@ def build_search_class(base) def delegate_scoped(source, destination, methods) methods.each do |method| destination.class_eval do - define_method method do |*args, &block| - scoping { source.public_send(method, *args, &block) } + define_method method do |*args, **kwargs, &block| + scoping do + source.public_send(method, *args, **kwargs, &block) + end end - ruby2_keywords method + method end end end diff --git a/lib/chewy/search/parameters.rb b/lib/chewy/search/parameters.rb index 1f15e6168..3f55e0384 100644 --- a/lib/chewy/search/parameters.rb +++ b/lib/chewy/search/parameters.rb @@ -1,5 +1,5 @@ -Dir.glob(File.join(File.dirname(__FILE__), 'parameters', 'concerns', '*.rb')).sort.each { |f| require f } -Dir.glob(File.join(File.dirname(__FILE__), 'parameters', '*.rb')).sort.each { |f| require f } +Dir.glob(File.join(File.dirname(__FILE__), 'parameters', 'concerns', '*.rb')).each { |f| require f } +Dir.glob(File.join(File.dirname(__FILE__), 'parameters', '*.rb')).each { |f| require f } module Chewy module Search @@ -53,7 +53,7 @@ def initialize(initial = {}, **kinitial) # @param other [Object] any object # @return [true, false] def ==(other) - super || other.is_a?(self.class) && compare_storages(other) + super || (other.is_a?(self.class) && compare_storages(other)) end # Clones the specified storage, performs the operation diff --git a/lib/chewy/search/parameters/indices.rb b/lib/chewy/search/parameters/indices.rb index dc3dc28ca..d3a59b292 100644 --- a/lib/chewy/search/parameters/indices.rb +++ b/lib/chewy/search/parameters/indices.rb @@ -17,7 +17,7 @@ class Indices < Storage # @param other [Chewy::Search::Parameters::Storage] any storage instance # @return [true, false] the result of comparison def ==(other) - super || other.class == self.class && other.render == render + super || (other.class == self.class && other.render == render) end # Just adds indices to indices. diff --git a/lib/chewy/search/parameters/storage.rb b/lib/chewy/search/parameters/storage.rb index 6fe00d601..e0effe4c3 100644 --- a/lib/chewy/search/parameters/storage.rb +++ b/lib/chewy/search/parameters/storage.rb @@ -35,7 +35,7 @@ def initialize(value = nil) # @param other [Chewy::Search::Parameters::Storage] any storage instance # @return [true, false] the result of comparision def ==(other) - super || other.class == self.class && other.value == value + super || (other.class == self.class && other.value == value) end # Replaces current value with normalized provided one. Doesn't diff --git a/spec/chewy/index/actions_spec.rb b/spec/chewy/index/actions_spec.rb index 0ca812d92..4a0d69893 100644 --- a/spec/chewy/index/actions_spec.rb +++ b/spec/chewy/index/actions_spec.rb @@ -610,7 +610,7 @@ specify 'with journal application' do cities p 'cities created1' - ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base) + ActiveRecord::Base.connection.close if defined?(ActiveRecord::Base) [ parallel_update, Thread.new do @@ -619,7 +619,7 @@ p 'end reset1' end ].map(&:join) - ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base) + ActiveRecord::Base.connection.reconnect! if defined?(ActiveRecord::Base) p 'expect1' expect(CitiesIndex::City.pluck(:_id, :name)).to contain_exactly(%w[1 NewName1], %w[2 Name2], %w[3 NewName3]) p 'end expect1' @@ -628,7 +628,7 @@ specify 'without journal application' do cities p 'cities created2' - ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base) + ActiveRecord::Base.connection.close if defined?(ActiveRecord::Base) [ parallel_update, Thread.new do @@ -637,7 +637,7 @@ p 'end reset2' end ].map(&:join) - ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base) + ActiveRecord::Base.connection.reconnect! if defined?(ActiveRecord::Base) p 'expect2' expect(CitiesIndex::City.pluck(:_id, :name)).to contain_exactly(%w[1 Name1], %w[2 Name2], %w[3 Name3]) p 'end expect2' diff --git a/spec/chewy/index/import/bulk_builder_spec.rb b/spec/chewy/index/import/bulk_builder_spec.rb index dd63c442c..1b2eb3a82 100644 --- a/spec/chewy/index/import/bulk_builder_spec.rb +++ b/spec/chewy/index/import/bulk_builder_spec.rb @@ -132,7 +132,7 @@ def derived before do stub_index(:cities) do crutch :names do |collection| - collection.map { |item| [item.id, "Name#{item.id}"] }.to_h + collection.to_h { |item| [item.id, "Name#{item.id}"] } end field :name, value: ->(o, c) { c.names[o.id] } @@ -198,7 +198,7 @@ def derived index_scope Comment crutch :content_with_crutches do |collection| # collection here is a current batch of products - collection.map { |comment| [comment.id, "[crutches] #{comment.content}"] }.to_h + collection.to_h { |comment| [comment.id, "[crutches] #{comment.content}"] } end field :content @@ -272,7 +272,7 @@ def root(comment) default_import_options raw_import: ->(hash) { SimpleComment.new(hash) } crutch :content_with_crutches do |collection| # collection here is a current batch of products - collection.map { |comment| [comment.id, "[crutches] #{comment.content}"] }.to_h + collection.to_h { |comment| [comment.id, "[crutches] #{comment.content}"] } end field :content diff --git a/spec/chewy/index/import_spec.rb b/spec/chewy/index/import_spec.rb index abc0152f3..16593113a 100644 --- a/spec/chewy/index/import_spec.rb +++ b/spec/chewy/index/import_spec.rb @@ -215,8 +215,8 @@ def subscribe_notification payload = subscribe_notification import dummy_cities, batch_size: 2 expect(payload).to eq(index: CitiesIndex, - errors: {index: {mapper_parsing_exception => %w[1 2 3]}}, - import: {index: 3}) + errors: {index: {mapper_parsing_exception => %w[1 2 3]}}, + import: {index: 3}) end end end @@ -567,7 +567,7 @@ def imported_comments before do stub_index(:cities) do crutch :names do |collection| - collection.map { |o| [o.name, "#{o.name}42"] }.to_h + collection.to_h { |o| [o.name, "#{o.name}42"] } end field :name, value: ->(o, c) { c.names[o.name] } field :rating diff --git a/spec/chewy/minitest/helpers_spec.rb b/spec/chewy/minitest/helpers_spec.rb index 41353b928..4d6f2d30e 100644 --- a/spec/chewy/minitest/helpers_spec.rb +++ b/spec/chewy/minitest/helpers_spec.rb @@ -10,7 +10,7 @@ def assert_includes(haystack, needle, _comment) expect(haystack).to include(needle) end - include ::Chewy::Minitest::Helpers + include Chewy::Minitest::Helpers def assert_equal(expected, actual, message) raise message unless expected == actual diff --git a/spec/chewy/minitest/search_index_receiver_spec.rb b/spec/chewy/minitest/search_index_receiver_spec.rb index 213d46de8..d93b97f41 100644 --- a/spec/chewy/minitest/search_index_receiver_spec.rb +++ b/spec/chewy/minitest/search_index_receiver_spec.rb @@ -24,6 +24,8 @@ def parse_request(request) SearchIndexReceiver.new end + let(:dummy_class) { Struct.new(:id) } + before do stub_index(:dummies) do root value: ->(_o) { {} } @@ -82,12 +84,12 @@ def parse_request(request) end specify 'validates that an object was indexed' do - dummy = OpenStruct.new(id: 1) + dummy = dummy_class.new(1) expect(receiver.indexed?(dummy, DummiesIndex)).to be(true) end specify 'doesn\'t validate than unindexed objects were indexed' do - dummy = OpenStruct.new(id: 2) + dummy = dummy_class.new(2) expect(receiver.indexed?(dummy, DummiesIndex)).to be(false) end end @@ -98,12 +100,12 @@ def parse_request(request) end specify 'validates than an object was deleted' do - dummy = OpenStruct.new(id: 1) + dummy = dummy_class.new(1) expect(receiver.deleted?(dummy, DummiesIndex)).to be(true) end specify 'doesn\'t validate than undeleted objects were deleted' do - dummy = OpenStruct.new(id: 2) + dummy = dummy_class.new(2) expect(receiver.deleted?(dummy, DummiesIndex)).to be(false) end end diff --git a/spec/chewy/rake_helper_spec.rb b/spec/chewy/rake_helper_spec.rb index 510e85f1a..f84ad129a 100644 --- a/spec/chewy/rake_helper_spec.rb +++ b/spec/chewy/rake_helper_spec.rb @@ -105,10 +105,10 @@ expect { described_class.reset(only: [CitiesIndex], output: output) } .to update_index(CitiesIndex) expect(output.string).to include( - "############################################################\n"\ - "WARN: You are risking to lose some changes during the reset.\n" \ - " Please consider enabling journaling.\n" \ - ' See https://github.com/toptal/chewy#journaling' + "############################################################\n" \ + "WARN: You are risking to lose some changes during the reset.\n " \ + "Please consider enabling journaling.\n " \ + 'See https://github.com/toptal/chewy#journaling' ) end end diff --git a/spec/chewy/rspec/helpers_spec.rb b/spec/chewy/rspec/helpers_spec.rb index 0b3938dcc..1d2e6cfd3 100644 --- a/spec/chewy/rspec/helpers_spec.rb +++ b/spec/chewy/rspec/helpers_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe :rspec_helper do - include ::Chewy::Rspec::Helpers + include Chewy::Rspec::Helpers before do stub_model(:city) diff --git a/spec/chewy/search/pagination/kaminari_examples.rb b/spec/chewy/search/pagination/kaminari_examples.rb index ccb033bfd..08a285716 100644 --- a/spec/chewy/search/pagination/kaminari_examples.rb +++ b/spec/chewy/search/pagination/kaminari_examples.rb @@ -24,7 +24,7 @@ let(:data) { Array.new(10) { |i| {id: i.next.to_s, name: "Name#{i.next}", age: 10 * i.next}.stringify_keys! } } before { ProductsIndex.import!(data.map { |h| double(h) }) } - before { allow(::Kaminari.config).to receive_messages(default_per_page: 3) } + before { allow(Kaminari.config).to receive_messages(default_per_page: 3) } describe '#per, #page' do specify { expect(search.map { |e| e.attributes.except(*except_fields) }).to match_array(data) } diff --git a/spec/chewy/search/pagination/kaminari_spec.rb b/spec/chewy/search/pagination/kaminari_spec.rb index c75eacaa8..bd117d4d6 100644 --- a/spec/chewy/search/pagination/kaminari_spec.rb +++ b/spec/chewy/search/pagination/kaminari_spec.rb @@ -6,7 +6,7 @@ let(:data) { Array.new(12) { |i| {id: i.next.to_s, name: "Name#{i.next}", age: 10 * i.next}.stringify_keys! } } before { ProductsIndex.import!(data.map { |h| double(h) }) } - before { allow(::Kaminari.config).to receive_messages(default_per_page: 17) } + before { allow(Kaminari.config).to receive_messages(default_per_page: 17) } specify { expect(search.objects.class).to eq(Kaminari::PaginatableArray) } specify { expect(search.objects.total_count).to eq(12) } diff --git a/spec/chewy/strategy/active_job_spec.rb b/spec/chewy/strategy/active_job_spec.rb index 71d9563d1..7fda880d4 100644 --- a/spec/chewy/strategy/active_job_spec.rb +++ b/spec/chewy/strategy/active_job_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -if defined?(::ActiveJob) +if defined?(ActiveJob) describe Chewy::Strategy::ActiveJob do around do |example| active_job_settings = Chewy.settings[:active_job] @@ -9,12 +9,12 @@ Chewy.settings[:active_job] = active_job_settings end before(:all) do - ::ActiveJob::Base.logger = Chewy.logger + ActiveJob::Base.logger = Chewy.logger end before do - ::ActiveJob::Base.queue_adapter = :test - ::ActiveJob::Base.queue_adapter.enqueued_jobs.clear - ::ActiveJob::Base.queue_adapter.performed_jobs.clear + ActiveJob::Base.queue_adapter = :test + ActiveJob::Base.queue_adapter.enqueued_jobs.clear + ActiveJob::Base.queue_adapter.performed_jobs.clear end before do @@ -39,7 +39,7 @@ Chewy.strategy(:active_job) do [city, other_city].map(&:save!) end - enqueued_job = ::ActiveJob::Base.queue_adapter.enqueued_jobs.first + enqueued_job = ActiveJob::Base.queue_adapter.enqueued_jobs.first expect(enqueued_job[:job]).to eq(Chewy::Strategy::ActiveJob::Worker) expect(enqueued_job[:queue]).to eq('low') end @@ -48,12 +48,12 @@ Chewy.strategy(:active_job) do [city, other_city].map(&:save!) end - enqueued_job = ::ActiveJob::Base.queue_adapter.enqueued_jobs.first + enqueued_job = ActiveJob::Base.queue_adapter.enqueued_jobs.first expect(enqueued_job[:queue]).to eq('low') end specify do - ::ActiveJob::Base.queue_adapter = :inline + ActiveJob::Base.queue_adapter = :inline expect { [city, other_city].map(&:save!) } .to update_index(CitiesIndex, strategy: :active_job) .and_reindex(city, other_city).only diff --git a/spec/chewy/strategy/lazy_sidekiq_spec.rb b/spec/chewy/strategy/lazy_sidekiq_spec.rb index e221c67ab..4078fb1a3 100644 --- a/spec/chewy/strategy/lazy_sidekiq_spec.rb +++ b/spec/chewy/strategy/lazy_sidekiq_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -if defined?(::Sidekiq) +if defined?(Sidekiq) require 'sidekiq/testing' describe Chewy::Strategy::LazySidekiq do @@ -10,7 +10,7 @@ Chewy.strategy(:bypass) { example.run } Chewy.settings[:sidekiq] = sidekiq_settings end - before { ::Sidekiq::Worker.clear_all } + before { Sidekiq::Worker.clear_all } context 'strategy' do before do @@ -32,14 +32,14 @@ end it 'updates indices asynchronously on record save' do - expect(::Sidekiq::Client).to receive(:push) + expect(Sidekiq::Client).to receive(:push) .with(hash_including( 'class' => Chewy::Strategy::LazySidekiq::IndicesUpdateWorker, 'queue' => 'low' )) .and_call_original .once - ::Sidekiq::Testing.inline! do + Sidekiq::Testing.inline! do expect { [city, other_city].map(&:save!) } .to update_index(CitiesIndex, strategy: :lazy_sidekiq) .and_reindex(city, other_city).only @@ -47,12 +47,12 @@ end it 'updates indices asynchronously with falling back to sidekiq strategy on record destroy' do - expect(::Sidekiq::Client).not_to receive(:push) + expect(Sidekiq::Client).not_to receive(:push) .with(hash_including( 'class' => Chewy::Strategy::LazySidekiq::IndicesUpdateWorker, 'queue' => 'low' )) - expect(::Sidekiq::Client).to receive(:push) + expect(Sidekiq::Client).to receive(:push) .with(hash_including( 'class' => Chewy::Strategy::Sidekiq::Worker, 'queue' => 'low', @@ -60,7 +60,7 @@ )) .and_call_original .once - ::Sidekiq::Testing.inline! do + Sidekiq::Testing.inline! do expect { [city, other_city].map(&:destroy) }.to update_index(CitiesIndex, strategy: :sidekiq) end end @@ -71,7 +71,7 @@ expect(other_city).to receive(:run_chewy_callbacks).and_call_original expect do - ::Sidekiq::Testing.inline! do + Sidekiq::Testing.inline! do Chewy::Strategy::LazySidekiq::IndicesUpdateWorker.new.perform({'City' => [city.id, other_city.id]}) end end.to update_index(CitiesIndex).and_reindex(city, other_city).only @@ -88,7 +88,7 @@ expect(other_city).to receive(:run_chewy_callbacks).and_call_original expect do - ::Sidekiq::Testing.inline! do + Sidekiq::Testing.inline! do Chewy::Strategy::LazySidekiq::IndicesUpdateWorker.new.perform({'City' => [city.id, other_city.id]}) end end.to update_index(CitiesIndex).and_reindex(city, other_city).only.no_refresh @@ -97,7 +97,7 @@ end context 'integration' do - around { |example| ::Sidekiq::Testing.inline! { example.run } } + around { |example| Sidekiq::Testing.inline! { example.run } } let(:update_condition) { true } diff --git a/spec/chewy/strategy/sidekiq_spec.rb b/spec/chewy/strategy/sidekiq_spec.rb index 943ac6f7e..13ca55e90 100644 --- a/spec/chewy/strategy/sidekiq_spec.rb +++ b/spec/chewy/strategy/sidekiq_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -if defined?(::Sidekiq) +if defined?(Sidekiq) require 'sidekiq/testing' describe Chewy::Strategy::Sidekiq do @@ -10,7 +10,7 @@ Chewy.strategy(:bypass) { example.run } Chewy.settings[:sidekiq] = sidekiq_settings end - before { ::Sidekiq::Worker.clear_all } + before { Sidekiq::Worker.clear_all } before do stub_model(:city) do update_index('cities') { self } @@ -30,8 +30,8 @@ end specify do - expect(::Sidekiq::Client).to receive(:push).with(hash_including('queue' => 'low')).and_call_original - ::Sidekiq::Testing.inline! do + expect(Sidekiq::Client).to receive(:push).with(hash_including('queue' => 'low')).and_call_original + Sidekiq::Testing.inline! do expect { [city, other_city].map(&:save!) } .to update_index(CitiesIndex, strategy: :sidekiq) .and_reindex(city, other_city).only diff --git a/spec/chewy_spec.rb b/spec/chewy_spec.rb index 4dcf2718a..3023b71e4 100644 --- a/spec/chewy_spec.rb +++ b/spec/chewy_spec.rb @@ -59,7 +59,7 @@ Chewy.current[:chewy_client] = nil allow(Chewy).to receive_messages(configuration: {transport_options: {proc: faraday_block}}) - allow(::Elasticsearch::Client).to receive(:new).with(expected_client_config) do |*_args, &passed_block| + allow(Elasticsearch::Client).to receive(:new).with(expected_client_config) do |*_args, &passed_block| # RSpec's `with(..., &block)` was used previously, but doesn't actually do # any verification of the passed block (even of its presence). expect(passed_block.source_location).to eq(faraday_block.source_location) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 48e901d6c..be66332f8 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -9,7 +9,7 @@ require 'timecop' -Kaminari::Hooks.init if defined?(::Kaminari::Hooks) +Kaminari::Hooks.init if defined?(Kaminari::Hooks) require 'support/fail_helpers' require 'support/class_helpers' From 7ffdcbef42ad82dc47bc862f2738dd27ed3082cb Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Wed, 13 Dec 2023 21:34:53 +0300 Subject: [PATCH 29/59] Make a release (#914) --- CHANGELOG.md | 11 ++++++++++- lib/chewy/version.rb | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd938af9d..d0a0979b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,21 @@ ### Bugs Fixed +## 7.4.0 (2023-12-13) + +### New Features + +### Changes + +* [#911](https://github.com/toptal/chewy/pull/911): Remove ruby 2.x. ([@konalegi][https://github.com/konalegi]) + +### Bugs Fixed + ## 7.3.5 (2023-12-13) ### New Features * [#890](https://github.com/toptal/chewy/pull/890): Add the [`knn`](https://www.elastic.co/guide/en/elasticsearch/reference/current/knn-search.html) option to the request. ([@jkostolansky][]) -* [#911](https://github.com/toptal/chewy/pull/911): Remove ruby 2.x. ([@konalegi][]) ### Changes diff --git a/lib/chewy/version.rb b/lib/chewy/version.rb index b22c523e3..cafc7e2e1 100644 --- a/lib/chewy/version.rb +++ b/lib/chewy/version.rb @@ -1,3 +1,3 @@ module Chewy - VERSION = '7.3.6'.freeze + VERSION = '7.4.0'.freeze end From 63d6458a6caa45b3ae8b0632787dd81f841d85c9 Mon Sep 17 00:00:00 2001 From: Tom de Vries Date: Sat, 16 Dec 2023 09:33:37 +0100 Subject: [PATCH 30/59] Update CHANGELOG.md (#915) Fix incorrect mention of version release 7.3.5; the second occurrence should be 7.3.6. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0a0979b6..ee8a9ba74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ ### Bugs Fixed -## 7.3.5 (2023-12-13) +## 7.3.6 (2023-12-13) ### New Features From ea0ed0ae1162d46147e46d5ecf82da8347dcaa14 Mon Sep 17 00:00:00 2001 From: R Gibim <9031589+Drowze@users.noreply.github.com> Date: Sat, 16 Dec 2023 17:28:18 -0300 Subject: [PATCH 31/59] Add helper to cleanup delayed sidekiq timechunks (#894) Mainly useful to avoid flaky tests when using a real redis database and not flushing the database in between tests --- CHANGELOG.md | 2 ++ README.md | 6 ++++++ lib/chewy/strategy/delayed_sidekiq.rb | 13 +++++++++++++ lib/chewy/strategy/delayed_sidekiq/scheduler.rb | 3 +++ spec/chewy/strategy/delayed_sidekiq_spec.rb | 12 ++++++++++++ 5 files changed, 36 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee8a9ba74..eb2863c94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### New Features +* [#894](https://github.com/toptal/chewy/pull/894): Way of cleaning redis from artifacts left by `delayed_sidekiq` strategy which could potentially cause flaky tests. ([@Drowze](https://github.com/Drowze)) + ### Changes ### Bugs Fixed diff --git a/README.md b/README.md index bc08c32a8..dbfe974ce 100644 --- a/README.md +++ b/README.md @@ -848,6 +848,12 @@ Explicit call of the reindex using `:delayed_sidekiq` strategy with `:update_fie CitiesIndex.import([1, 2, 3], update_fields: [:name], strategy: :delayed_sidekiq) ``` +While running tests with delayed_sidekiq strategy and Sidekiq is using a real redis instance that is NOT cleaned up in between tests (via e.g. `Sidekiq.redis(&:flushdb)`), you'll want to cleanup some redis keys in between tests to avoid state leaking and flaky tests. Chewy provides a convenience method for that: +```ruby +# it might be a good idea to also add to your testing setup, e.g.: a rspec `before` hook +Chewy::Strategy::DelayedSidekiq.clear_timechunks! +``` + #### `:active_job` This does the same thing as `:atomic`, but using ActiveJob. This will inherit the ActiveJob configuration settings including the `active_job.queue_adapter` setting for the environment. Patch `Chewy::Strategy::ActiveJob::Worker` for index updates improving. diff --git a/lib/chewy/strategy/delayed_sidekiq.rb b/lib/chewy/strategy/delayed_sidekiq.rb index 2c694071c..0bdd25c88 100644 --- a/lib/chewy/strategy/delayed_sidekiq.rb +++ b/lib/chewy/strategy/delayed_sidekiq.rb @@ -5,6 +5,19 @@ class Strategy class DelayedSidekiq < Sidekiq require_relative 'delayed_sidekiq/scheduler' + # cleanup the redis sets used internally. Useful mainly in tests to avoid + # leak and potential flaky tests. + def self.clear_timechunks! + ::Sidekiq.redis do |redis| + timechunk_sets = redis.smembers(Chewy::Strategy::DelayedSidekiq::Scheduler::ALL_SETS_KEY) + break if timechunk_sets.empty? + + redis.pipelined do |pipeline| + timechunk_sets.each { |set| pipeline.del(set) } + end + end + end + def leave @stash.each do |type, ids| next if ids.empty? diff --git a/lib/chewy/strategy/delayed_sidekiq/scheduler.rb b/lib/chewy/strategy/delayed_sidekiq/scheduler.rb index 9f4bf386e..f1010a3ee 100644 --- a/lib/chewy/strategy/delayed_sidekiq/scheduler.rb +++ b/lib/chewy/strategy/delayed_sidekiq/scheduler.rb @@ -18,6 +18,7 @@ class Scheduler DEFAULT_MARGIN = 2 DEFAULT_QUEUE = 'chewy' KEY_PREFIX = 'chewy:delayed_sidekiq' + ALL_SETS_KEY = "#{KEY_PREFIX}:all_sets".freeze FALLBACK_FIELDS = 'all' FIELDS_IDS_SEPARATOR = ';' IDS_SEPARATOR = ',' @@ -68,8 +69,10 @@ def postpone ::Sidekiq.redis do |redis| # warning: Redis#sadd will always return an Integer in Redis 5.0.0. Use Redis#sadd? instead if redis.respond_to?(:sadd?) + redis.sadd?(ALL_SETS_KEY, timechunks_key) redis.sadd?(timechunk_key, serialize_data) else + redis.sadd(ALL_SETS_KEY, timechunks_key) redis.sadd(timechunk_key, serialize_data) end diff --git a/spec/chewy/strategy/delayed_sidekiq_spec.rb b/spec/chewy/strategy/delayed_sidekiq_spec.rb index ac9f792c7..c0d43e576 100644 --- a/spec/chewy/strategy/delayed_sidekiq_spec.rb +++ b/spec/chewy/strategy/delayed_sidekiq_spec.rb @@ -186,5 +186,17 @@ end end end + + describe '#clear_delayed_sidekiq_timechunks test helper' do + it 'clears redis from the timechunk sorted sets to avoid leak between tests' do + timechunks_set = -> { Sidekiq.redis { |redis| redis.zrange('chewy:delayed_sidekiq:CitiesIndex:timechunks', 0, -1) } } + + expect { CitiesIndex.import!([1], strategy: :delayed_sidekiq) } + .to change { timechunks_set.call.size }.by(1) + + expect { Chewy::Strategy::DelayedSidekiq.clear_timechunks! } + .to change { timechunks_set.call.size }.to(0) + end + end end end From cf070863245389f8a0c91def3a8d5c91261e1690 Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Mon, 15 Jan 2024 10:27:38 +0300 Subject: [PATCH 32/59] Add before request filter (#919) --- CHANGELOG.md | 1 + README.md | 17 +++++++++++++++++ lib/chewy.rb | 8 ++------ lib/chewy/config.rb | 4 +++- lib/chewy/elastic_client.rb | 31 +++++++++++++++++++++++++++++++ spec/chewy/elastic_client_spec.rb | 26 ++++++++++++++++++++++++++ spec/chewy_spec.rb | 11 +++++++---- 7 files changed, 87 insertions(+), 11 deletions(-) create mode 100644 lib/chewy/elastic_client.rb create mode 100644 spec/chewy/elastic_client_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index eb2863c94..c4bc165c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### New Features * [#894](https://github.com/toptal/chewy/pull/894): Way of cleaning redis from artifacts left by `delayed_sidekiq` strategy which could potentially cause flaky tests. ([@Drowze](https://github.com/Drowze)) +* [#919](https://github.com/toptal/chewy/pull/919): Add pre-request filter ([@konalegi][https://github.com/konalegi]) ### Changes diff --git a/README.md b/README.md index dbfe974ce..8c0baec80 100644 --- a/README.md +++ b/README.md @@ -1281,6 +1281,23 @@ If you use `DatabaseCleaner` in your tests with [the `transaction` strategy](htt Chewy.use_after_commit_callbacks = !Rails.env.test? ``` +### Pre-request Filter + +Should you need to inspect the query prior to it being dispatched to ElasticSearch during any queries, you can use the `before_es_request_filter`. `before_es_request_filter` is a callable object, as demonstrated below: + +```ruby +Chewy.before_es_request_filter = -> (method_name, args, kw_args) { ... } +``` + +While using the `before_es_request_filter`, please consider the following: + +* `before_es_request_filter` acts as a simple proxy before any request made via the `ElasticSearch::Client`. The arguments passed to this filter include: + * `method_name` - The name of the method being called. Examples are search, count, bulk and etc. + * `args` and `kw_args` - These are the positional arguments provided in the method call. +* The operation is synchronous, so avoid executing any heavy or time-consuming operations within the filter to prevent performance degradation. +* The return value of the proc is disregarded. This filter is intended for inspection or modification of the query rather than generating a response. +* Any exception raised inside the callback will propagate upward and halt the execution of the query. It is essential to handle potential errors adequately to ensure the stability of your search functionality. + ## Contributing 1. Fork it (http://github.com/toptal/chewy/fork) diff --git a/lib/chewy.rb b/lib/chewy.rb index cac447724..919f9bac8 100644 --- a/lib/chewy.rb +++ b/lib/chewy.rb @@ -49,6 +49,7 @@ def try_require(path) require 'chewy/fields/root' require 'chewy/journal' require 'chewy/railtie' if defined?(Rails::Railtie) +require 'chewy/elastic_client' ActiveSupport.on_load(:active_record) do include Chewy::Index::Observe::ActiveRecordMethods @@ -97,12 +98,7 @@ def derive_name(index_name) # Main elasticsearch-ruby client instance # def client - Chewy.current[:chewy_client] ||= begin - client_configuration = configuration.deep_dup - client_configuration.delete(:prefix) # used by Chewy, not relevant to Elasticsearch::Client - block = client_configuration[:transport_options].try(:delete, :proc) - ::Elasticsearch::Client.new(client_configuration, &block) - end + Chewy.current[:chewy_client] ||= Chewy::ElasticClient.new end # Sends wait_for_status request to ElasticSearch with status diff --git a/lib/chewy/config.rb b/lib/chewy/config.rb index 4d7a695f7..65c60b368 100644 --- a/lib/chewy/config.rb +++ b/lib/chewy/config.rb @@ -38,7 +38,9 @@ class Config # for type mappings like `_all`. :default_root_options, # Default field type for any field in any Chewy type. Defaults to 'text'. - :default_field_type + :default_field_type, + # Callback called on each search request to be done into ES + :before_es_request_filter attr_reader :transport_logger, :transport_tracer, # Chewy search request DSL base class, used by every index. diff --git a/lib/chewy/elastic_client.rb b/lib/chewy/elastic_client.rb new file mode 100644 index 000000000..41a985ccc --- /dev/null +++ b/lib/chewy/elastic_client.rb @@ -0,0 +1,31 @@ +module Chewy + # Replacement for Chewy.client + class ElasticClient + def self.build_es_client(configuration = Chewy.configuration) + client_configuration = configuration.deep_dup + client_configuration.delete(:prefix) # used by Chewy, not relevant to Elasticsearch::Client + block = client_configuration[:transport_options].try(:delete, :proc) + ::Elasticsearch::Client.new(client_configuration, &block) + end + + def initialize(elastic_client = self.class.build_es_client) + @elastic_client = elastic_client + end + + private + + def method_missing(name, *args, **kwargs, &block) + inspect_payload(name, args, kwargs) + + @elastic_client.__send__(name, *args, **kwargs, &block) + end + + def respond_to_missing?(name, _include_private = false) + @elastic_client.respond_to?(name) || super + end + + def inspect_payload(name, args, kwargs) + Chewy.config.before_es_request_filter&.call(name, args, kwargs) + end + end +end diff --git a/spec/chewy/elastic_client_spec.rb b/spec/chewy/elastic_client_spec.rb new file mode 100644 index 000000000..79d3946ba --- /dev/null +++ b/spec/chewy/elastic_client_spec.rb @@ -0,0 +1,26 @@ +require 'spec_helper' + +describe Chewy::ElasticClient do + describe 'payload inspection' do + let(:filter) { instance_double('Proc') } + let!(:filter_previous_value) { Chewy.before_es_request_filter } + + before do + Chewy.massacre + stub_index(:products) do + field :id, type: :integer + end + ProductsIndex.create + Chewy.before_es_request_filter = filter + end + + after do + Chewy.before_es_request_filter = filter_previous_value + end + + it 'call filter with the request body' do + expect(filter).to receive(:call).with(:search, [{body: {size: 0}, index: ['products']}], {}) + Chewy.client.search({index: ['products'], body: {size: 0}}).to_a + end + end +end diff --git a/spec/chewy_spec.rb b/spec/chewy_spec.rb index 3023b71e4..0c3e0d0ec 100644 --- a/spec/chewy_spec.rb +++ b/spec/chewy_spec.rb @@ -57,18 +57,21 @@ before do Chewy.current[:chewy_client] = nil - allow(Chewy).to receive_messages(configuration: {transport_options: {proc: faraday_block}}) + end + + specify do + expect(Chewy).to receive_messages(configuration: {transport_options: {proc: faraday_block}}) - allow(Elasticsearch::Client).to receive(:new).with(expected_client_config) do |*_args, &passed_block| + expect(Elasticsearch::Client).to receive(:new).with(expected_client_config) do |*_args, &passed_block| # RSpec's `with(..., &block)` was used previously, but doesn't actually do # any verification of the passed block (even of its presence). expect(passed_block.source_location).to eq(faraday_block.source_location) mock_client end - end - its(:client) { is_expected.to eq(mock_client) } + expect(Chewy.client).to be_a(Chewy::ElasticClient) + end after { Chewy.current[:chewy_client] = initial_client } end From f383e470f7631d668672dd83469623294cc7d13e Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Mon, 15 Jan 2024 10:37:55 +0300 Subject: [PATCH 33/59] Bump chewy to 7.5.0 (#920) --- CHANGELOG.md | 10 +++++++--- lib/chewy/version.rb | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4bc165c4..03b551a25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,17 @@ ### New Features -* [#894](https://github.com/toptal/chewy/pull/894): Way of cleaning redis from artifacts left by `delayed_sidekiq` strategy which could potentially cause flaky tests. ([@Drowze](https://github.com/Drowze)) -* [#919](https://github.com/toptal/chewy/pull/919): Add pre-request filter ([@konalegi][https://github.com/konalegi]) - ### Changes ### Bugs Fixed +## 7.5.0 (2023-01-15) + +### New Features + +* [#894](https://github.com/toptal/chewy/pull/894): Way of cleaning redis from artifacts left by `delayed_sidekiq` strategy which could potentially cause flaky tests. ([@Drowze](https://github.com/Drowze)) +* [#919](https://github.com/toptal/chewy/pull/919): Add pre-request filter ([@konalegi][https://github.com/konalegi]) + ## 7.4.0 (2023-12-13) ### New Features diff --git a/lib/chewy/version.rb b/lib/chewy/version.rb index cafc7e2e1..fd2ba8200 100644 --- a/lib/chewy/version.rb +++ b/lib/chewy/version.rb @@ -1,3 +1,3 @@ module Chewy - VERSION = '7.4.0'.freeze + VERSION = '7.5.0'.freeze end From 895483566ea0e03aa82886d6da76f0d6132a89b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jan 2024 13:29:04 +0100 Subject: [PATCH 34/59] Update rubocop requirement from 1.48 to 1.60.0 (#921) Updates the requirements on [rubocop](https://github.com/rubocop/rubocop) to permit the latest version. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.48.0...v1.60.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .rubocop.yml | 3 +++ gemfiles/base.gemfile | 2 +- lib/chewy/errors.rb | 4 ++-- lib/chewy/rake_helper.rb | 6 +----- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index e6c827173..6c2157253 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -59,3 +59,6 @@ Metrics/ModuleLength: Exclude: - 'lib/chewy/rake_helper.rb' - '**/*_spec.rb' + +Style/ArgumentsForwarding: + Enabled: false diff --git a/gemfiles/base.gemfile b/gemfiles/base.gemfile index 656b9395e..3800758a3 100644 --- a/gemfiles/base.gemfile +++ b/gemfiles/base.gemfile @@ -6,7 +6,7 @@ gem 'rake' gem 'rspec', '>= 3.7.0' gem 'rspec-collection_matchers' gem 'rspec-its' -gem 'rubocop', '1.48' +gem 'rubocop', '1.60.0' gem 'sqlite3' gem 'timecop' gem 'unparser' diff --git a/lib/chewy/errors.rb b/lib/chewy/errors.rb index 5b198ed26..fe139640a 100644 --- a/lib/chewy/errors.rb +++ b/lib/chewy/errors.rb @@ -7,7 +7,7 @@ class UndefinedIndex < Error class UndefinedUpdateStrategy < Error def initialize(_type) - super <<-MESSAGE + super(<<-MESSAGE) Index update strategy is undefined for current context. Please wrap your code with `Chewy.strategy(:strategy_name) block.` MESSAGE @@ -27,7 +27,7 @@ def initialize(type, import_errors) message << " on #{documents.count} documents: #{documents}\n" end end - super message + super(message) end end diff --git a/lib/chewy/rake_helper.rb b/lib/chewy/rake_helper.rb index a59fa02a3..32de31f6e 100644 --- a/lib/chewy/rake_helper.rb +++ b/lib/chewy/rake_helper.rb @@ -320,11 +320,7 @@ def indexes_from(only: nil, except: nil) all_indexes end - indexes = if except.present? - indexes - normalize_indexes(Array.wrap(except)) - else - indexes - end + indexes -= normalize_indexes(Array.wrap(except)) if except.present? indexes.sort_by(&:derivable_name) end From c833003a98d5ba82c67ff4f82af72e8f476d7697 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jan 2024 17:50:53 +0800 Subject: [PATCH 35/59] Update rubocop requirement from 1.60.0 to 1.60.1 (#923) Updates the requirements on [rubocop](https://github.com/rubocop/rubocop) to permit the latest version. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.60.0...v1.60.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gemfiles/base.gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemfiles/base.gemfile b/gemfiles/base.gemfile index 3800758a3..265cd2e7e 100644 --- a/gemfiles/base.gemfile +++ b/gemfiles/base.gemfile @@ -6,7 +6,7 @@ gem 'rake' gem 'rspec', '>= 3.7.0' gem 'rspec-collection_matchers' gem 'rspec-its' -gem 'rubocop', '1.60.0' +gem 'rubocop', '1.60.1' gem 'sqlite3' gem 'timecop' gem 'unparser' From 8604b4a18171a7d23e9c6323de9a2df9926d1a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Bu=C5=82at?= Date: Tue, 30 Jan 2024 12:48:08 +0100 Subject: [PATCH 36/59] Make default index scope cleanup behavior configurable (#925) * Make default index scope cleanup behavior configurable Instead of warning, you can completely ignore cases when the scope include order/offset/limit or you can raise an exception instead. * Update changelog --- CHANGELOG.md | 2 + README.md | 18 ++++++ lib/chewy/config.rb | 6 +- lib/chewy/errors.rb | 3 + lib/chewy/index/adapter/active_record.rb | 14 ++++- .../chewy/index/adapter/active_record_spec.rb | 62 +++++++++++++++++++ 6 files changed, 102 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03b551a25..8da3eb05c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### New Features +* [#925](https://github.com/toptal/chewy/pull/925): Add configuration option for default scope cleanup behavior. ([@barthez][]) + ### Changes ### Bugs Fixed diff --git a/README.md b/README.md index 8c0baec80..4082fd038 100644 --- a/README.md +++ b/README.md @@ -1298,6 +1298,24 @@ While using the `before_es_request_filter`, please consider the following: * The return value of the proc is disregarded. This filter is intended for inspection or modification of the query rather than generating a response. * Any exception raised inside the callback will propagate upward and halt the execution of the query. It is essential to handle potential errors adequately to ensure the stability of your search functionality. +### Import scope clean-up behavior + +Whenever you set the `import_scope` for the index, in the case of ActiveRecord, +options for order, offset and limit will be removed. You can set the behavior of +chewy, before the clean-up itself. + +The default behavior is a warning sent to the Chewy logger (`:warn`). Another more +restrictive option is raising an exception (`:raise`). Both options have a +negative impact on performance since verifying whether the code uses any of +these options requires building AREL query. + +To avoid the loading time impact, you can ignore the check (`:ignore`) before +the clean-up. + +``` +Chewy.import_scope_cleanup_behavior = :ignore +``` + ## Contributing 1. Fork it (http://github.com/toptal/chewy/fork) diff --git a/lib/chewy/config.rb b/lib/chewy/config.rb index 65c60b368..fdae4ae45 100644 --- a/lib/chewy/config.rb +++ b/lib/chewy/config.rb @@ -40,7 +40,10 @@ class Config # Default field type for any field in any Chewy type. Defaults to 'text'. :default_field_type, # Callback called on each search request to be done into ES - :before_es_request_filter + :before_es_request_filter, + # Behavior when import scope for index includes order, offset or limit. + # Can be :ignore, :warn, :raise. Defaults to :warn + :import_scope_cleanup_behavior attr_reader :transport_logger, :transport_tracer, # Chewy search request DSL base class, used by every index. @@ -62,6 +65,7 @@ def initialize @indices_path = 'app/chewy' @default_root_options = {} @default_field_type = 'text'.freeze + @import_scope_cleanup_behavior = :warn @search_class = build_search_class(Chewy::Search::Request) end diff --git a/lib/chewy/errors.rb b/lib/chewy/errors.rb index fe139640a..d470bd94f 100644 --- a/lib/chewy/errors.rb +++ b/lib/chewy/errors.rb @@ -36,4 +36,7 @@ def initialize(join_field_type, join_field_name, relations) super("`#{join_field_type}` set for the join field `#{join_field_name}` is not on the :relations list (#{relations})") end end + + class ImportScopeCleanupError < Error + end end diff --git a/lib/chewy/index/adapter/active_record.rb b/lib/chewy/index/adapter/active_record.rb index 05b029aa7..8682b3b13 100644 --- a/lib/chewy/index/adapter/active_record.rb +++ b/lib/chewy/index/adapter/active_record.rb @@ -13,9 +13,19 @@ def self.accepts?(target) private def cleanup_default_scope! - if Chewy.logger && (@default_scope.arel.orders.present? || + behavior = Chewy.config.import_scope_cleanup_behavior + + if behavior != :ignore && (@default_scope.arel.orders.present? || @default_scope.arel.limit.present? || @default_scope.arel.offset.present?) - Chewy.logger.warn('Default type scope order, limit and offset are ignored and will be nullified') + if behavior == :warn && Chewy.logger + gem_dir = File.realpath('../..', __dir__) + source = caller.grep_v(Regexp.new(gem_dir)).first + Chewy.logger.warn( + "Default type scope order, limit and offset are ignored and will be nullified (called from: #{source})" + ) + elsif behavior == :raise + raise ImportScopeCleanupError, 'Default type scope order, limit and offset are ignored and will be nullified' + end end @default_scope = @default_scope.reorder(nil).limit(nil).offset(nil) diff --git a/spec/chewy/index/adapter/active_record_spec.rb b/spec/chewy/index/adapter/active_record_spec.rb index 4f2f8b9ab..062f17cb3 100644 --- a/spec/chewy/index/adapter/active_record_spec.rb +++ b/spec/chewy/index/adapter/active_record_spec.rb @@ -35,6 +35,68 @@ def rating specify { expect(described_class.new(City.where(rating: 10)).default_scope).to eq(City.where(rating: 10)) } end + describe '.new' do + context 'with logger' do + let(:test_logger) { Logger.new('/dev/null') } + let(:default_scope_behavior) { :warn } + + around do |example| + previous_logger = Chewy.logger + Chewy.logger = test_logger + + previous_default_scope_behavior = Chewy.config.import_scope_cleanup_behavior + Chewy.config.import_scope_cleanup_behavior = default_scope_behavior + + example.run + ensure + Chewy.logger = previous_logger + Chewy.config.import_scope_cleanup_behavior = previous_default_scope_behavior + end + + specify do + expect(test_logger).to receive(:warn) + described_class.new(City.order(:id)) + end + + specify do + expect(test_logger).to receive(:warn) + described_class.new(City.offset(10)) + end + + specify do + expect(test_logger).to receive(:warn) + described_class.new(City.limit(10)) + end + + context 'ignore import scope warning' do + let(:default_scope_behavior) { :ignore } + + specify do + expect(test_logger).not_to receive(:warn) + described_class.new(City.order(:id)) + end + + specify do + expect(test_logger).not_to receive(:warn) + described_class.new(City.offset(10)) + end + + specify do + expect(test_logger).not_to receive(:warn) + described_class.new(City.limit(10)) + end + end + + context 'raise exception on import scope with order/limit/offset' do + let(:default_scope_behavior) { :raise } + + specify { expect { described_class.new(City.order(:id)) }.to raise_error(Chewy::ImportScopeCleanupError) } + specify { expect { described_class.new(City.limit(10)) }.to raise_error(Chewy::ImportScopeCleanupError) } + specify { expect { described_class.new(City.offset(10)) }.to raise_error(Chewy::ImportScopeCleanupError) } + end + end + end + describe '#type_name' do specify { expect(described_class.new(City).type_name).to eq('city') } specify { expect(described_class.new(City.order(:id)).type_name).to eq('city') } From 7460f8e7b2af4712604748ef9d5faa4daea1b417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Bu=C5=82at?= Date: Tue, 30 Jan 2024 13:56:33 +0100 Subject: [PATCH 37/59] Prepare 7.5.1 release (#926) --- CHANGELOG.md | 10 +++++++++- lib/chewy/version.rb | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8da3eb05c..073d61140 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,21 @@ ### New Features +### Changes + +### Bugs Fixed + +## 7.5.1 (2024-01-30) + +### New Features + * [#925](https://github.com/toptal/chewy/pull/925): Add configuration option for default scope cleanup behavior. ([@barthez][]) ### Changes ### Bugs Fixed -## 7.5.0 (2023-01-15) +## 7.5.0 (2024-01-15) ### New Features diff --git a/lib/chewy/version.rb b/lib/chewy/version.rb index fd2ba8200..e5d371949 100644 --- a/lib/chewy/version.rb +++ b/lib/chewy/version.rb @@ -1,3 +1,3 @@ module Chewy - VERSION = '7.5.0'.freeze + VERSION = '7.5.1'.freeze end From 35a0b7bdc9194c91589f2285dbdece59b17ea5d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 Feb 2024 09:26:40 +0100 Subject: [PATCH 38/59] Update rubocop requirement from 1.60.1 to 1.60.2 (#927) Updates the requirements on [rubocop](https://github.com/rubocop/rubocop) to permit the latest version. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.60.1...v1.60.2) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gemfiles/base.gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemfiles/base.gemfile b/gemfiles/base.gemfile index 265cd2e7e..20d34feba 100644 --- a/gemfiles/base.gemfile +++ b/gemfiles/base.gemfile @@ -6,7 +6,7 @@ gem 'rake' gem 'rspec', '>= 3.7.0' gem 'rspec-collection_matchers' gem 'rspec-its' -gem 'rubocop', '1.60.1' +gem 'rubocop', '1.60.2' gem 'sqlite3' gem 'timecop' gem 'unparser' From 4cebde6645d33425436cd9f4327dd5db60329355 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Mar 2024 09:48:53 +0100 Subject: [PATCH 39/59] Update rubocop requirement from 1.60.2 to 1.61.0 (#931) Updates the requirements on [rubocop](https://github.com/rubocop/rubocop) to permit the latest version. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.60.2...v1.61.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gemfiles/base.gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemfiles/base.gemfile b/gemfiles/base.gemfile index 20d34feba..fa54219ad 100644 --- a/gemfiles/base.gemfile +++ b/gemfiles/base.gemfile @@ -6,7 +6,7 @@ gem 'rake' gem 'rspec', '>= 3.7.0' gem 'rspec-collection_matchers' gem 'rspec-its' -gem 'rubocop', '1.60.2' +gem 'rubocop', '1.61.0' gem 'sqlite3' gem 'timecop' gem 'unparser' From 4fde2ac4a6a6e35c3f5ed2a563efb91b9e3d673c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Mar 2024 10:03:53 +0200 Subject: [PATCH 40/59] Update rubocop requirement from 1.61.0 to 1.62.1 (#932) Updates the requirements on [rubocop](https://github.com/rubocop/rubocop) to permit the latest version. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.61.0...v1.62.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gemfiles/base.gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemfiles/base.gemfile b/gemfiles/base.gemfile index fa54219ad..844bb85fb 100644 --- a/gemfiles/base.gemfile +++ b/gemfiles/base.gemfile @@ -6,7 +6,7 @@ gem 'rake' gem 'rspec', '>= 3.7.0' gem 'rspec-collection_matchers' gem 'rspec-its' -gem 'rubocop', '1.61.0' +gem 'rubocop', '1.62.1' gem 'sqlite3' gem 'timecop' gem 'unparser' From cdb5ee75d362a7d7734304761dbce56c7e794ca4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 16:25:06 +0200 Subject: [PATCH 41/59] Update rubocop requirement from 1.62.1 to 1.63.0 (#936) Updates the requirements on [rubocop](https://github.com/rubocop/rubocop) to permit the latest version. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.62.1...v1.63.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gemfiles/base.gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemfiles/base.gemfile b/gemfiles/base.gemfile index 844bb85fb..4e2e6fcde 100644 --- a/gemfiles/base.gemfile +++ b/gemfiles/base.gemfile @@ -6,7 +6,7 @@ gem 'rake' gem 'rspec', '>= 3.7.0' gem 'rspec-collection_matchers' gem 'rspec-its' -gem 'rubocop', '1.62.1' +gem 'rubocop', '1.63.0' gem 'sqlite3' gem 'timecop' gem 'unparser' From 41e474130b56a46a9e7da4eaee08472d98efab66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Apr 2024 13:04:45 +0100 Subject: [PATCH 42/59] Update rubocop requirement from 1.63.0 to 1.63.2 (#939) * Update rubocop requirement from 1.63.0 to 1.63.2 Updates the requirements on [rubocop](https://github.com/rubocop/rubocop) to permit the latest version. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.63.0...v1.63.2) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Fix specs --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Samuel Ebeagu --- gemfiles/base.gemfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gemfiles/base.gemfile b/gemfiles/base.gemfile index 4e2e6fcde..67d0b1dad 100644 --- a/gemfiles/base.gemfile +++ b/gemfiles/base.gemfile @@ -6,7 +6,7 @@ gem 'rake' gem 'rspec', '>= 3.7.0' gem 'rspec-collection_matchers' gem 'rspec-its' -gem 'rubocop', '1.63.0' -gem 'sqlite3' +gem 'rubocop', '1.63.2' +gem 'sqlite3', '~> 1.4' gem 'timecop' gem 'unparser' From 3d559bb29b60281956c536022e356dac96059502 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Apr 2024 09:56:52 +0200 Subject: [PATCH 43/59] Update rubocop requirement from 1.63.2 to 1.63.3 (#945) Updates the requirements on [rubocop](https://github.com/rubocop/rubocop) to permit the latest version. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.63.2...v1.63.3) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gemfiles/base.gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemfiles/base.gemfile b/gemfiles/base.gemfile index 67d0b1dad..ce0e6e889 100644 --- a/gemfiles/base.gemfile +++ b/gemfiles/base.gemfile @@ -6,7 +6,7 @@ gem 'rake' gem 'rspec', '>= 3.7.0' gem 'rspec-collection_matchers' gem 'rspec-its' -gem 'rubocop', '1.63.2' +gem 'rubocop', '1.63.3' gem 'sqlite3', '~> 1.4' gem 'timecop' gem 'unparser' From 7e65698053b5d02403e33c7e49b382d6eaac7ec4 Mon Sep 17 00:00:00 2001 From: Matt Jankowski Date: Wed, 1 May 2024 10:38:48 -0400 Subject: [PATCH 44/59] Relax allowed `elasticsearch` dependency version (#933) * Relax allowed `elasticsearch` dependency version * Update tracer/logger access style * Add changelog notes --- CHANGELOG.md | 2 ++ chewy.gemspec | 2 +- lib/chewy/config.rb | 4 ++-- spec/chewy/config_spec.rb | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 073d61140..ca2374640 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ ### Changes +* [#933](https://github.com/toptal/chewy/pull/933): Relax allowed `elasticsearch` dependency versions. ([@mjankowski][]) + ### Bugs Fixed ## 7.5.1 (2024-01-30) diff --git a/chewy.gemspec b/chewy.gemspec index 866ed3c36..675081dde 100644 --- a/chewy.gemspec +++ b/chewy.gemspec @@ -17,7 +17,7 @@ Gem::Specification.new do |spec| spec.require_paths = ['lib'] spec.add_dependency 'activesupport', '>= 5.2' # Remove with major version bump, 8.x - spec.add_dependency 'elasticsearch', '>= 7.12.0', '< 7.14.0' + spec.add_dependency 'elasticsearch', '>= 7.14.0', '< 8' spec.add_dependency 'elasticsearch-dsl' spec.metadata['rubygems_mfa_required'] = 'true' end diff --git a/lib/chewy/config.rb b/lib/chewy/config.rb index fdae4ae45..1ec687e1b 100644 --- a/lib/chewy/config.rb +++ b/lib/chewy/config.rb @@ -70,12 +70,12 @@ def initialize end def transport_logger=(logger) - Chewy.client.transport.logger = logger + Chewy.client.transport.transport.logger = logger @transport_logger = logger end def transport_tracer=(tracer) - Chewy.client.transport.tracer = tracer + Chewy.client.transport.transport.tracer = tracer @transport_tracer = tracer end diff --git a/spec/chewy/config_spec.rb b/spec/chewy/config_spec.rb index f9d311342..1d0321d80 100644 --- a/spec/chewy/config_spec.rb +++ b/spec/chewy/config_spec.rb @@ -22,7 +22,7 @@ specify do expect { subject.transport_logger = logger } - .to change { Chewy.client.transport.logger }.to(logger) + .to change { Chewy.client.transport.transport.logger }.to(logger) end specify do expect { subject.transport_logger = logger } @@ -40,7 +40,7 @@ specify do expect { subject.transport_tracer = tracer } - .to change { Chewy.client.transport.tracer }.to(tracer) + .to change { Chewy.client.transport.transport.tracer }.to(tracer) end specify do expect { subject.transport_tracer = tracer } From f45460f0c731776d6d5a2f6babff878d9a0cf74a Mon Sep 17 00:00:00 2001 From: Matt Jankowski Date: Thu, 2 May 2024 04:04:11 -0400 Subject: [PATCH 45/59] Fix intermittent time-based failure in delayed sidekiq spec (#947) * Fix intermittent time-based failure in delayed sidekiq spec This example was previously failing when it was run at exactly an "even" 10 second increment time. For example, it would fail at "12:00:10", but pass at "12:00:09" and "12:00:11". This leads to intermittent failures on CI (presumably around ~10% of runs). Updated to use a helper method in the spec which more closely mirrors the scheduler code. * Add changelog notes --- CHANGELOG.md | 2 ++ spec/chewy/strategy/delayed_sidekiq_spec.rb | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca2374640..bfe3a60f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ ### Bugs Fixed +* [#947](https://github.com/toptal/chewy/pull/947): Fix intermittent time-based failure in delayed sidekiq spec. ([@mjankowski][]) + ## 7.5.1 (2024-01-30) ### New Features diff --git a/spec/chewy/strategy/delayed_sidekiq_spec.rb b/spec/chewy/strategy/delayed_sidekiq_spec.rb index c0d43e576..ae7a07dc7 100644 --- a/spec/chewy/strategy/delayed_sidekiq_spec.rb +++ b/spec/chewy/strategy/delayed_sidekiq_spec.rb @@ -47,7 +47,7 @@ expect(Sidekiq::Client).to receive(:push).with( hash_including( 'queue' => 'chewy', - 'at' => (Time.current.to_i.ceil(-1) + 2.seconds).to_i, + 'at' => expected_at_time.to_i, 'class' => Chewy::Strategy::DelayedSidekiq::Worker, 'args' => ['CitiesIndex', an_instance_of(Integer)] ) @@ -62,6 +62,11 @@ end end end + + def expected_at_time + target = described_class::Scheduler::DEFAULT_LATENCY.seconds.from_now.to_i + target - (target % described_class::Scheduler::DEFAULT_LATENCY) + described_class::Scheduler::DEFAULT_MARGIN.seconds + end end context 'with custom config' do From 04e598404e07541a820f7693fc56e4d1f0fedadc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 13:19:13 +0300 Subject: [PATCH 46/59] Update rubocop requirement from 1.63.3 to 1.63.4 (#946) Updates the requirements on [rubocop](https://github.com/rubocop/rubocop) to permit the latest version. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.63.3...v1.63.4) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gemfiles/base.gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemfiles/base.gemfile b/gemfiles/base.gemfile index ce0e6e889..37da307c5 100644 --- a/gemfiles/base.gemfile +++ b/gemfiles/base.gemfile @@ -6,7 +6,7 @@ gem 'rake' gem 'rspec', '>= 3.7.0' gem 'rspec-collection_matchers' gem 'rspec-its' -gem 'rubocop', '1.63.3' +gem 'rubocop', '1.63.4' gem 'sqlite3', '~> 1.4' gem 'timecop' gem 'unparser' From c99741e5badc97c58c40536fe956132bd58a4b95 Mon Sep 17 00:00:00 2001 From: Viktor Sych Date: Fri, 3 May 2024 11:15:54 +0300 Subject: [PATCH 47/59] [fix] delayed_sidekiq race condition (#937) --- .github/workflows/ruby.yml | 12 +++++ CHANGELOG.md | 1 + README.md | 9 ++-- gemfiles/base.gemfile | 2 +- lib/chewy/strategy/delayed_sidekiq.rb | 8 +-- .../strategy/delayed_sidekiq/scheduler.rb | 49 +++++++++++++------ lib/chewy/strategy/delayed_sidekiq/worker.rb | 36 +++++++++++--- spec/chewy/strategy/delayed_sidekiq_spec.rb | 13 ++--- 8 files changed, 94 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 979dadf97..825a8d4a8 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -23,6 +23,18 @@ jobs: env: BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile + services: + redis: + # Docker Hub image + image: redis + ports: + - '6379:6379' + # Set health checks to wait until redis has started + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index bfe3a60f2..1df3819ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * [#933](https://github.com/toptal/chewy/pull/933): Relax allowed `elasticsearch` dependency versions. ([@mjankowski][]) ### Bugs Fixed +* [#937](https://github.com/toptal/chewy/pull/937): Fix for race condition while using the `delayed_sidekiq` strategy. Also, fix for Redis bloating in case of reindexing error ([@skcc321](https://github.com/skcc321)) * [#947](https://github.com/toptal/chewy/pull/947): Fix intermittent time-based failure in delayed sidekiq spec. ([@mjankowski][]) diff --git a/README.md b/README.md index 4082fd038..23c13993e 100644 --- a/README.md +++ b/README.md @@ -776,9 +776,12 @@ Chewy.settings[:sidekiq] = {queue: :low} #### `:delayed_sidekiq` -It accumulates ids of records to be reindexed during the latency window in redis and then does the reindexing of all accumulated records at once. -The strategy is very useful in case of frequently mutated records. -It supports `update_fields` option, so it will try to select just enough data from the DB +It accumulates IDs of records to be reindexed during the latency window in Redis and then performs the reindexing of all accumulated records at once. +This strategy is very useful in the case of frequently mutated records. +It supports the `update_fields` option, so it will attempt to select just enough data from the database. + +Keep in mind, this strategy does not guarantee reindexing in the event of Sidekiq worker termination or an error during the reindexing phase. +This behavior is intentional to prevent continuous growth of Redis db. There are three options that can be defined in the index: ```ruby diff --git a/gemfiles/base.gemfile b/gemfiles/base.gemfile index 37da307c5..f4e0c5075 100644 --- a/gemfiles/base.gemfile +++ b/gemfiles/base.gemfile @@ -1,8 +1,8 @@ gem 'database_cleaner' gem 'elasticsearch-extensions' gem 'method_source' -gem 'mock_redis' gem 'rake' +gem 'redis', require: false gem 'rspec', '>= 3.7.0' gem 'rspec-collection_matchers' gem 'rspec-its' diff --git a/lib/chewy/strategy/delayed_sidekiq.rb b/lib/chewy/strategy/delayed_sidekiq.rb index 0bdd25c88..23b373b2d 100644 --- a/lib/chewy/strategy/delayed_sidekiq.rb +++ b/lib/chewy/strategy/delayed_sidekiq.rb @@ -9,11 +9,11 @@ class DelayedSidekiq < Sidekiq # leak and potential flaky tests. def self.clear_timechunks! ::Sidekiq.redis do |redis| - timechunk_sets = redis.smembers(Chewy::Strategy::DelayedSidekiq::Scheduler::ALL_SETS_KEY) - break if timechunk_sets.empty? + keys_to_delete = redis.keys("#{Scheduler::KEY_PREFIX}*") - redis.pipelined do |pipeline| - timechunk_sets.each { |set| pipeline.del(set) } + # Delete keys one by one + keys_to_delete.each do |key| + redis.del(key) end end end diff --git a/lib/chewy/strategy/delayed_sidekiq/scheduler.rb b/lib/chewy/strategy/delayed_sidekiq/scheduler.rb index f1010a3ee..d931c338b 100644 --- a/lib/chewy/strategy/delayed_sidekiq/scheduler.rb +++ b/lib/chewy/strategy/delayed_sidekiq/scheduler.rb @@ -12,13 +12,43 @@ class Strategy class DelayedSidekiq require_relative 'worker' + LUA_SCRIPT = <<~LUA + local timechunk_key = KEYS[1] + local timechunks_key = KEYS[2] + local serialize_data = ARGV[1] + local at = ARGV[2] + local ttl = tonumber(ARGV[3]) + + local schedule_job = false + + -- Check if the 'sadd?' method is available + if redis.call('exists', 'sadd?') == 1 then + redis.call('sadd?', timechunk_key, serialize_data) + else + redis.call('sadd', timechunk_key, serialize_data) + end + + -- Set expiration for timechunk_key + redis.call('expire', timechunk_key, ttl) + + -- Check if timechunk_key exists in the sorted set + if not redis.call('zrank', timechunks_key, timechunk_key) then + -- Add timechunk_key to the sorted set + redis.call('zadd', timechunks_key, at, timechunk_key) + -- Set expiration for timechunks_key + redis.call('expire', timechunks_key, ttl) + schedule_job = true + end + + return schedule_job + LUA + class Scheduler DEFAULT_TTL = 60 * 60 * 24 # in seconds DEFAULT_LATENCY = 10 DEFAULT_MARGIN = 2 DEFAULT_QUEUE = 'chewy' KEY_PREFIX = 'chewy:delayed_sidekiq' - ALL_SETS_KEY = "#{KEY_PREFIX}:all_sets".freeze FALLBACK_FIELDS = 'all' FIELDS_IDS_SEPARATOR = ';' IDS_SEPARATOR = ',' @@ -67,21 +97,8 @@ def initialize(type, ids, options = {}) # | chewy:delayed_sidekiq:CitiesIndex:1679347868 def postpone ::Sidekiq.redis do |redis| - # warning: Redis#sadd will always return an Integer in Redis 5.0.0. Use Redis#sadd? instead - if redis.respond_to?(:sadd?) - redis.sadd?(ALL_SETS_KEY, timechunks_key) - redis.sadd?(timechunk_key, serialize_data) - else - redis.sadd(ALL_SETS_KEY, timechunks_key) - redis.sadd(timechunk_key, serialize_data) - end - - redis.expire(timechunk_key, ttl) - - unless redis.zrank(timechunks_key, timechunk_key) - redis.zadd(timechunks_key, at, timechunk_key) - redis.expire(timechunks_key, ttl) - + # do the redis stuff in a single command to avoid concurrency issues + if redis.eval(LUA_SCRIPT, keys: [timechunk_key, timechunks_key], argv: [serialize_data, at, ttl]) ::Sidekiq::Client.push( 'queue' => sidekiq_queue, 'at' => at + margin, diff --git a/lib/chewy/strategy/delayed_sidekiq/worker.rb b/lib/chewy/strategy/delayed_sidekiq/worker.rb index 4d17a4cd1..af5fa793d 100644 --- a/lib/chewy/strategy/delayed_sidekiq/worker.rb +++ b/lib/chewy/strategy/delayed_sidekiq/worker.rb @@ -6,13 +6,40 @@ class DelayedSidekiq class Worker include ::Sidekiq::Worker + LUA_SCRIPT = <<~LUA + local type = ARGV[1] + local score = tonumber(ARGV[2]) + local prefix = ARGV[3] + local timechunks_key = prefix .. ":" .. type .. ":timechunks" + + -- Get timechunk_keys with scores less than or equal to the specified score + local timechunk_keys = redis.call('zrangebyscore', timechunks_key, '-inf', score) + + -- Get all members from the sets associated with the timechunk_keys + local members = {} + for _, timechunk_key in ipairs(timechunk_keys) do + local set_members = redis.call('smembers', timechunk_key) + for _, member in ipairs(set_members) do + table.insert(members, member) + end + end + + -- Remove timechunk_keys and their associated sets + for _, timechunk_key in ipairs(timechunk_keys) do + redis.call('del', timechunk_key) + end + + -- Remove timechunks with scores less than or equal to the specified score + redis.call('zremrangebyscore', timechunks_key, '-inf', score) + + return members + LUA + def perform(type, score, options = {}) options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async ::Sidekiq.redis do |redis| - timechunks_key = "#{Scheduler::KEY_PREFIX}:#{type}:timechunks" - timechunk_keys = redis.zrangebyscore(timechunks_key, -1, score) - members = timechunk_keys.flat_map { |timechunk_key| redis.smembers(timechunk_key) }.compact + members = redis.eval(LUA_SCRIPT, keys: [], argv: [type, score, Scheduler::KEY_PREFIX]) # extract ids and fields & do the reset of records ids, fields = extract_ids_and_fields(members) @@ -22,9 +49,6 @@ def perform(type, score, options = {}) index.strategy_config.delayed_sidekiq.reindex_wrapper.call do options.any? ? index.import!(ids, **options) : index.import!(ids) end - - redis.del(timechunk_keys) - redis.zremrangebyscore(timechunks_key, -1, score) end end diff --git a/spec/chewy/strategy/delayed_sidekiq_spec.rb b/spec/chewy/strategy/delayed_sidekiq_spec.rb index ae7a07dc7..78a1219b8 100644 --- a/spec/chewy/strategy/delayed_sidekiq_spec.rb +++ b/spec/chewy/strategy/delayed_sidekiq_spec.rb @@ -2,7 +2,7 @@ if defined?(Sidekiq) require 'sidekiq/testing' - require 'mock_redis' + require 'redis' describe Chewy::Strategy::DelayedSidekiq do around do |example| @@ -10,9 +10,10 @@ end before do - redis = MockRedis.new + redis = Redis.new allow(Sidekiq).to receive(:redis).and_yield(redis) Sidekiq::Worker.clear_all + described_class.clear_timechunks! end before do @@ -35,7 +36,7 @@ it "respects 'refresh: false' options" do allow(Chewy).to receive(:disable_refresh_async).and_return(true) - expect(CitiesIndex).to receive(:import!).with([city.id, other_city.id], refresh: false) + expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id]), refresh: false) scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id, other_city.id]) scheduler.postpone Chewy::Strategy::DelayedSidekiq::Worker.drain @@ -108,7 +109,7 @@ def expected_at_time context 'two reindex call within the timewindow' do it 'accumulates all ids does the reindex one time' do Timecop.freeze do - expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id]).once + expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id])).once scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id]) scheduler.postpone scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id]) @@ -120,7 +121,7 @@ def expected_at_time context 'one call with update_fields another one without update_fields' do it 'does reindex of all fields' do Timecop.freeze do - expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id]).once + expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id])).once scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name']) scheduler.postpone scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id]) @@ -133,7 +134,7 @@ def expected_at_time context 'both calls with different update fields' do it 'deos reindex with union of fields' do Timecop.freeze do - expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id], update_fields: %w[description name]).once + expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id]), update_fields: %w[name description]).once scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name']) scheduler.postpone scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id], update_fields: ['description']) From 17cd230dbd495894491c976cb32215aa10e5ceaf Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Mon, 6 May 2024 11:20:33 +0300 Subject: [PATCH 48/59] Bump version to 7.6.0 (#948) --- CHANGELOG.md | 6 ++++++ lib/chewy/version.rb | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1df3819ae..25291ca22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ ### Changes +### Bugs Fixed + +## 7.6.0 (2024-05-03) + +### Changes + * [#933](https://github.com/toptal/chewy/pull/933): Relax allowed `elasticsearch` dependency versions. ([@mjankowski][]) ### Bugs Fixed diff --git a/lib/chewy/version.rb b/lib/chewy/version.rb index e5d371949..a0c87a408 100644 --- a/lib/chewy/version.rb +++ b/lib/chewy/version.rb @@ -1,3 +1,3 @@ module Chewy - VERSION = '7.5.1'.freeze + VERSION = '7.6.0'.freeze end From a51f162075e53b79e13aa7726ef98d67948febf8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 14:52:33 +0800 Subject: [PATCH 49/59] Update rubocop requirement from 1.63.4 to 1.63.5 (#949) Updates the requirements on [rubocop](https://github.com/rubocop/rubocop) to permit the latest version. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.63.4...v1.63.5) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gemfiles/base.gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemfiles/base.gemfile b/gemfiles/base.gemfile index f4e0c5075..ed4fbf8fb 100644 --- a/gemfiles/base.gemfile +++ b/gemfiles/base.gemfile @@ -6,7 +6,7 @@ gem 'redis', require: false gem 'rspec', '>= 3.7.0' gem 'rspec-collection_matchers' gem 'rspec-its' -gem 'rubocop', '1.63.4' +gem 'rubocop', '1.63.5' gem 'sqlite3', '~> 1.4' gem 'timecop' gem 'unparser' From 86f9e3c64941732a9a12c350df445b343a1e3661 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 13:45:22 +0100 Subject: [PATCH 50/59] Update rubocop requirement from 1.63.5 to 1.64.1 (#953) * Update rubocop requirement from 1.63.5 to 1.64.1 Updates the requirements on [rubocop](https://github.com/rubocop/rubocop) to permit the latest version. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.63.5...v1.64.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Fix rubocop --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Samuel Ebeagu --- gemfiles/base.gemfile | 2 +- lib/chewy/fields/root.rb | 2 +- spec/chewy/fields/base_spec.rb | 4 ++-- spec/chewy/index/observe/callback_spec.rb | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gemfiles/base.gemfile b/gemfiles/base.gemfile index ed4fbf8fb..d537eb66e 100644 --- a/gemfiles/base.gemfile +++ b/gemfiles/base.gemfile @@ -6,7 +6,7 @@ gem 'redis', require: false gem 'rspec', '>= 3.7.0' gem 'rspec-collection_matchers' gem 'rspec-its' -gem 'rubocop', '1.63.5' +gem 'rubocop', '1.64.1' gem 'sqlite3', '~> 1.4' gem 'timecop' gem 'unparser' diff --git a/lib/chewy/fields/root.rb b/lib/chewy/fields/root.rb index d6e125ba8..e821cb9ca 100644 --- a/lib/chewy/fields/root.rb +++ b/lib/chewy/fields/root.rb @@ -4,7 +4,7 @@ class Root < Chewy::Fields::Base attr_reader :dynamic_templates, :id def initialize(name, **options) - super(name, **options) + super @value ||= -> { self } @dynamic_templates = [] diff --git a/spec/chewy/fields/base_spec.rb b/spec/chewy/fields/base_spec.rb index bd8149bd6..cf82bde2a 100644 --- a/spec/chewy/fields/base_spec.rb +++ b/spec/chewy/fields/base_spec.rb @@ -5,7 +5,7 @@ specify { expect(described_class.new('name', type: 'integer').options[:type]).to eq('integer') } describe '#compose' do - let(:field) { described_class.new(:name, value: ->(o) { o.value }) } + let(:field) { described_class.new(:name, value: lambda(&:value)) } specify { expect(field.compose(double(value: 'hello'))).to eq(name: 'hello') } specify { expect(field.compose(double(value: %w[hello world]))).to eq(name: %w[hello world]) } @@ -23,7 +23,7 @@ context 'nested fields' do before do - field.children.push(described_class.new(:subname1, value: ->(o) { o.subvalue1 })) + field.children.push(described_class.new(:subname1, value: lambda(&:subvalue1))) field.children.push(described_class.new(:subname2, value: -> { subvalue2 })) field.children.push(described_class.new(:subname3)) end diff --git a/spec/chewy/index/observe/callback_spec.rb b/spec/chewy/index/observe/callback_spec.rb index 825e3b405..84cd6219b 100644 --- a/spec/chewy/index/observe/callback_spec.rb +++ b/spec/chewy/index/observe/callback_spec.rb @@ -21,7 +21,7 @@ end context 'when executable is has arity 1' do - let(:executable) { ->(record) { record.population } } + let(:executable) { lambda(&:population) } it 'calls exectuable within context' do expect(callback.call(city)).to eq(city.population) From 2d6a3f5397d9844e0bbc16357b7c52cf2309df65 Mon Sep 17 00:00:00 2001 From: Diego Guerra Date: Wed, 10 Jul 2024 15:58:46 +0800 Subject: [PATCH 51/59] Rename team (#957) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4968d730e..2ef2a3547 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -.github/workflows @toptal/platform-sre +.github/workflows @toptal/sre From 7b4ae3020d7c8daeeae7151ce5b2ab247f0011a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 10:45:43 +0800 Subject: [PATCH 52/59] Update rubocop requirement from 1.64.1 to 1.65.0 (#958) Updates the requirements on [rubocop](https://github.com/rubocop/rubocop) to permit the latest version. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.64.1...v1.65.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gemfiles/base.gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemfiles/base.gemfile b/gemfiles/base.gemfile index d537eb66e..b6c490e82 100644 --- a/gemfiles/base.gemfile +++ b/gemfiles/base.gemfile @@ -6,7 +6,7 @@ gem 'redis', require: false gem 'rspec', '>= 3.7.0' gem 'rspec-collection_matchers' gem 'rspec-its' -gem 'rubocop', '1.64.1' +gem 'rubocop', '1.65.0' gem 'sqlite3', '~> 1.4' gem 'timecop' gem 'unparser' From 0c50cb8d85d86c27dd664abc876f5faa83bdd4fa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Aug 2024 09:53:44 +0200 Subject: [PATCH 53/59] Update rubocop requirement from 1.65.0 to 1.65.1 (#961) Updates the requirements on [rubocop](https://github.com/rubocop/rubocop) to permit the latest version. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.65.0...v1.65.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gemfiles/base.gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemfiles/base.gemfile b/gemfiles/base.gemfile index b6c490e82..83269e4bf 100644 --- a/gemfiles/base.gemfile +++ b/gemfiles/base.gemfile @@ -6,7 +6,7 @@ gem 'redis', require: false gem 'rspec', '>= 3.7.0' gem 'rspec-collection_matchers' gem 'rspec-its' -gem 'rubocop', '1.65.0' +gem 'rubocop', '1.65.1' gem 'sqlite3', '~> 1.4' gem 'timecop' gem 'unparser' From 84c0eed871e835f55d560097ef141cd5fe504967 Mon Sep 17 00:00:00 2001 From: Alexander Sviridov Date: Fri, 30 Aug 2024 12:03:03 +0300 Subject: [PATCH 54/59] ElasticSearch v. 8 upgrade (#962) * ES 8.x upgrade * Fix termnite after changes * fixup es * Elastic security on: configuration example * Mass index deletion disabled by default in Elastic * Elastic docker file: increase sleep, bump image * Spec fixes * bump dependency * Bump version, update docs --------- Co-authored-by: Danil Nurgaliev --- .github/workflows/ruby.yml | 10 +++--- CHANGELOG.md | 12 +++++++ README.md | 34 ++++++++++++++++++- chewy.gemspec | 5 +-- docker-compose.yml | 15 ++++++++ lib/chewy.rb | 4 +++ lib/chewy/config.rb | 4 +-- lib/chewy/errors.rb | 3 ++ lib/chewy/index/actions.rb | 10 +++--- lib/chewy/index/aliases.rb | 2 +- lib/chewy/index/syncer.rb | 2 +- lib/chewy/minitest/helpers.rb | 2 +- lib/chewy/search/request.rb | 8 ++--- lib/chewy/search/response.rb | 7 ++++ lib/chewy/search/scrolling.rb | 3 +- spec/chewy/config_spec.rb | 4 +-- spec/chewy/elastic_client_spec.rb | 2 +- spec/chewy/fields/time_fields_spec.rb | 2 +- spec/chewy/index/actions_spec.rb | 18 +++++----- spec/chewy/index/aliases_spec.rb | 2 +- spec/chewy/index/import/bulk_builder_spec.rb | 4 +-- spec/chewy/index/import/bulk_request_spec.rb | 2 +- spec/chewy/index/import/routine_spec.rb | 2 +- spec/chewy/index/import_spec.rb | 30 ++++++++-------- spec/chewy/index/specification_spec.rb | 5 +-- spec/chewy/index/syncer_spec.rb | 2 +- spec/chewy/index_spec.rb | 2 +- spec/chewy/journal_spec.rb | 4 +-- spec/chewy/minitest/helpers_spec.rb | 2 +- spec/chewy/multi_search_spec.rb | 2 +- spec/chewy/rake_helper_spec.rb | 2 +- spec/chewy/rspec/update_index_spec.rb | 2 +- spec/chewy/runtime_spec.rb | 4 +-- spec/chewy/search/loader_spec.rb | 2 +- .../search/pagination/kaminari_examples.rb | 2 +- spec/chewy/search/request_spec.rb | 2 +- spec/chewy/search/response_spec.rb | 4 +-- spec/chewy/search/scrolling_spec.rb | 2 +- spec/chewy/search_spec.rb | 2 +- spec/chewy/stash_spec.rb | 2 +- spec/chewy/strategy/delayed_sidekiq_spec.rb | 2 +- spec/chewy/strategy_spec.rb | 2 +- spec/chewy_spec.rb | 11 +++--- spec/spec_helper.rb | 26 ++++++++++++++ 44 files changed, 184 insertions(+), 85 deletions(-) create mode 100644 docker-compose.yml diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 825a8d4a8..c9293a9aa 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -41,11 +41,11 @@ jobs: with: ruby-version: ${{ matrix.ruby }} bundler-cache: true - - name: Run Elasticsearch - uses: elastic/elastic-github-actions/elasticsearch@9de0f78f306e4ebc0838f057e6b754364685e759 - with: - stack-version: 7.10.1 - port: 9250 + - name: Start containers + run: | + docker compose up elasticsearch_test -d + sleep 15 + - name: Tests run: bundle exec rspec diff --git a/CHANGELOG.md b/CHANGELOG.md index 25291ca22..0c27c85fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,18 @@ ### Bugs Fixed +## 8.0.0-beta (2024-08-27) + +### New Features + +* [#962](https://github.com/toptal/chewy/pull/962): ElasticSearch v.8 support added + +* `delete_all_enabled` setting introduced to align Chewy.massacre with wildcard indices deletion disabled in ES 8 by default + +### Changes + +### Bugs Fixed + ## 7.6.0 (2024-05-03) ### Changes diff --git a/README.md b/README.md index 23c13993e..da00cef6c 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ Chewy is compatible with MRI 3.0-3.2¹. | Chewy version | Elasticsearch version | | ------------- | ---------------------------------- | +| 8.0.0 | 8.x | | 7.2.x | 7.x | | 7.1.x | 7.x | | 7.0.x | 6.8, 7.x | @@ -97,7 +98,36 @@ development: Make sure you have Elasticsearch up and running. You can [install](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html) it locally, but the easiest way is to use [Docker](https://www.docker.com/get-started): ```shell -$ docker run --rm --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:7.11.1 +$ docker run --rm --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "xpack.security.enabled=false" elasticsearch:8.15.0 +``` + +### Security + +Please note that starting from version 8 ElasticSearch has security features enabled by default. +Docker command above has it disabled for local testing convenience. If you want to enable it, omit +`"xpack.security.enabled=false"` part from Docker command, and run these command after starting container (container name `es8` assumed): + +Reset password for `elastic` user: +``` +docker container exec es8 '/usr/share/elasticsearch/bin/elasticsearch-reset-password' -u elastic +``` + +Extract CA certificate generated by ElasticSearch on first run: +``` +docker container cp es8:/usr/share/elasticsearch/config/certs/http_ca.crt tmp/ +``` + +And then add them to settings: + +```yaml +# config/chewy.yml +development: + host: 'localhost:9200' + user: 'elastic' + password: 'SomeLongPassword' + transport_options: + ssl: + ca_file: './tmp/http_ca.crt' ``` ### Index @@ -941,6 +971,8 @@ Controller actions are wrapped with the configurable value of `Chewy.request_str It is also a good idea to set up the `:bypass` strategy inside your test suite and import objects manually only when needed, and use `Chewy.massacre` when needed to flush test ES indices before every example. This will allow you to minimize unnecessary ES requests and reduce overhead. +Deprecation note: since version 8 wildcard removing of indices is disabled by default. You can enable it for a cluster with setting `action.destructive_requires_name` to false. + ```ruby RSpec.configure do |config| config.before(:suite) do diff --git a/chewy.gemspec b/chewy.gemspec index 675081dde..decbe7355 100644 --- a/chewy.gemspec +++ b/chewy.gemspec @@ -11,13 +11,14 @@ Gem::Specification.new do |spec| spec.description = 'Chewy provides functionality for Elasticsearch index handling, documents import mappings and chainable query DSL' spec.homepage = 'https://github.com/toptal/chewy' spec.license = 'MIT' + spec.required_ruby_version = '~> 3.0' spec.files = `git ls-files`.split($RS) spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } spec.require_paths = ['lib'] - spec.add_dependency 'activesupport', '>= 5.2' # Remove with major version bump, 8.x - spec.add_dependency 'elasticsearch', '>= 7.14.0', '< 8' + spec.add_dependency 'activesupport', '>= 6.1' + spec.add_dependency 'elasticsearch', '>= 8.14', '< 9.0' spec.add_dependency 'elasticsearch-dsl' spec.metadata['rubygems_mfa_required'] = 'true' end diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..0c1309de2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +version: "3.4" +services: + elasticsearch_test: + image: "elasticsearch:8.15.0" + environment: + - bootstrap.memory_lock=${ES_MEMORY_LOCK:-false} + - "ES_JAVA_OPTS=-Xms${TEST_ES_HEAP_SIZE:-500m} -Xmx${TEST_ES_HEAP_SIZE:-500m}" + - discovery.type=single-node + - xpack.security.enabled=false + ports: + - "127.0.0.1:9250:9200" + ulimits: + nofile: + soft: 65536 + hard: 65536 diff --git a/lib/chewy.rb b/lib/chewy.rb index 919f9bac8..dfeed3416 100644 --- a/lib/chewy.rb +++ b/lib/chewy.rb @@ -116,6 +116,10 @@ def wait_for_status # Be careful, if current prefix is blank, this will destroy all the indexes. # def massacre + unless Chewy.settings[:delete_all_enabled] + raise FeatureDisabled, 'Feature disabled by default in ES 8. You can enable it in the cluster and set `delete_all_enabled` option in settings' + end + Chewy.client.indices.delete(index: [Chewy.configuration[:prefix], '*'].reject(&:blank?).join('_')) Chewy.wait_for_status end diff --git a/lib/chewy/config.rb b/lib/chewy/config.rb index 1ec687e1b..fdae4ae45 100644 --- a/lib/chewy/config.rb +++ b/lib/chewy/config.rb @@ -70,12 +70,12 @@ def initialize end def transport_logger=(logger) - Chewy.client.transport.transport.logger = logger + Chewy.client.transport.logger = logger @transport_logger = logger end def transport_tracer=(tracer) - Chewy.client.transport.transport.tracer = tracer + Chewy.client.transport.tracer = tracer @transport_tracer = tracer end diff --git a/lib/chewy/errors.rb b/lib/chewy/errors.rb index d470bd94f..e16848794 100644 --- a/lib/chewy/errors.rb +++ b/lib/chewy/errors.rb @@ -39,4 +39,7 @@ def initialize(join_field_type, join_field_name, relations) class ImportScopeCleanupError < Error end + + class FeatureDisabled < Error + end end diff --git a/lib/chewy/index/actions.rb b/lib/chewy/index/actions.rb index afc7debcc..a146f47cc 100644 --- a/lib/chewy/index/actions.rb +++ b/lib/chewy/index/actions.rb @@ -32,7 +32,7 @@ def exists? # def create(*args, **kwargs) create!(*args, **kwargs) - rescue Elasticsearch::Transport::Transport::Errors::BadRequest + rescue Elastic::Transport::Transport::Errors::BadRequest false end @@ -83,9 +83,9 @@ def delete(suffix = nil) result = client.indices.delete index: index_names.join(',') Chewy.wait_for_status if result result - # es-ruby >= 1.0.10 handles Elasticsearch::Transport::Transport::Errors::NotFound + # es-ruby >= 1.0.10 handles Elastic::Transport::Transport::Errors::NotFound # by itself, rescue is for previous versions - rescue Elasticsearch::Transport::Transport::Errors::NotFound + rescue Elastic::Transport::Transport::Errors::NotFound false end @@ -99,9 +99,9 @@ def delete(suffix = nil) # UsersIndex.delete '01-2014' # deletes `users_01-2014` index # def delete!(suffix = nil) - # es-ruby >= 1.0.10 handles Elasticsearch::Transport::Transport::Errors::NotFound + # es-ruby >= 1.0.10 handles Elastic::Transport::Transport::Errors::NotFound # by itself, so it is raised here - delete(suffix) or raise Elasticsearch::Transport::Transport::Errors::NotFound + delete(suffix) or raise Elastic::Transport::Transport::Errors::NotFound end # Deletes and recreates index. Supports suffixes. diff --git a/lib/chewy/index/aliases.rb b/lib/chewy/index/aliases.rb index cfcf74732..9b4a2caef 100644 --- a/lib/chewy/index/aliases.rb +++ b/lib/chewy/index/aliases.rb @@ -22,7 +22,7 @@ def aliases def empty_if_not_found yield - rescue Elasticsearch::Transport::Transport::Errors::NotFound + rescue Elastic::Transport::Transport::Errors::NotFound [] end end diff --git a/lib/chewy/index/syncer.rb b/lib/chewy/index/syncer.rb index b8365bb50..47408c9af 100644 --- a/lib/chewy/index/syncer.rb +++ b/lib/chewy/index/syncer.rb @@ -213,7 +213,7 @@ def outdated_sync_field_type @outdated_sync_field_type = mappings .fetch('properties', {}) .fetch(@index.outdated_sync_field.to_s, {})['type'] - rescue Elasticsearch::Transport::Transport::Errors::NotFound + rescue Elastic::Transport::Transport::Errors::NotFound nil end end diff --git a/lib/chewy/minitest/helpers.rb b/lib/chewy/minitest/helpers.rb index e9fbd1362..0254c8f97 100644 --- a/lib/chewy/minitest/helpers.rb +++ b/lib/chewy/minitest/helpers.rb @@ -142,7 +142,7 @@ def index_everything! teardown do # always destroy indexes between tests # Prevent croll pollution of test cases due to indexing - Chewy.massacre + drop_indices end end end diff --git a/lib/chewy/search/request.rb b/lib/chewy/search/request.rb index f3b1031b4..27d5cdefe 100644 --- a/lib/chewy/search/request.rb +++ b/lib/chewy/search/request.rb @@ -46,7 +46,7 @@ class Request delegate :hits, :wrappers, :objects, :records, :documents, :object_hash, :record_hash, :document_hash, - :total, :max_score, :took, :timed_out?, to: :response + :total, :max_score, :took, :timed_out?, :terminated_early?, to: :response delegate :each, :size, :to_a, :[], to: :wrappers alias_method :to_ary, :to_a alias_method :total_count, :total @@ -854,7 +854,7 @@ def count else Chewy.client.count(only(WHERE_STORAGES).render)['count'] end - rescue Elasticsearch::Transport::Transport::Errors::NotFound + rescue Elastic::Transport::Transport::Errors::NotFound 0 end @@ -891,7 +891,7 @@ def exists? def first(limit = UNDEFINED) request_limit = limit == UNDEFINED ? 1 : limit - if performed? && (request_limit <= size || size == total) + if performed? && (terminated_early? || request_limit <= size || size == total) limit == UNDEFINED ? wrappers.first : wrappers.first(limit) else result = except(EXTRA_STORAGES).limit(request_limit).to_a @@ -1035,7 +1035,7 @@ def perform(additional = {}) request_body = render.merge(additional) ActiveSupport::Notifications.instrument 'search_query.chewy', notification_payload(request: request_body) do Chewy.client.search(request_body) - rescue Elasticsearch::Transport::Transport::Errors::NotFound + rescue Elastic::Transport::Transport::Errors::NotFound {} end end diff --git a/lib/chewy/search/response.rb b/lib/chewy/search/response.rb index 0a5becb24..5409a113d 100644 --- a/lib/chewy/search/response.rb +++ b/lib/chewy/search/response.rb @@ -47,6 +47,13 @@ def timed_out? @timed_out ||= @body['timed_out'] end + # Has the request been terminated early? + # + # @return [true, false] + def terminated_early? + @terminated_early ||= @body['terminated_early'] + end + # The `suggest` response part. Returns empty hash if suggests # were not requested. # diff --git a/lib/chewy/search/scrolling.rb b/lib/chewy/search/scrolling.rb index 6074b3252..f0c738374 100644 --- a/lib/chewy/search/scrolling.rb +++ b/lib/chewy/search/scrolling.rb @@ -39,7 +39,8 @@ def scroll_batches(batch_size: Request::DEFAULT_BATCH_SIZE, scroll: Request::DEF hits = hits.first(last_batch_size) if last_batch_size != 0 && fetched >= total yield(hits) if hits.present? scroll_id = result['_scroll_id'] - break if fetched >= total + + break if result['terminated_early'] || fetched >= total result = perform_scroll(scroll: scroll, scroll_id: scroll_id) end diff --git a/spec/chewy/config_spec.rb b/spec/chewy/config_spec.rb index 1d0321d80..f9d311342 100644 --- a/spec/chewy/config_spec.rb +++ b/spec/chewy/config_spec.rb @@ -22,7 +22,7 @@ specify do expect { subject.transport_logger = logger } - .to change { Chewy.client.transport.transport.logger }.to(logger) + .to change { Chewy.client.transport.logger }.to(logger) end specify do expect { subject.transport_logger = logger } @@ -40,7 +40,7 @@ specify do expect { subject.transport_tracer = tracer } - .to change { Chewy.client.transport.transport.tracer }.to(tracer) + .to change { Chewy.client.transport.tracer }.to(tracer) end specify do expect { subject.transport_tracer = tracer } diff --git a/spec/chewy/elastic_client_spec.rb b/spec/chewy/elastic_client_spec.rb index 79d3946ba..58cdc0cc7 100644 --- a/spec/chewy/elastic_client_spec.rb +++ b/spec/chewy/elastic_client_spec.rb @@ -6,7 +6,7 @@ let!(:filter_previous_value) { Chewy.before_es_request_filter } before do - Chewy.massacre + drop_indices stub_index(:products) do field :id, type: :integer end diff --git a/spec/chewy/fields/time_fields_spec.rb b/spec/chewy/fields/time_fields_spec.rb index 31aef9777..a9ddc8c45 100644 --- a/spec/chewy/fields/time_fields_spec.rb +++ b/spec/chewy/fields/time_fields_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe 'Time fields' do - before { Chewy.massacre } + before { drop_indices } before do stub_index(:posts) do diff --git a/spec/chewy/index/actions_spec.rb b/spec/chewy/index/actions_spec.rb index 4a0d69893..27d5682b4 100644 --- a/spec/chewy/index/actions_spec.rb +++ b/spec/chewy/index/actions_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Index::Actions do - before { Chewy.massacre } + before { drop_indices } before do stub_index :dummies @@ -78,12 +78,12 @@ specify do expect do DummiesIndex.create! - end.to raise_error(Elasticsearch::Transport::Transport::Errors::BadRequest).with_message(/already exists.*dummies/) + end.to raise_error(Elastic::Transport::Transport::Errors::BadRequest).with_message(/already exists.*dummies/) end specify do expect do DummiesIndex.create!('2013') - end.to raise_error(Elasticsearch::Transport::Transport::Errors::BadRequest).with_message(/Invalid alias name \[dummies\]/) + end.to raise_error(Elastic::Transport::Transport::Errors::BadRequest).with_message(/Invalid alias name \[dummies\]/) end end @@ -100,7 +100,7 @@ specify do expect do DummiesIndex.create!('2013') - end.to raise_error(Elasticsearch::Transport::Transport::Errors::BadRequest).with_message(/already exists.*dummies_2013/) + end.to raise_error(Elastic::Transport::Transport::Errors::BadRequest).with_message(/already exists.*dummies_2013/) end specify { expect(DummiesIndex.create!('2014')['acknowledged']).to eq(true) } @@ -190,11 +190,11 @@ end describe '.delete!' do - specify { expect { DummiesIndex.delete! }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) } + specify { expect { DummiesIndex.delete! }.to raise_error(Elastic::Transport::Transport::Errors::NotFound) } specify do expect do DummiesIndex.delete!('2013') - end.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + end.to raise_error(Elastic::Transport::Transport::Errors::NotFound) end context do @@ -768,7 +768,7 @@ .to receive(:clear_cache) .and_call_original expect { CitiesIndex.clear_cache({index: unexisted_index_name}) } - .to raise_error Elasticsearch::Transport::Transport::Errors::NotFound + .to raise_error Elastic::Transport::Transport::Errors::NotFound end end @@ -820,7 +820,7 @@ .to receive(:reindex) .and_call_original expect { CitiesIndex.reindex(source: unexisting_index, dest: dest_index_with_prefix) } - .to raise_error Elasticsearch::Transport::Transport::Errors::NotFound + .to raise_error Elastic::Transport::Transport::Errors::NotFound end end @@ -883,7 +883,7 @@ context 'index name' do specify do expect { CitiesIndex.update_mapping(unexisting_index, body_hash) } - .to raise_error Elasticsearch::Transport::Transport::Errors::NotFound + .to raise_error Elastic::Transport::Transport::Errors::NotFound end end diff --git a/spec/chewy/index/aliases_spec.rb b/spec/chewy/index/aliases_spec.rb index dc29e1189..ad5419c7d 100644 --- a/spec/chewy/index/aliases_spec.rb +++ b/spec/chewy/index/aliases_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Index::Aliases do - before { Chewy.massacre } + before { drop_indices } before { stub_index :dummies } diff --git a/spec/chewy/index/import/bulk_builder_spec.rb b/spec/chewy/index/import/bulk_builder_spec.rb index 1b2eb3a82..5fe598bed 100644 --- a/spec/chewy/index/import/bulk_builder_spec.rb +++ b/spec/chewy/index/import/bulk_builder_spec.rb @@ -17,7 +17,7 @@ def derived end describe Chewy::Index::Import::BulkBuilder do - before { Chewy.massacre } + before { drop_indices } subject { described_class.new(index, to_index: to_index, delete: delete, fields: fields) } let(:index) { CitiesIndex } @@ -216,7 +216,7 @@ def derived end def do_raw_index_comment(options:, data:) - CommentsIndex.client.index(options.merge(index: 'comments', type: '_doc', refresh: true, body: data)) + CommentsIndex.client.index(options.merge(index: 'comments', refresh: true, body: data)) end def raw_index_comment(comment) diff --git a/spec/chewy/index/import/bulk_request_spec.rb b/spec/chewy/index/import/bulk_request_spec.rb index 0869804a1..bf33d2b16 100644 --- a/spec/chewy/index/import/bulk_request_spec.rb +++ b/spec/chewy/index/import/bulk_request_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Index::Import::BulkRequest do - before { Chewy.massacre } + before { drop_indices } subject { described_class.new(index, suffix: suffix, bulk_size: bulk_size, **bulk_options) } let(:suffix) {} diff --git a/spec/chewy/index/import/routine_spec.rb b/spec/chewy/index/import/routine_spec.rb index 5d1064b7b..55588e314 100644 --- a/spec/chewy/index/import/routine_spec.rb +++ b/spec/chewy/index/import/routine_spec.rb @@ -2,7 +2,7 @@ # TODO: add more specs here later describe Chewy::Index::Import::Routine do - before { Chewy.massacre } + before { drop_indices } before do stub_index(:cities) do field :name diff --git a/spec/chewy/index/import_spec.rb b/spec/chewy/index/import_spec.rb index 16593113a..b200a4599 100644 --- a/spec/chewy/index/import_spec.rb +++ b/spec/chewy/index/import_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Index::Import do - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) @@ -204,10 +204,10 @@ def subscribe_notification end end - let(:mapper_parsing_exception) do + let(:document_parsing_exception) do { - 'type' => 'mapper_parsing_exception', - 'reason' => 'object mapping for [name] tried to parse field [name] as object, but found a concrete value' + 'type' => 'document_parsing_exception', + 'reason' => '[1:9] object mapping for [name] tried to parse field [name] as object, but found a concrete value' } end @@ -215,7 +215,7 @@ def subscribe_notification payload = subscribe_notification import dummy_cities, batch_size: 2 expect(payload).to eq(index: CitiesIndex, - errors: {index: {mapper_parsing_exception => %w[1 2 3]}}, + errors: {index: {document_parsing_exception => %w[1 2 3]}}, import: {index: 3}) end end @@ -270,8 +270,8 @@ def subscribe_notification expect(payload).to eq( errors: { index: {{ - 'type' => 'mapper_parsing_exception', - 'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value' + 'type' => 'document_parsing_exception', + 'reason' => '[1:27] object mapping for [object] tried to parse field [object] as object, but found a concrete value' } => %w[2 4]} }, import: {index: 6}, @@ -293,8 +293,8 @@ def subscribe_notification expect(payload).to eq( errors: { index: {{ - 'type' => 'mapper_parsing_exception', - 'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value' + 'type' => 'document_parsing_exception', + 'reason' => '[1:27] object mapping for [object] tried to parse field [object] as object, but found a concrete value' } => %w[2 4]} }, import: {index: 6}, @@ -319,8 +319,8 @@ def subscribe_notification expect(payload).to eq( errors: { index: {{ - 'type' => 'mapper_parsing_exception', - 'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value' + 'type' => 'document_parsing_exception', + 'reason' => '[1:27] object mapping for [object] tried to parse field [object] as object, but found a concrete value' } => %w[2 4]} }, import: {index: 6}, @@ -383,8 +383,8 @@ def subscribe_notification # Full match doesn't work here. expect(payload[:errors][:update].keys).to match([ - hash_including('type' => 'document_missing_exception', 'reason' => '[_doc][1]: document missing'), - hash_including('type' => 'document_missing_exception', 'reason' => '[_doc][3]: document missing') + hash_including('type' => 'document_missing_exception', 'reason' => '[1]: document missing'), + hash_including('type' => 'document_missing_exception', 'reason' => '[3]: document missing') ]) expect(payload[:errors][:update].values).to eq([['1'], ['3']]) expect(imported_cities).to match_array([ @@ -431,8 +431,8 @@ def subscribe_notification expect(payload).to eq( errors: { update: {{ - 'type' => 'mapper_parsing_exception', - 'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value' + 'type' => 'document_parsing_exception', + 'reason' => '[1:26] object mapping for [object] tried to parse field [object] as object, but found a concrete value' } => %w[2 4]} }, import: {index: 6}, diff --git a/spec/chewy/index/specification_spec.rb b/spec/chewy/index/specification_spec.rb index 68a34b8d5..cd1b3bd7b 100644 --- a/spec/chewy/index/specification_spec.rb +++ b/spec/chewy/index/specification_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Index::Specification do - before { Chewy.massacre } + before { drop_indices } let(:index1) do stub_index(:places) do @@ -46,7 +46,6 @@ specify do expect { specification1.lock! }.to change { Chewy::Stash::Specification.all.hits }.from([]).to([{ '_index' => 'chewy_specifications', - '_type' => '_doc', '_id' => 'places', '_score' => 1.0, '_source' => {'specification' => Base64.encode64({ @@ -62,7 +61,6 @@ specify do expect { specification5.lock! }.to change { Chewy::Stash::Specification.all.hits }.to([{ '_index' => 'chewy_specifications', - '_type' => '_doc', '_id' => 'places', '_score' => 1.0, '_source' => {'specification' => Base64.encode64({ @@ -71,7 +69,6 @@ }.to_json)} }, { '_index' => 'chewy_specifications', - '_type' => '_doc', '_id' => 'namespace/cities', '_score' => 1.0, '_source' => {'specification' => Base64.encode64({ diff --git a/spec/chewy/index/syncer_spec.rb b/spec/chewy/index/syncer_spec.rb index e71617f04..176cf047a 100644 --- a/spec/chewy/index/syncer_spec.rb +++ b/spec/chewy/index/syncer_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Index::Syncer, :orm do - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) stub_index(:cities) do diff --git a/spec/chewy/index_spec.rb b/spec/chewy/index_spec.rb index 57565861b..d96d1bf6f 100644 --- a/spec/chewy/index_spec.rb +++ b/spec/chewy/index_spec.rb @@ -177,7 +177,7 @@ def self.by_id; end context do before do - Chewy.massacre + drop_indices PlacesIndex.import!( double(colors: ['red']), double(colors: %w[red green]), diff --git a/spec/chewy/journal_spec.rb b/spec/chewy/journal_spec.rb index 3d518be09..e392f8aba 100644 --- a/spec/chewy/journal_spec.rb +++ b/spec/chewy/journal_spec.rb @@ -21,7 +21,7 @@ default_import_options journal: true end - Chewy.massacre + drop_indices Chewy.settings[:prefix] = 'some_prefix' Timecop.freeze(time) end @@ -145,7 +145,7 @@ def timestamp(time) end context do - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) do update_index 'cities', :self diff --git a/spec/chewy/minitest/helpers_spec.rb b/spec/chewy/minitest/helpers_spec.rb index 4d6f2d30e..c411a14e4 100644 --- a/spec/chewy/minitest/helpers_spec.rb +++ b/spec/chewy/minitest/helpers_spec.rb @@ -17,7 +17,7 @@ def assert_equal(expected, actual, message) end before do - Chewy.massacre + drop_indices end before do diff --git a/spec/chewy/multi_search_spec.rb b/spec/chewy/multi_search_spec.rb index 8ca1e2cd2..2e7afaefb 100644 --- a/spec/chewy/multi_search_spec.rb +++ b/spec/chewy/multi_search_spec.rb @@ -2,7 +2,7 @@ require 'chewy/multi_search' describe Chewy::MultiSearch do - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) diff --git a/spec/chewy/rake_helper_spec.rb b/spec/chewy/rake_helper_spec.rb index f84ad129a..353164217 100644 --- a/spec/chewy/rake_helper_spec.rb +++ b/spec/chewy/rake_helper_spec.rb @@ -2,7 +2,7 @@ require 'rake' describe Chewy::RakeHelper, :orm do - before { Chewy.massacre } + before { drop_indices } before do described_class.instance_variable_set(:@journal_exists, journal_exists) diff --git a/spec/chewy/rspec/update_index_spec.rb b/spec/chewy/rspec/update_index_spec.rb index 5982f4c51..8054bcce6 100644 --- a/spec/chewy/rspec/update_index_spec.rb +++ b/spec/chewy/rspec/update_index_spec.rb @@ -9,7 +9,7 @@ end before do - Chewy.massacre + drop_indices DummiesIndex.create! end diff --git a/spec/chewy/runtime_spec.rb b/spec/chewy/runtime_spec.rb index edc85231b..e8cc457d3 100644 --- a/spec/chewy/runtime_spec.rb +++ b/spec/chewy/runtime_spec.rb @@ -3,7 +3,7 @@ describe Chewy::Runtime do describe '.version' do specify { expect(described_class.version).to be_a(described_class::Version) } - specify { expect(described_class.version).to be >= '7.0' } - specify { expect(described_class.version).to be < '8.0' } + specify { expect(described_class.version).to be >= '8.0' } + specify { expect(described_class.version).to be < '9.0' } end end diff --git a/spec/chewy/search/loader_spec.rb b/spec/chewy/search/loader_spec.rb index 1bfde065a..31dfff1b5 100644 --- a/spec/chewy/search/loader_spec.rb +++ b/spec/chewy/search/loader_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Search::Loader do - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) diff --git a/spec/chewy/search/pagination/kaminari_examples.rb b/spec/chewy/search/pagination/kaminari_examples.rb index 08a285716..aa86c9700 100644 --- a/spec/chewy/search/pagination/kaminari_examples.rb +++ b/spec/chewy/search/pagination/kaminari_examples.rb @@ -1,7 +1,7 @@ require 'spec_helper' shared_examples :kaminari do |request_base_class| - before { Chewy.massacre } + before { drop_indices } before do stub_index(:products) do diff --git a/spec/chewy/search/request_spec.rb b/spec/chewy/search/request_spec.rb index 414de49a7..b73b99161 100644 --- a/spec/chewy/search/request_spec.rb +++ b/spec/chewy/search/request_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Search::Request do - before { Chewy.massacre } + before { drop_indices } before do stub_index(:products) do diff --git a/spec/chewy/search/response_spec.rb b/spec/chewy/search/response_spec.rb index 7b291288d..3cf5830d2 100644 --- a/spec/chewy/search/response_spec.rb +++ b/spec/chewy/search/response_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Search::Response, :orm do - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) @@ -39,7 +39,7 @@ specify { expect(subject.hits).to all be_a(Hash) } specify do expect(subject.hits.flat_map(&:keys).uniq) - .to match_array(%w[_id _index _type _score _source sort]) + .to match_array(%w[_id _index _score _source sort]) end context do diff --git a/spec/chewy/search/scrolling_spec.rb b/spec/chewy/search/scrolling_spec.rb index 4dfe68941..003d899c7 100644 --- a/spec/chewy/search/scrolling_spec.rb +++ b/spec/chewy/search/scrolling_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Search::Scrolling, :orm do - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) diff --git a/spec/chewy/search_spec.rb b/spec/chewy/search_spec.rb index d7cafa40d..b8ead283e 100644 --- a/spec/chewy/search_spec.rb +++ b/spec/chewy/search_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Search do - before { Chewy.massacre } + before { drop_indices } before do stub_index(:products) diff --git a/spec/chewy/stash_spec.rb b/spec/chewy/stash_spec.rb index 2d2774ff5..fc0df7487 100644 --- a/spec/chewy/stash_spec.rb +++ b/spec/chewy/stash_spec.rb @@ -5,7 +5,7 @@ def fetch_deleted_number(response) response['deleted'] || response['_indices']['_all']['deleted'] end - before { Chewy.massacre } + before { drop_indices } before do stub_model(:city) diff --git a/spec/chewy/strategy/delayed_sidekiq_spec.rb b/spec/chewy/strategy/delayed_sidekiq_spec.rb index 78a1219b8..a4a8fdd09 100644 --- a/spec/chewy/strategy/delayed_sidekiq_spec.rb +++ b/spec/chewy/strategy/delayed_sidekiq_spec.rb @@ -134,7 +134,7 @@ def expected_at_time context 'both calls with different update fields' do it 'deos reindex with union of fields' do Timecop.freeze do - expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id]), update_fields: %w[name description]).once + expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id]), update_fields: match_array(%w[name description])).once scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name']) scheduler.postpone scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id], update_fields: ['description']) diff --git a/spec/chewy/strategy_spec.rb b/spec/chewy/strategy_spec.rb index 25a2344bd..817e2dfc3 100644 --- a/spec/chewy/strategy_spec.rb +++ b/spec/chewy/strategy_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Chewy::Strategy do - before { Chewy.massacre } + before { drop_indices } subject(:strategy) { Chewy::Strategy.new } describe '#current' do diff --git a/spec/chewy_spec.rb b/spec/chewy_spec.rb index 0c3e0d0ec..e946aecca 100644 --- a/spec/chewy_spec.rb +++ b/spec/chewy_spec.rb @@ -31,8 +31,8 @@ end end - describe '.massacre' do - before { Chewy.massacre } + xdescribe '.massacre' do + before { drop_indices } before do allow(Chewy).to receive_messages(configuration: Chewy.configuration.merge(prefix: 'prefix1')) @@ -40,7 +40,7 @@ allow(Chewy).to receive_messages(configuration: Chewy.configuration.merge(prefix: 'prefix2')) stub_index(:developers).create! - Chewy.massacre + drop_indices allow(Chewy).to receive_messages(configuration: Chewy.configuration.merge(prefix: 'prefix1')) end @@ -84,7 +84,8 @@ # To avoid flaky issues when previous specs were run allow(Chewy::Index).to receive(:descendants).and_return([CitiesIndex, PlacesIndex]) - Chewy.massacre + CitiesIndex.delete + PlacesIndex.delete end specify do @@ -111,7 +112,7 @@ expect(CitiesIndex.exists?).to eq true expect(PlacesIndex.exists?).to eq true - expect { Chewy.create_indices! }.to raise_error(Elasticsearch::Transport::Transport::Errors::BadRequest) + expect { Chewy.create_indices! }.to raise_error(Elastic::Transport::Transport::Errors::BadRequest) end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index be66332f8..c67ff0f88 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -28,6 +28,32 @@ } } +# To work with security enabled: +# +# user = ENV['ES_USER'] || 'elastic' +# password = ENV['ES_PASSWORD'] || '' +# ca_cert = ENV['ES_CA_CERT'] || './tmp/http_ca.crt' +# +# Chewy.settings.merge!( +# user: user, +# password: password, +# transport_options: { +# ssl: { +# ca_file: ca_cert +# } +# } +# ) + +# Low-level substitute for now-obsolete drop_indices +def drop_indices + response = Chewy.client.cat.indices + indices = response.body.lines.map { |line| line.split[2] } + return if indices.blank? + + Chewy.client.indices.delete(index: indices) + Chewy.wait_for_status +end + # Chewy.transport_logger = Logger.new(STDERR) RSpec.configure do |config| From d2c38f77e8b2840cd14182d9605acc2fc34fbd9f Mon Sep 17 00:00:00 2001 From: Danil Nurgaliev Date: Wed, 4 Sep 2024 11:03:23 +0300 Subject: [PATCH 55/59] Bump chewy version to 8.0.0-beta (#963) --- lib/chewy/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/chewy/version.rb b/lib/chewy/version.rb index a0c87a408..6ba954467 100644 --- a/lib/chewy/version.rb +++ b/lib/chewy/version.rb @@ -1,3 +1,3 @@ module Chewy - VERSION = '7.6.0'.freeze + VERSION = '8.0.0-beta'.freeze end From 5193d28bb9a1077abb6a55a14b10a3f63d21b530 Mon Sep 17 00:00:00 2001 From: Ebeagu Samuel Date: Fri, 13 Sep 2024 12:35:48 +0100 Subject: [PATCH 56/59] Assign dependabot PR's to sre (#966) --- .github/CODEOWNERS | 2 +- .github/dependabot.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2ef2a3547..ed69bcd4e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -.github/workflows @toptal/sre +* @toptal/sre diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a675b1060..eab5262c2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -20,7 +20,7 @@ updates: - "ruby" - "dependencies" reviewers: - - "toptal/devx" + - "toptal/sre" registries: - toptal-github insecure-external-code-execution: allow @@ -38,5 +38,5 @@ updates: - "dependencies" - "gha" reviewers: - - "toptal/devx" + - "toptal/sre" open-pull-requests-limit: 3 From 17597e622a37a3866f47ee4b96e37798244bb439 Mon Sep 17 00:00:00 2001 From: Jorge Tomas Date: Mon, 30 Sep 2024 08:45:56 +0200 Subject: [PATCH 57/59] Adding Ruby 3.3 and Rails 7.2 to CI matrix (#967) --- .github/workflows/ruby.yml | 7 +++++-- gemfiles/rails.7.2.activerecord.gemfile | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 gemfiles/rails.7.2.activerecord.gemfile diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index c9293a9aa..cd799c571 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -16,8 +16,11 @@ jobs: strategy: fail-fast: false matrix: - ruby: [ '3.0', '3.1', '3.2' ] - gemfile: [rails.6.1.activerecord, rails.7.0.activerecord, rails.7.1.activerecord] + ruby: [ '3.0', '3.1', '3.2', '3.3' ] + gemfile: [rails.6.1.activerecord, rails.7.0.activerecord, rails.7.1.activerecord, rails.7.2.activerecord] + exclude: + - ruby: '3.0' + gemfile: rails.7.2.activerecord name: ${{ matrix.ruby }}-${{ matrix.gemfile }} env: diff --git a/gemfiles/rails.7.2.activerecord.gemfile b/gemfiles/rails.7.2.activerecord.gemfile new file mode 100644 index 000000000..2cbbb94c0 --- /dev/null +++ b/gemfiles/rails.7.2.activerecord.gemfile @@ -0,0 +1,14 @@ +source 'https://rubygems.org' + +gem 'activejob', '~> 7.2.0' +gem 'activerecord', '~> 7.2.0' +gem 'activesupport', '~> 7.2.0' +gem 'kaminari-core', '~> 1.1.0', require: false +gem 'parallel', require: false +gem 'rspec_junit_formatter', '~> 0.4.1' +gem 'sidekiq', require: false + +gem 'rexml' + +gemspec path: '../' +eval_gemfile 'base.gemfile' From 21c540757c1bf361f8788e52dee6e035c03ad032 Mon Sep 17 00:00:00 2001 From: Jorge Tomas Date: Mon, 7 Oct 2024 10:11:41 +0200 Subject: [PATCH 58/59] updating ruby and active record versions based on test suite (#968) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index da00cef6c..096bdb578 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,9 @@ Or install it yourself as: ### Ruby -Chewy is compatible with MRI 3.0-3.2¹. +Chewy is compatible with MRI 3.0-3.3¹. -> ¹ Ruby 3 is only supported with Rails 6.1 +> ¹ Ruby 3 is supported with Rails 6.1, 7.0, 7.1 and 7.2 ### Elasticsearch compatibility matrix @@ -67,7 +67,7 @@ various Chewy versions. ### Active Record -5.2, 6.0, 6.1 Active Record versions are supported by all Chewy versions. +6.1, 7.0, 7.1, 7.2 Active Record versions are supported by Chewy. ## Getting Started From 6a2176f7d92c4818f1a1f572aed339b39aef087e Mon Sep 17 00:00:00 2001 From: Sundus Yousuf Date: Tue, 8 Oct 2024 13:54:39 -0500 Subject: [PATCH 59/59] Fix id conversion issue in delayed_sidekiq strategy (#964) * Fix id conversion issue in delayed_sidekiq strategy Ensure ids extracted from Redis remain strings, preventing UUID issues. Previously, ids were being converted to integers, causing problems with UUIDs in the `delayed_sidekiq` strategy. This update also enhances the test suite: - Existing tests are updated. - A new test ensures the issue is resolved. Due to SQLite's lack of UUID support, a `stub_uuid_model` method is added. This method stubs models with UUIDs, using `SecureRandom.uuid` for the primary key. Move table creations to individual methods. Having every table creation inside a single block casued Rubocop `Metrics/BlockLength` error. To fix it I moved each table creation to an individual method. * Add changelog notes --------- Co-authored-by: Sundus Yousuf --- CHANGELOG.md | 2 + lib/chewy/strategy/delayed_sidekiq/worker.rb | 2 +- spec/chewy/strategy/delayed_sidekiq_spec.rb | 37 ++++++++++++++----- spec/support/active_record.rb | 39 ++++++++++++++++++-- 4 files changed, 65 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c27c85fa..a7963c22f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### Bugs Fixed +* [#964](https://github.com/toptal/chewy/pull/964): Fix `delayed_sidekiq` worker to handle UUID primary keys correctly. + ## 8.0.0-beta (2024-08-27) ### New Features diff --git a/lib/chewy/strategy/delayed_sidekiq/worker.rb b/lib/chewy/strategy/delayed_sidekiq/worker.rb index af5fa793d..7b539c3ca 100644 --- a/lib/chewy/strategy/delayed_sidekiq/worker.rb +++ b/lib/chewy/strategy/delayed_sidekiq/worker.rb @@ -68,7 +68,7 @@ def extract_ids_and_fields(members) fields = nil if fields.include?(Scheduler::FALLBACK_FIELDS) - [ids.map(&:to_i), fields] + [ids, fields] end end end diff --git a/spec/chewy/strategy/delayed_sidekiq_spec.rb b/spec/chewy/strategy/delayed_sidekiq_spec.rb index a4a8fdd09..03ead7755 100644 --- a/spec/chewy/strategy/delayed_sidekiq_spec.rb +++ b/spec/chewy/strategy/delayed_sidekiq_spec.rb @@ -21,13 +21,23 @@ update_index('cities') { self } end + stub_uuid_model(:user) do + update_index('users') { self } + end + stub_index(:cities) do index_scope City end + + stub_index(:users) do + index_scope User + end end let(:city) { City.create!(name: 'hello') } let(:other_city) { City.create!(name: 'world') } + let(:user) { User.create!(name: 'John') } + let(:other_user) { User.create!(name: 'Jane') } it 'does not trigger immediate reindex due to it`s async nature' do expect { [city, other_city].map(&:save!) } @@ -36,12 +46,19 @@ it "respects 'refresh: false' options" do allow(Chewy).to receive(:disable_refresh_async).and_return(true) - expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id]), refresh: false) + expect(CitiesIndex).to receive(:import!).with(match_array([city.id.to_s, other_city.id.to_s]), refresh: false) scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id, other_city.id]) scheduler.postpone Chewy::Strategy::DelayedSidekiq::Worker.drain end + it 'works with models with string primary key' do + expect(UsersIndex).to receive(:import!).with(match_array([user.id, other_user.id])) + scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(UsersIndex, [user.id, other_user.id]) + scheduler.postpone + Chewy::Strategy::DelayedSidekiq::Worker.drain + end + context 'with default config' do it 'does schedule a job that triggers reindex with default options' do Timecop.freeze do @@ -109,7 +126,7 @@ def expected_at_time context 'two reindex call within the timewindow' do it 'accumulates all ids does the reindex one time' do Timecop.freeze do - expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id])).once + expect(CitiesIndex).to receive(:import!).with(match_array([city.id.to_s, other_city.id.to_s])).once scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id]) scheduler.postpone scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id]) @@ -121,7 +138,7 @@ def expected_at_time context 'one call with update_fields another one without update_fields' do it 'does reindex of all fields' do Timecop.freeze do - expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id])).once + expect(CitiesIndex).to receive(:import!).with(match_array([city.id.to_s, other_city.id.to_s])).once scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name']) scheduler.postpone scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id]) @@ -134,7 +151,7 @@ def expected_at_time context 'both calls with different update fields' do it 'deos reindex with union of fields' do Timecop.freeze do - expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id]), update_fields: match_array(%w[name description])).once + expect(CitiesIndex).to receive(:import!).with(match_array([city.id.to_s, other_city.id.to_s]), update_fields: match_array(%w[name description])).once scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name']) scheduler.postpone scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id], update_fields: ['description']) @@ -148,8 +165,8 @@ def expected_at_time context 'two calls within different timewindows' do it 'does two separate reindexes' do Timecop.freeze do - expect(CitiesIndex).to receive(:import!).with([city.id]).once - expect(CitiesIndex).to receive(:import!).with([other_city.id]).once + expect(CitiesIndex).to receive(:import!).with([city.id.to_s]).once + expect(CitiesIndex).to receive(:import!).with([other_city.id.to_s]).once Timecop.travel(20.seconds.ago) do scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id]) scheduler.postpone @@ -164,8 +181,8 @@ def expected_at_time context 'first call has update_fields' do it 'does first reindex with the expected update_fields and second without update_fields' do Timecop.freeze do - expect(CitiesIndex).to receive(:import!).with([city.id], update_fields: ['name']).once - expect(CitiesIndex).to receive(:import!).with([other_city.id]).once + expect(CitiesIndex).to receive(:import!).with([city.id.to_s], update_fields: ['name']).once + expect(CitiesIndex).to receive(:import!).with([other_city.id.to_s]).once Timecop.travel(20.seconds.ago) do scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name']) scheduler.postpone @@ -180,8 +197,8 @@ def expected_at_time context 'both calls have update_fields option' do it 'does both reindexes with their expected update_fields option' do Timecop.freeze do - expect(CitiesIndex).to receive(:import!).with([city.id], update_fields: ['name']).once - expect(CitiesIndex).to receive(:import!).with([other_city.id], update_fields: ['description']).once + expect(CitiesIndex).to receive(:import!).with([city.id.to_s], update_fields: ['name']).once + expect(CitiesIndex).to receive(:import!).with([other_city.id.to_s], update_fields: ['description']).once Timecop.travel(20.seconds.ago) do scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name']) scheduler.postpone diff --git a/spec/support/active_record.rb b/spec/support/active_record.rb index d081e5be4..e64660563 100644 --- a/spec/support/active_record.rb +++ b/spec/support/active_record.rb @@ -6,17 +6,16 @@ ActiveRecord::Base.raise_in_transactional_callbacks = true end -ActiveRecord::Base.connection.execute("DROP TABLE IF EXISTS 'countries'") -ActiveRecord::Base.connection.execute("DROP TABLE IF EXISTS 'cities'") -ActiveRecord::Base.connection.execute("DROP TABLE IF EXISTS 'locations'") -ActiveRecord::Schema.define do +def create_countries_table create_table :countries do |t| t.column :name, :string t.column :country_code, :string t.column :rating, :integer t.column :updated_at, :datetime end +end +def create_cities_table create_table :cities do |t| t.column :country_id, :integer t.column :name, :string @@ -25,13 +24,17 @@ t.column :rating, :integer t.column :updated_at, :datetime end +end +def create_locations_table create_table :locations do |t| t.column :city_id, :integer t.column :lat, :string t.column :lon, :string end +end +def create_comments_table create_table :comments do |t| t.column :content, :string t.column :comment_type, :string @@ -40,6 +43,26 @@ end end +def create_users_table + create_table :users, id: false do |t| + t.column :id, :string + t.column :name, :string + end +end + +ActiveRecord::Base.connection.execute("DROP TABLE IF EXISTS 'countries'") +ActiveRecord::Base.connection.execute("DROP TABLE IF EXISTS 'cities'") +ActiveRecord::Base.connection.execute("DROP TABLE IF EXISTS 'locations'") +ActiveRecord::Base.connection.execute("DROP TABLE IF EXISTS 'comments'") +ActiveRecord::Base.connection.execute("DROP TABLE IF EXISTS 'users'") +ActiveRecord::Schema.define do + create_countries_table + create_cities_table + create_locations_table + create_comments_table + create_users_table +end + module ActiveRecordClassHelpers extend ActiveSupport::Concern @@ -73,6 +96,14 @@ def expects_no_query(except: nil, &block) def stub_model(name, superclass = nil, &block) stub_class(name, superclass || ActiveRecord::Base, &block) end + + def stub_uuid_model(name, superclass = nil, &block) + stub_class(name, superclass || ActiveRecord::Base) do + before_create { self.id = SecureRandom.uuid } + define_singleton_method(:primary_key, -> { 'id' }) + class_eval(&block) + end + end end RSpec.configure do |config|