From d5312f12f451c27fcf9570c6627f4823ae25452b Mon Sep 17 00:00:00 2001 From: Andrew Kane Date: Wed, 15 Jun 2022 21:29:54 -0700 Subject: [PATCH] Added allow_missing option to reindex --- lib/searchkick/bulk_reindex_job.rb | 3 ++- lib/searchkick/index.rb | 16 +++++++++------- lib/searchkick/indexer.rb | 12 ++++++++++-- lib/searchkick/model.rb | 4 ++-- lib/searchkick/process_batch_job.rb | 2 +- lib/searchkick/record_indexer.rb | 16 ++++++++++------ lib/searchkick/reindex_v2_job.rb | 3 ++- lib/searchkick/relation_indexer.rb | 3 ++- test/partial_reindex_test.rb | 13 +++++++++++++ 9 files changed, 51 insertions(+), 21 deletions(-) diff --git a/lib/searchkick/bulk_reindex_job.rb b/lib/searchkick/bulk_reindex_job.rb index 7d3cb46..f80dc5a 100644 --- a/lib/searchkick/bulk_reindex_job.rb +++ b/lib/searchkick/bulk_reindex_job.rb @@ -14,7 +14,8 @@ module Searchkick relation = Searchkick.load_records(relation, record_ids) relation = relation.search_import if relation.respond_to?(:search_import) - RecordIndexer.new(index).reindex(relation, mode: :inline, method_name: method_name, full: false) + # TODO support allow_missing + RecordIndexer.new(index).reindex(relation, mode: :inline, method_name: method_name, allow_missing: false, full: false) RelationIndexer.new(index).batch_completed(batch_id) if batch_id end end diff --git a/lib/searchkick/index.rb b/lib/searchkick/index.rb index 1bc3744..c9706db 100644 --- a/lib/searchkick/index.rb +++ b/lib/searchkick/index.rb @@ -163,11 +163,11 @@ module Searchkick end alias_method :import, :bulk_index - def bulk_update(records, method_name) + def bulk_update(records, method_name, allow_missing: false) return if records.empty? notify_bulk(records, "Update") do - queue_update(records, method_name) + queue_update(records, method_name, allow_missing: allow_missing) end end @@ -211,10 +211,10 @@ module Searchkick # note: this is designed to be used internally # so it does not check object matches index class - def reindex(object, method_name: nil, full: false, **options) + def reindex(object, method_name: nil, allow_missing: false, full: false, **options) if object.is_a?(Array) # note: purposefully skip full - return reindex_records(object, method_name: method_name, **options) + return reindex_records(object, method_name: method_name, allow_missing: allow_missing, **options) end if !object.respond_to?(:searchkick_klass) @@ -233,7 +233,7 @@ module Searchkick raise ArgumentError, "unsupported keywords: #{options.keys.map(&:inspect).join(", ")}" if options.any? # import only - import_scope(relation, method_name: method_name, mode: mode) + import_scope(relation, method_name: method_name, mode: mode, allow_missing: allow_missing) self.refresh if refresh true else @@ -322,8 +322,10 @@ module Searchkick Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| RecordData.new(self, r).delete_data }) end - def queue_update(records, method_name) - Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).update_data(method_name) }) + def queue_update(records, method_name, allow_missing:) + items = records.map { |r| RecordData.new(self, r).update_data(method_name) } + items.each { |i| i.instance_variable_set(:@allow_missing, true) } if allow_missing + Searchkick.indexer.queue(items) end def relation_indexer diff --git a/lib/searchkick/indexer.rb b/lib/searchkick/indexer.rb index dc2e816..3c8954d 100644 --- a/lib/searchkick/indexer.rb +++ b/lib/searchkick/indexer.rb @@ -24,12 +24,20 @@ module Searchkick # note: delete does not set error when item not found first_with_error = response["items"].map do |item| (item["index"] || item["delete"] || item["update"]) - end.find { |item| item["error"] } - raise ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'" + end.find.with_index { |item, i| item["error"] && !allow_missing?(items[i], item["error"]) } + if first_with_error + raise ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'" + end end # maybe return response in future nil end + + private + + def allow_missing?(item, error) + error["type"] == "document_missing_exception" && item.instance_variable_defined?(:@allow_missing) + end end end diff --git a/lib/searchkick/model.rb b/lib/searchkick/model.rb index 41aa487..3bf1508 100644 --- a/lib/searchkick/model.rb +++ b/lib/searchkick/model.rb @@ -27,8 +27,8 @@ module Searchkick mod = Module.new include(mod) mod.module_eval do - def reindex(method_name = nil, mode: nil, refresh: false) - self.class.searchkick_index.reindex([self], method_name: method_name, mode: mode, refresh: refresh, single: true) + def reindex(method_name = nil, mode: nil, refresh: false, allow_missing: false) + self.class.searchkick_index.reindex([self], method_name: method_name, mode: mode, refresh: refresh, allow_missing: allow_missing, single: true) end unless base.method_defined?(:reindex) def similar(**options) diff --git a/lib/searchkick/process_batch_job.rb b/lib/searchkick/process_batch_job.rb index 22d96b4..ae9321a 100644 --- a/lib/searchkick/process_batch_job.rb +++ b/lib/searchkick/process_batch_job.rb @@ -14,7 +14,7 @@ module Searchkick end relation = Searchkick.scope(model) - RecordIndexer.new(index).reindex_items(relation, items, method_name: nil) + RecordIndexer.new(index).reindex_items(relation, items, method_name: nil, allow_missing: false) end end end diff --git a/lib/searchkick/record_indexer.rb b/lib/searchkick/record_indexer.rb index 14f2d7b..7c24b90 100644 --- a/lib/searchkick/record_indexer.rb +++ b/lib/searchkick/record_indexer.rb @@ -6,7 +6,7 @@ module Searchkick @index = index end - def reindex(records, mode:, method_name:, full: false, single: false) + def reindex(records, mode:, method_name:, allow_missing:, full: false, single: false) # prevents exists? check if records is a relation records = records.to_a return if records.empty? @@ -17,6 +17,10 @@ module Searchkick raise Error, "Active Job not found" end + if allow_missing + raise Error, "Allow missing not supported with async option yet" + end + # we could likely combine ReindexV2Job, BulkReindexJob, and ProcessBatchJob # but keep them separate for now if single @@ -51,7 +55,7 @@ module Searchkick index.reindex_queue.push_records(records) when true, :inline index_records, other_records = records.partition { |r| index_record?(r) } - import_inline(index_records, !full ? other_records : [], method_name: method_name, single: single) + import_inline(index_records, !full ? other_records : [], method_name: method_name, allow_missing: allow_missing, single: single) else raise ArgumentError, "Invalid value for mode" end @@ -60,7 +64,7 @@ module Searchkick true end - def reindex_items(klass, items, method_name:, single: false) + def reindex_items(klass, items, method_name:, allow_missing:, single: false) routing = items.to_h { |r| [r[:id], r[:routing]] } record_ids = routing.keys @@ -76,7 +80,7 @@ module Searchkick construct_record(klass, id, routing[id]) end - import_inline(records, delete_records, method_name: method_name, single: single) + import_inline(records, delete_records, method_name: method_name, allow_missing: allow_missing, single: single) end private @@ -86,13 +90,13 @@ module Searchkick end # import in single request with retries - def import_inline(index_records, delete_records, method_name:, single:) + def import_inline(index_records, delete_records, method_name:, allow_missing:, single:) return if index_records.empty? && delete_records.empty? maybe_bulk(index_records, delete_records, method_name, single) do if index_records.any? if method_name - index.bulk_update(index_records, method_name) + index.bulk_update(index_records, method_name, allow_missing: allow_missing) else index.bulk_index(index_records) end diff --git a/lib/searchkick/reindex_v2_job.rb b/lib/searchkick/reindex_v2_job.rb index 0946f87..2dd463b 100644 --- a/lib/searchkick/reindex_v2_job.rb +++ b/lib/searchkick/reindex_v2_job.rb @@ -11,7 +11,8 @@ module Searchkick # but keep for now for backwards compatibility model = model.unscoped if model.respond_to?(:unscoped) items = [{id: id, routing: routing}] - RecordIndexer.new(index).reindex_items(model, items, method_name: method_name, single: true) + # TODO support allow_missing + RecordIndexer.new(index).reindex_items(model, items, method_name: method_name, allow_missing: false, single: true) end end end diff --git a/lib/searchkick/relation_indexer.rb b/lib/searchkick/relation_indexer.rb index 7ab58be..f9748e2 100644 --- a/lib/searchkick/relation_indexer.rb +++ b/lib/searchkick/relation_indexer.rb @@ -6,7 +6,7 @@ module Searchkick @index = index end - def reindex(relation, mode:, method_name: nil, full: false, resume: false, scope: nil) + def reindex(relation, mode:, method_name: nil, allow_missing: false, full: false, resume: false, scope: nil) # apply scopes if scope relation = relation.send(scope) @@ -32,6 +32,7 @@ module Searchkick reindex_options = { mode: mode, method_name: method_name, + allow_missing: allow_missing, full: full } record_indexer = RecordIndexer.new(index) diff --git a/test/partial_reindex_test.rb b/test/partial_reindex_test.rb index c02f058..efbe2d8 100644 --- a/test/partial_reindex_test.rb +++ b/test/partial_reindex_test.rb @@ -71,4 +71,17 @@ class PartialReindexTest < Minitest::Test end assert_match "document missing", error.message end + + def test_allow_missing + store [{name: "Hi", color: "Blue"}] + + product = Product.first + Product.search_index.remove(product) + + product.reindex(:search_name, allow_missing: true) + Product.where(id: product.id).reindex(:search_name, allow_missing: true) + Searchkick.callbacks(:bulk) do + product.reindex(:search_name, allow_missing: true) + end + end end -- libgit2 0.21.0