Added allow_missing option to reindex

Andrew Kane
1 parent 13fd0f4e
Showing 9 changed files with 51 additions and 21 deletions Show diff stats
lib/searchkick/bulk_reindex_job.rb
lib/searchkick/index.rb
lib/searchkick/indexer.rb
lib/searchkick/model.rb
lib/searchkick/process_batch_job.rb
lib/searchkick/record_indexer.rb
lib/searchkick/reindex_v2_job.rb
lib/searchkick/relation_indexer.rb
test/partial_reindex_test.rb
@@ -14,7 +14,8 @@ module Searchkick
       relation = Searchkick.load_records(relation, record_ids)
       relation = relation.search_import if relation.respond_to?(:search_import)
-      RecordIndexer.new(index).reindex(relation, mode: :inline, method_name: method_name, full: false)
+      # TODO support allow_missing
+      RecordIndexer.new(index).reindex(relation, mode: :inline, method_name: method_name, allow_missing: false, full: false)
       RelationIndexer.new(index).batch_completed(batch_id) if batch_id
     end
   end
@@ -163,11 +163,11 @@ module Searchkick
     end
     alias_method :import, :bulk_index
-    def bulk_update(records, method_name)
+    def bulk_update(records, method_name, allow_missing: false)
       return if records.empty?
       notify_bulk(records, "Update") do
-        queue_update(records, method_name)
+        queue_update(records, method_name, allow_missing: allow_missing)
       end
     end
@@ -211,10 +211,10 @@ module Searchkick
     # note: this is designed to be used internally
     # so it does not check object matches index class
-    def reindex(object, method_name: nil, full: false, **options)
+    def reindex(object, method_name: nil, allow_missing: false, full: false, **options)
       if object.is_a?(Array)
         # note: purposefully skip full
-        return reindex_records(object, method_name: method_name, **options)
+        return reindex_records(object, method_name: method_name, allow_missing: allow_missing, **options)
       end
       if !object.respond_to?(:searchkick_klass)
@@ -233,7 +233,7 @@ module Searchkick
         raise ArgumentError, "unsupported keywords: #{options.keys.map(&:inspect).join(", ")}" if options.any?
         # import only
-        import_scope(relation, method_name: method_name, mode: mode)
+        import_scope(relation, method_name: method_name, mode: mode, allow_missing: allow_missing)
         self.refresh if refresh
         true
       else
@@ -322,8 +322,10 @@ module Searchkick
       Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| RecordData.new(self, r).delete_data })
     end
-    def queue_update(records, method_name)
-      Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).update_data(method_name) })
+    def queue_update(records, method_name, allow_missing:)
+      items = records.map { |r| RecordData.new(self, r).update_data(method_name) }
+      items.each { |i| i.instance_variable_set(:@allow_missing, true) } if allow_missing
+      Searchkick.indexer.queue(items)
     end
     def relation_indexer
@@ -24,12 +24,20 @@ module Searchkick
         # note: delete does not set error when item not found
         first_with_error = response["items"].map do |item|
           (item["index"] || item["delete"] || item["update"])
-        end.find { |item| item["error"] }
-        raise ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'"
+        end.find.with_index { |item, i| item["error"] && !allow_missing?(items[i], item["error"]) }
+        if first_with_error
+          raise ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'"
+        end
       end
       # maybe return response in future
       nil
     end
+
+    private
+
+    def allow_missing?(item, error)
+      error["type"] == "document_missing_exception" && item.instance_variable_defined?(:@allow_missing)
+    end
   end
 end
@@ -27,8 +27,8 @@ module Searchkick
       mod = Module.new
       include(mod)
       mod.module_eval do
-        def reindex(method_name = nil, mode: nil, refresh: false)
-          self.class.searchkick_index.reindex([self], method_name: method_name, mode: mode, refresh: refresh, single: true)
+        def reindex(method_name = nil, mode: nil, refresh: false, allow_missing: false)
+          self.class.searchkick_index.reindex([self], method_name: method_name, mode: mode, refresh: refresh, allow_missing: allow_missing, single: true)
         end unless base.method_defined?(:reindex)
         def similar(**options)
@@ -14,7 +14,7 @@ module Searchkick
         end
       relation = Searchkick.scope(model)
-      RecordIndexer.new(index).reindex_items(relation, items, method_name: nil)
+      RecordIndexer.new(index).reindex_items(relation, items, method_name: nil, allow_missing: false)
     end
   end
 end
@@ -6,7 +6,7 @@ module Searchkick
       @index = index
     end
-    def reindex(records, mode:, method_name:, full: false, single: false)
+    def reindex(records, mode:, method_name:, allow_missing:, full: false, single: false)
       # prevents exists? check if records is a relation
       records = records.to_a
       return if records.empty?
@@ -17,6 +17,10 @@ module Searchkick
           raise Error, "Active Job not found"
         end
+        if allow_missing
+          raise Error, "Allow missing not supported with async option yet"
+        end
+
         # we could likely combine ReindexV2Job, BulkReindexJob, and ProcessBatchJob
         # but keep them separate for now
         if single
@@ -51,7 +55,7 @@ module Searchkick
         index.reindex_queue.push_records(records)
       when true, :inline
         index_records, other_records = records.partition { |r| index_record?(r) }
-        import_inline(index_records, !full ? other_records : [], method_name: method_name, single: single)
+        import_inline(index_records, !full ? other_records : [], method_name: method_name, allow_missing: allow_missing, single: single)
       else
         raise ArgumentError, "Invalid value for mode"
       end
@@ -60,7 +64,7 @@ module Searchkick
       true
     end
-    def reindex_items(klass, items, method_name:, single: false)
+    def reindex_items(klass, items, method_name:, allow_missing:, single: false)
       routing = items.to_h { |r| [r[:id], r[:routing]] }
       record_ids = routing.keys
@@ -76,7 +80,7 @@ module Searchkick
           construct_record(klass, id, routing[id])
         end
-      import_inline(records, delete_records, method_name: method_name, single: single)
+      import_inline(records, delete_records, method_name: method_name, allow_missing: allow_missing, single: single)
     end
     private
@@ -86,13 +90,13 @@ module Searchkick
     end
     # import in single request with retries
-    def import_inline(index_records, delete_records, method_name:, single:)
+    def import_inline(index_records, delete_records, method_name:, allow_missing:, single:)
       return if index_records.empty? && delete_records.empty?
       maybe_bulk(index_records, delete_records, method_name, single) do
         if index_records.any?
           if method_name
-            index.bulk_update(index_records, method_name)
+            index.bulk_update(index_records, method_name, allow_missing: allow_missing)
           else
             index.bulk_index(index_records)
           end
@@ -11,7 +11,8 @@ module Searchkick
       # but keep for now for backwards compatibility
       model = model.unscoped if model.respond_to?(:unscoped)
       items = [{id: id, routing: routing}]
-      RecordIndexer.new(index).reindex_items(model, items, method_name: method_name, single: true)
+      # TODO support allow_missing
+      RecordIndexer.new(index).reindex_items(model, items, method_name: method_name, allow_missing: false, single: true)
     end
   end
 end
@@ -6,7 +6,7 @@ module Searchkick
       @index = index
     end
-    def reindex(relation, mode:, method_name: nil, full: false, resume: false, scope: nil)
+    def reindex(relation, mode:, method_name: nil, allow_missing: false, full: false, resume: false, scope: nil)
       # apply scopes
       if scope
         relation = relation.send(scope)
@@ -32,6 +32,7 @@ module Searchkick
       reindex_options = {
         mode: mode,
         method_name: method_name,
+        allow_missing: allow_missing,
         full: full
       }
       record_indexer = RecordIndexer.new(index)
@@ -71,4 +71,17 @@ class PartialReindexTest &lt; Minitest::Test
     end
     assert_match "document missing", error.message
   end
+
+  def test_allow_missing
+    store [{name: "Hi", color: "Blue"}]
+
+    product = Product.first
+    Product.search_index.remove(product)
+
+    product.reindex(:search_name, allow_missing: true)
+    Product.where(id: product.id).reindex(:search_name, allow_missing: true)
+    Searchkick.callbacks(:bulk) do
+      product.reindex(:search_name, allow_missing: true)
+    end
+  end
 end
	@@ -14,7 +14,7 @@ module Searchkick		@@ -14,7 +14,7 @@ module Searchkick
14	end	14	end
15		15
16	relation = Searchkick.scope(model)	16	relation = Searchkick.scope(model)
17	- RecordIndexer.new(index).reindex_items(relation, items, method_name: nil)	17	+ RecordIndexer.new(index).reindex_items(relation, items, method_name: nil, allow_missing: false)
18	end	18	end
19	end	19	end
20	end	20	end
	@@ -71,4 +71,17 @@ class PartialReindexTest < Minitest::Test		@@ -71,4 +71,17 @@ class PartialReindexTest < Minitest::Test
71	end	71	end
72	assert_match "document missing", error.message	72	assert_match "document missing", error.message
73	end	73	end
		74	+
		75	+ def test_allow_missing
		76	+ store [{name: "Hi", color: "Blue"}]
		77	+
		78	+ product = Product.first
		79	+ Product.search_index.remove(product)
		80	+
		81	+ product.reindex(:search_name, allow_missing: true)
		82	+ Product.where(id: product.id).reindex(:search_name, allow_missing: true)
		83	+ Searchkick.callbacks(:bulk) do
		84	+ product.reindex(:search_name, allow_missing: true)
		85	+ end
		86	+ end
74	end	87	end