Added support for async reindexing for Mongoid and NoBrainer

Andrew Kane
1 parent c80447a2
Showing 2 changed files with 26 additions and 14 deletions Show diff stats
lib/searchkick/index.rb
test/reindex_test.rb
@@ -267,17 +267,21 @@ module Searchkick
           batches_count.times do |i|
             batch_id = i + 1
             min_id = starting_id + (i * batch_size)
-            Searchkick::BulkReindexJob.perform_later(
-              class_name: scope.model_name.name,
-              min_id: min_id,
-              max_id: min_id + batch_size - 1,
-              index_name: name,
-              batch_id: batch_id
-            )
-            redis.sadd(batches_key, batch_id) if redis
+            bulk_reindex_job scope, batch_id, min_id: min_id, max_id: min_id + batch_size - 1
           end
         else
-          raise Searchkick::Error, "async option only supported for ActiveRecord"
+          batch_id = 1
+          items = []
+          scope = scope.only(:_id) if scope.respond_to?(:only)
+          scope.each do |item|
+            items << item
+            if items.length == batch_size
+              bulk_reindex_job scope, batch_id, record_ids: items.map { |i| i.id.to_s }
+              items = []
+              batch_id += 1
+            end
+          end
+          bulk_reindex_job scope, batch_id, record_ids: items.map { |i| i.id.to_s }
         end
       elsif scope.respond_to?(:find_in_batches)
         if resume
@@ -442,6 +446,15 @@ module Searchkick
       end
     end
  
+    def bulk_reindex_job(scope, batch_id, options)
+      Searchkick::BulkReindexJob.perform_later({
+        class_name: scope.model_name.name,
+        index_name: name,
+        batch_id: batch_id
+      }.merge(options))
+      redis.sadd(batches_key, batch_id) if redis
+    end
+
     def with_retries
       retries = 0
  
 require_relative "test_helper"
  
 class ReindexTest < Minitest::Test
-  def setup
+  def test_scoped
     skip if nobrainer?
-    super
-  end
  
-  def test_scoped
     store_names ["Product A"]
     Searchkick.callbacks(false) do
       store_names ["Product B", "Product C"]
@@ -16,6 +13,8 @@ class ReindexTest &lt; Minitest::Test
   end
  
   def test_associations
+    skip if nobrainer?
+
     store_names ["Product A"]
     store = Store.create!(name: "Test")
     Product.create!(name: "Product B", store_id: store.id)
@@ -24,7 +23,7 @@ class ReindexTest &lt; Minitest::Test
   end
  
   def test_async
-    skip unless defined?(ActiveJob) && defined?(ActiveRecord)
+    skip if !defined?(ActiveJob)
  
     Searchkick.callbacks(false) do
       store_names ["Product A"]
...	...	@@ -267,17 +267,21 @@ module Searchkick
267	267	batches_count.times do \|i\|
268	268	batch_id = i + 1
269	269	min_id = starting_id + (i * batch_size)
270		- Searchkick::BulkReindexJob.perform_later(
271		- class_name: scope.model_name.name,
272		- min_id: min_id,
273		- max_id: min_id + batch_size - 1,
274		- index_name: name,
275		- batch_id: batch_id
276		- )
277		- redis.sadd(batches_key, batch_id) if redis
	270	+ bulk_reindex_job scope, batch_id, min_id: min_id, max_id: min_id + batch_size - 1
278	271	end
279	272	else
280		- raise Searchkick::Error, "async option only supported for ActiveRecord"
	273	+ batch_id = 1
	274	+ items = []
	275	+ scope = scope.only(:_id) if scope.respond_to?(:only)
	276	+ scope.each do \|item\|
	277	+ items << item
	278	+ if items.length == batch_size
	279	+ bulk_reindex_job scope, batch_id, record_ids: items.map { \|i\| i.id.to_s }
	280	+ items = []
	281	+ batch_id += 1
	282	+ end
	283	+ end
	284	+ bulk_reindex_job scope, batch_id, record_ids: items.map { \|i\| i.id.to_s }
281	285	end
282	286	elsif scope.respond_to?(:find_in_batches)
283	287	if resume
...	...	@@ -442,6 +446,15 @@ module Searchkick
442	446	end
443	447	end
444	448
	449	+ def bulk_reindex_job(scope, batch_id, options)
	450	+ Searchkick::BulkReindexJob.perform_later({
	451	+ class_name: scope.model_name.name,
	452	+ index_name: name,
	453	+ batch_id: batch_id
	454	+ }.merge(options))
	455	+ redis.sadd(batches_key, batch_id) if redis
	456	+ end
	457	+
445	458	def with_retries
446	459	retries = 0
447	460
...	...
1	1	require_relative "test_helper"
2	2
3	3	class ReindexTest < Minitest::Test
4		- def setup
	4	+ def test_scoped
5	5	skip if nobrainer?
6		- super
7		- end
8	6
9		- def test_scoped
10	7	store_names ["Product A"]
11	8	Searchkick.callbacks(false) do
12	9	store_names ["Product B", "Product C"]
...	...	@@ -16,6 +13,8 @@ class ReindexTest < Minitest::Test
16	13	end
17	14
18	15	def test_associations
	16	+ skip if nobrainer?
	17	+
19	18	store_names ["Product A"]
20	19	store = Store.create!(name: "Test")
21	20	Product.create!(name: "Product B", store_id: store.id)
...	...	@@ -24,7 +23,7 @@ class ReindexTest < Minitest::Test
24	23	end
25	24
26	25	def test_async
27		- skip unless defined?(ActiveJob) && defined?(ActiveRecord)
	26	+ skip if !defined?(ActiveJob)
28	27
29	28	Searchkick.callbacks(false) do
30	29	store_names ["Product A"]
...	...