Added partial_reindex method - closes #766

Andrew Kane
1 parent 84c52170
Showing 6 changed files with 69 additions and 6 deletions Show diff stats
CHANGELOG.md
README.md
lib/searchkick/index.rb
lib/searchkick/model.rb
test/partial_reindex_test.rb
test/test_helper.rb
 ## 1.4.1 [unreleased]
  
+- Added `partial_reindex` method
 - Added `debug` option to `search` method
  
 ## 1.4.0
@@ -1213,6 +1213,18 @@ Product.where(&quot;id &gt; 100000&quot;).find_in_batches do |batch|
 end
 ```
  
+Reindex a subset of attributes [master]
+
+```ruby
+class Product < ActiveRecord::Base
+  def search_prices
+    {price: price}
+  end
+end
+
+Product.partial_reindex(:search_prices)
+```
+
 Remove old indices
  
 ```ruby
@@ -67,6 +67,10 @@ module Searchkick
     end
     alias_method :import, :bulk_index
  
+    def bulk_update(records, method_name)
+      Searchkick.queue_items(records.map { |r| {update: record_data(r).merge(data: {doc: search_data(r, method_name)})} })
+    end
+
     def record_data(r)
       data = {
         _index: name,
@@ -221,6 +225,7 @@ module Searchkick
  
     def import_scope(scope, options = {})
       batch_size = @options[:batch_size] || 1000
+      method_name = options[:method_name]
  
       # use scope for import
       scope = scope.search_import if scope.respond_to?(:search_import)
@@ -234,7 +239,7 @@ module Searchkick
         end
  
         scope.find_in_batches batch_size: batch_size do |batch|
-          import batch.select(&:should_index?)
+          import_or_update batch.select(&:should_index?), method_name
         end
       else
         # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
@@ -244,14 +249,18 @@ module Searchkick
         scope.all.each do |item|
           items << item if item.should_index?
           if items.length == batch_size
-            import items
+            import_or_update items, method_name
             items = []
           end
         end
-        import items
+        import_or_update items, method_name
       end
     end
  
+    def import_or_update(records, method_name)
+      method_name ? bulk_update(records, method_name) : import(records)
+    end
+
     # other
  
     def tokens(text, options = {})
@@ -285,8 +294,9 @@ module Searchkick
       id.is_a?(Numeric) ? id : id.to_s
     end
  
-    def search_data(record)
-      source = record.search_data
+    def search_data(record, method_name = nil)
+      partial_reindex = !method_name.nil?
+      source = record.send(method_name || :search_data)
       options = record.class.searchkick_options
  
       # stringify fields
@@ -302,7 +312,7 @@ module Searchkick
  
       # hack to prevent generator field doesn't exist error
       (options[:suggest] || []).map(&:to_s).each do |field|
-        source[field] = nil unless source[field]
+        source[field] = nil if !source[field] && !partial_reindex
       end
  
       # locations
@@ -54,6 +54,13 @@ module Searchkick
           end
           alias_method :reindex, :searchkick_reindex unless method_defined?(:reindex)
  
+          def searchkick_partial_reindex(method_name)
+            searchkick_index.import_scope(searchkick_klass, method_name: method_name)
+            searchkick_index.refresh
+            true
+          end
+          alias_method :partial_reindex, :searchkick_partial_reindex unless method_defined?(:partial_reindex)
+
           def clean_indices
             searchkick_index.clean_indices
           end
@@ -0,0 +1,27 @@
+require_relative "test_helper"
+
+class PartialReindexTest < Minitest::Test
+  def test_basic
+    store [{name: "Hi", color: "Blue"}]
+
+    # normal search
+    assert_search "hi", ["Hi"], fields: [:name], load: false
+    assert_search "blue", ["Hi"], fields: [:color], load: false
+
+    # update
+    Product.first.update_column :name, "Bye"
+    Product.first.update_column :color, "Red"
+    Product.searchkick_index.refresh
+
+    # index not updated
+    assert_search "hi", ["Hi"], fields: [:name], load: false
+    assert_search "blue", ["Hi"], fields: [:color], load: false
+
+    # partial reindex
+    Product.partial_reindex(:search_name)
+
+    # name updated, but not color
+    assert_search "bye", ["Bye"], fields: [:name], load: false
+    assert_search "blue", ["Bye"], fields: [:color], load: false
+  end
+end
@@ -309,6 +309,12 @@ class Product
   def should_index?
     name != "DO NOT INDEX"
   end
+
+  def search_name
+    {
+      name: name
+    }
+  end
 end
  
 class Store
1	1	## 1.4.1 [unreleased]
2	2
	3	+- Added `partial_reindex` method
3	4	- Added `debug` option to `search` method
4	5
5	6	## 1.4.0
...	...
...	...	@@ -1213,6 +1213,18 @@ Product.where("id > 100000").find_in_batches do \|batch\|
1213	1213	end
1214	1214	```
1215	1215
	1216	+Reindex a subset of attributes [master]
	1217	+
	1218	+```ruby
	1219	+class Product < ActiveRecord::Base
	1220	+ def search_prices
	1221	+ {price: price}
	1222	+ end
	1223	+end
	1224	+
	1225	+Product.partial_reindex(:search_prices)
	1226	+```
	1227	+
1216	1228	Remove old indices
1217	1229
1218	1230	```ruby
...	...
...	...	@@ -67,6 +67,10 @@ module Searchkick
67	67	end
68	68	alias_method :import, :bulk_index
69	69
	70	+ def bulk_update(records, method_name)
	71	+ Searchkick.queue_items(records.map { \|r\| {update: record_data(r).merge(data: {doc: search_data(r, method_name)})} })
	72	+ end
	73	+
70	74	def record_data(r)
71	75	data = {
72	76	_index: name,
...	...	@@ -221,6 +225,7 @@ module Searchkick
221	225
222	226	def import_scope(scope, options = {})
223	227	batch_size = @options[:batch_size] \|\| 1000
	228	+ method_name = options[:method_name]
224	229
225	230	# use scope for import
226	231	scope = scope.search_import if scope.respond_to?(:search_import)
...	...	@@ -234,7 +239,7 @@ module Searchkick
234	239	end
235	240
236	241	scope.find_in_batches batch_size: batch_size do \|batch\|
237		- import batch.select(&:should_index?)
	242	+ import_or_update batch.select(&:should_index?), method_name
238	243	end
239	244	else
240	245	# https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
...	...	@@ -244,14 +249,18 @@ module Searchkick
244	249	scope.all.each do \|item\|
245	250	items << item if item.should_index?
246	251	if items.length == batch_size
247		- import items
	252	+ import_or_update items, method_name
248	253	items = []
249	254	end
250	255	end
251		- import items
	256	+ import_or_update items, method_name
252	257	end
253	258	end
254	259
	260	+ def import_or_update(records, method_name)
	261	+ method_name ? bulk_update(records, method_name) : import(records)
	262	+ end
	263	+
255	264	# other
256	265
257	266	def tokens(text, options = {})
...	...	@@ -285,8 +294,9 @@ module Searchkick
285	294	id.is_a?(Numeric) ? id : id.to_s
286	295	end
287	296
288		- def search_data(record)
289		- source = record.search_data
	297	+ def search_data(record, method_name = nil)
	298	+ partial_reindex = !method_name.nil?
	299	+ source = record.send(method_name \|\| :search_data)
290	300	options = record.class.searchkick_options
291	301
292	302	# stringify fields
...	...	@@ -302,7 +312,7 @@ module Searchkick
302	312
303	313	# hack to prevent generator field doesn't exist error
304	314	(options[:suggest] \|\| []).map(&:to_s).each do \|field\|
305		- source[field] = nil unless source[field]
	315	+ source[field] = nil if !source[field] && !partial_reindex
306	316	end
307	317
308	318	# locations
...	...
...	...	@@ -54,6 +54,13 @@ module Searchkick
54	54	end
55	55	alias_method :reindex, :searchkick_reindex unless method_defined?(:reindex)
56	56
	57	+ def searchkick_partial_reindex(method_name)
	58	+ searchkick_index.import_scope(searchkick_klass, method_name: method_name)
	59	+ searchkick_index.refresh
	60	+ true
	61	+ end
	62	+ alias_method :partial_reindex, :searchkick_partial_reindex unless method_defined?(:partial_reindex)
	63	+
57	64	def clean_indices
58	65	searchkick_index.clean_indices
59	66	end
...	...
...	...	@@ -0,0 +1,27 @@
	1	+require_relative "test_helper"
	2	+
	3	+class PartialReindexTest < Minitest::Test
	4	+ def test_basic
	5	+ store [{name: "Hi", color: "Blue"}]
	6	+
	7	+ # normal search
	8	+ assert_search "hi", ["Hi"], fields: [:name], load: false
	9	+ assert_search "blue", ["Hi"], fields: [:color], load: false
	10	+
	11	+ # update
	12	+ Product.first.update_column :name, "Bye"
	13	+ Product.first.update_column :color, "Red"
	14	+ Product.searchkick_index.refresh
	15	+
	16	+ # index not updated
	17	+ assert_search "hi", ["Hi"], fields: [:name], load: false
	18	+ assert_search "blue", ["Hi"], fields: [:color], load: false
	19	+
	20	+ # partial reindex
	21	+ Product.partial_reindex(:search_name)
	22	+
	23	+ # name updated, but not color
	24	+ assert_search "bye", ["Bye"], fields: [:name], load: false
	25	+ assert_search "blue", ["Bye"], fields: [:color], load: false
	26	+ end
	27	+end
...	...
...	...	@@ -309,6 +309,12 @@ class Product
309	309	def should_index?
310	310	name != "DO NOT INDEX"
311	311	end
	312	+
	313	+ def search_name
	314	+ {
	315	+ name: name
	316	+ }
	317	+ end
312	318	end
313	319
314	320	class Store
...	...