Merge upstream

rainsense
2 parents 71626568 3dc4776b
Showing 11 changed files with 106 additions and 20 deletions Show diff stats
.travis.yml
CHANGELOG.md
README.md
lib/searchkick.rb
lib/searchkick/index.rb
lib/searchkick/index_options.rb
lib/searchkick/logging.rb
lib/searchkick/multi_search.rb
lib/searchkick/query.rb
test/highlight_test.rb
test/multi_search_test.rb
@@ -33,6 +33,7 @@ matrix:
     - gemfile: Gemfile
       env: ELASTICSEARCH_VERSION=5.0.1
       jdk: oraclejdk8
+  allow_failures:
     - gemfile: Gemfile
-      env: ELASTICSEARCH_VERSION=6.0.0-alpha2
+      env: ELASTICSEARCH_VERSION=6.0.0-beta1
       jdk: oraclejdk8
@@ -2,6 +2,8 @@
  
 - Added `_all` and `default_fields` options
 - Added global `index_prefix` option
+- Added `wait` option to async reindex
+- Raise error for `reindex_status` when Redis not configured
  
 ## 2.3.1
  
@@ -1214,6 +1214,12 @@ And use:
 Searchkick.reindex_status(index_name)
 ```
  
+You can also have Searchkick wait for reindexing to complete [master]
+
+```ruby
+Searchkick.reindex(async: {wait: true})
+```
+
 You can use [ActiveJob::TrafficControl](https://github.com/nickelser/activejob-traffic_control) to control concurrency. Install the gem:
  
 ```ruby
@@ -8,6 +8,7 @@ require &quot;searchkick/indexer&quot;
 require "searchkick/reindex_queue"
 require "searchkick/results"
 require "searchkick/query"
+require "searchkick/multi_search"
 require "searchkick/model"
 require "searchkick/tasks"
 require "searchkick/middleware"
@@ -101,14 +102,8 @@ module Searchkick
     end
   end
  
-  def self.multi_search(queries)
-    if queries.any?
-      responses = client.msearch(body: queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"]
-      queries.each_with_index do |query, i|
-        query.handle_response(responses[i])
-      end
-    end
-    queries
+  def self.multi_search(queries, retry_misspellings: false)
+    Searchkick::MultiSearch.new(queries, retry_misspellings: retry_misspellings).perform
   end
  
   # callbacks
@@ -153,6 +148,8 @@ module Searchkick
         completed: batches_left == 0,
         batches_left: batches_left
       }
+    else
+      raise Searchkick::Error, "Redis not configured"
     end
   end
  
@@ -15,7 +15,13 @@ module Searchkick
     end
  
     def delete
-      client.indices.delete index: name
+      if !Searchkick.server_below?("6.0.0-alpha1") && alias_exists?
+        # can't call delete directly on aliases in ES 6
+        indices = client.indices.get_alias(name: name).keys
+        client.indices.delete index: indices
+      else
+        client.indices.delete index: name
+      end
     end
  
     def exists?
@@ -228,7 +234,8 @@ module Searchkick
       end
  
       # check if alias exists
-      if alias_exists?
+      alias_exists = alias_exists?
+      if alias_exists
         # import before promotion
         index.import_scope(scope, resume: resume, async: async, full: true) if import
  
@@ -246,6 +253,24 @@ module Searchkick
       end
  
       if async
+        if async.is_a?(Hash) && async[:wait]
+          puts "Created index: #{index.name}"
+          puts "Jobs queued. Waiting..."
+          loop do
+            sleep 3
+            status = Searchkick.reindex_status(index.name)
+            break if status[:completed]
+            puts "Batches left: #{status[:batches_left]}"
+          end
+          # already promoted if alias didn't exist
+          if alias_exists
+            puts "Jobs complete. Promoting..."
+            promote(index.name, update_refresh_interval: !refresh_interval.nil?)
+          end
+          clean_indices unless retain
+          puts "SUCCESS!"
+        end
+
         {index_name: index.name}
       else
         index.refresh
@@ -162,11 +162,6 @@ module Searchkick
           settings[:similarity] = {default: {type: options[:similarity]}}
         end
  
-        unless below60
-          settings[:mapping] ||= {}
-          settings[:mapping][:single_type] = false
-        end
-
         settings.deep_merge!(options[:settings] || {})
  
         # synonyms
@@ -188,7 +183,7 @@ module Searchkick
           # - Only apply the synonym expansion at index time
           # - Don't have the synonym filter applied search
           # - Use directional synonyms where appropriate. You want to make sure that you're not injecting terms that are too general.
-          settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_synonym")
+          settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_synonym") if below60
           settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_synonym"
  
           %w(word_start word_middle word_end).each do |type|
@@ -129,7 +129,7 @@ module Searchkick
   end
  
   module SearchkickWithInstrumentation
-    def multi_search(searches)
+    def multi_search(searches, **options)
       event = {
         name: "Multi Search",
         body: searches.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join
@@ -0,0 +1,42 @@
+module Searchkick
+  class MultiSearch
+    attr_reader :queries
+
+    def initialize(queries, retry_misspellings: false)
+      @queries = queries
+      @retry_misspellings = retry_misspellings
+    end
+
+    def perform
+      if queries.any?
+        perform_search(queries, retry_misspellings: @retry_misspellings)
+      end
+    end
+
+    private
+
+    def perform_search(queries, retry_misspellings: true)
+      responses = client.msearch(body: queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"]
+
+      retry_queries = []
+      queries.each_with_index do |query, i|
+        if retry_misspellings && query.retry_misspellings?(responses[i])
+          query.send(:prepare) # okay, since we don't want to expose this method outside Searchkick
+          retry_queries << query
+        else
+          query.handle_response(responses[i])
+        end
+      end
+
+      if retry_misspellings && retry_queries.any?
+        perform_search(retry_queries, retry_misspellings: false)
+      end
+
+      queries
+    end
+
+    def client
+      Searchkick.client
+    end
+  end
+end
@@ -79,7 +79,7 @@ module Searchkick
       @execute ||= begin
         begin
           response = execute_search
-          if @misspellings_below && response["hits"]["total"] < @misspellings_below
+          if retry_misspellings?(response)
             prepare
             response = execute_search
           end
@@ -160,6 +160,10 @@ module Searchkick
       @execute = Searchkick::Results.new(searchkick_klass, response, opts)
     end
  
+    def retry_misspellings?(response)
+      @misspellings_below && response["hits"]["total"] < @misspellings_below
+    end
+
     private
  
     def handle_error(e)
@@ -32,7 +32,7 @@ class HighlightTest &lt; Minitest::Test
  
   def test_field_options
     store_names ["Two Door Cinema Club are a Northern Irish indie rock band"]
-    fragment_size = ENV["MATCH"] == "word_start" ? 26 : 20
+    fragment_size = ENV["MATCH"] == "word_start" ? 26 : 21
     assert_equal "Two Door <em>Cinema</em> Club are", Product.search("cinema", fields: [:name], highlight: {fields: {name: {fragment_size: fragment_size}}}).first.search_highlights[:name]
   end
  
@@ -19,4 +19,18 @@ class MultiSearchTest &lt; Minitest::Test
     assert !products.error
     assert stores.error
   end
+
+  def test_misspellings_below_unmet
+    store_names ["abc", "abd", "aee"]
+    products = Product.search("abc", misspellings: {below: 2}, execute: false)
+    Searchkick.multi_search([products])
+    assert_equal ["abc"], products.map(&:name)
+  end
+
+  def test_misspellings_below_unmet_retry
+    store_names ["abc", "abd", "aee"]
+    products = Product.search("abc", misspellings: {below: 2}, execute: false)
+    Searchkick.multi_search([products], retry_misspellings: true)
+    assert_equal ["abc", "abd"], products.map(&:name)
+  end
 end
...	...	@@ -33,6 +33,7 @@ matrix:
33	33	- gemfile: Gemfile
34	34	env: ELASTICSEARCH_VERSION=5.0.1
35	35	jdk: oraclejdk8
	36	+ allow_failures:
36	37	- gemfile: Gemfile
37		- env: ELASTICSEARCH_VERSION=6.0.0-alpha2
	38	+ env: ELASTICSEARCH_VERSION=6.0.0-beta1
38	39	jdk: oraclejdk8
...	...
...	...	@@ -2,6 +2,8 @@
2	2
3	3	- Added `_all` and `default_fields` options
4	4	- Added global `index_prefix` option
	5	+- Added `wait` option to async reindex
	6	+- Raise error for `reindex_status` when Redis not configured
5	7
6	8	## 2.3.1
7	9
...	...
...	...	@@ -1214,6 +1214,12 @@ And use:
1214	1214	Searchkick.reindex_status(index_name)
1215	1215	```
1216	1216
	1217	+You can also have Searchkick wait for reindexing to complete [master]
	1218	+
	1219	+```ruby
	1220	+Searchkick.reindex(async: {wait: true})
	1221	+```
	1222	+
1217	1223	You can use [ActiveJob::TrafficControl](https://github.com/nickelser/activejob-traffic_control) to control concurrency. Install the gem:
1218	1224
1219	1225	```ruby
...	...
...	...	@@ -8,6 +8,7 @@ require "searchkick/indexer"
8	8	require "searchkick/reindex_queue"
9	9	require "searchkick/results"
10	10	require "searchkick/query"
	11	+require "searchkick/multi_search"
11	12	require "searchkick/model"
12	13	require "searchkick/tasks"
13	14	require "searchkick/middleware"
...	...	@@ -101,14 +102,8 @@ module Searchkick
101	102	end
102	103	end
103	104
104		- def self.multi_search(queries)
105		- if queries.any?
106		- responses = client.msearch(body: queries.flat_map { \|q\| [q.params.except(:body), q.body] })["responses"]
107		- queries.each_with_index do \|query, i\|
108		- query.handle_response(responses[i])
109		- end
110		- end
111		- queries
	105	+ def self.multi_search(queries, retry_misspellings: false)
	106	+ Searchkick::MultiSearch.new(queries, retry_misspellings: retry_misspellings).perform
112	107	end
113	108
114	109	# callbacks
...	...	@@ -153,6 +148,8 @@ module Searchkick
153	148	completed: batches_left == 0,
154	149	batches_left: batches_left
155	150	}
	151	+ else
	152	+ raise Searchkick::Error, "Redis not configured"
156	153	end
157	154	end
158	155
...	...
...	...	@@ -15,7 +15,13 @@ module Searchkick
15	15	end
16	16
17	17	def delete
18		- client.indices.delete index: name
	18	+ if !Searchkick.server_below?("6.0.0-alpha1") && alias_exists?
	19	+ # can't call delete directly on aliases in ES 6
	20	+ indices = client.indices.get_alias(name: name).keys
	21	+ client.indices.delete index: indices
	22	+ else
	23	+ client.indices.delete index: name
	24	+ end
19	25	end
20	26
21	27	def exists?
...	...	@@ -228,7 +234,8 @@ module Searchkick
228	234	end
229	235
230	236	# check if alias exists
231		- if alias_exists?
	237	+ alias_exists = alias_exists?
	238	+ if alias_exists
232	239	# import before promotion
233	240	index.import_scope(scope, resume: resume, async: async, full: true) if import
234	241
...	...	@@ -246,6 +253,24 @@ module Searchkick
246	253	end
247	254
248	255	if async
	256	+ if async.is_a?(Hash) && async[:wait]
	257	+ puts "Created index: #{index.name}"
	258	+ puts "Jobs queued. Waiting..."
	259	+ loop do
	260	+ sleep 3
	261	+ status = Searchkick.reindex_status(index.name)
	262	+ break if status[:completed]
	263	+ puts "Batches left: #{status[:batches_left]}"
	264	+ end
	265	+ # already promoted if alias didn't exist
	266	+ if alias_exists
	267	+ puts "Jobs complete. Promoting..."
	268	+ promote(index.name, update_refresh_interval: !refresh_interval.nil?)
	269	+ end
	270	+ clean_indices unless retain
	271	+ puts "SUCCESS!"
	272	+ end
	273	+
249	274	{index_name: index.name}
250	275	else
251	276	index.refresh
...	...
...	...	@@ -162,11 +162,6 @@ module Searchkick
162	162	settings[:similarity] = {default: {type: options[:similarity]}}
163	163	end
164	164
165		- unless below60
166		- settings[:mapping] \|\|= {}
167		- settings[:mapping][:single_type] = false
168		- end
169		-
170	165	settings.deep_merge!(options[:settings] \|\| {})
171	166
172	167	# synonyms
...	...	@@ -188,7 +183,7 @@ module Searchkick
188	183	# - Only apply the synonym expansion at index time
189	184	# - Don't have the synonym filter applied search
190	185	# - Use directional synonyms where appropriate. You want to make sure that you're not injecting terms that are too general.
191		- settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_synonym")
	186	+ settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_synonym") if below60
192	187	settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_synonym"
193	188
194	189	%w(word_start word_middle word_end).each do \|type\|
...	...
...	...	@@ -129,7 +129,7 @@ module Searchkick
129	129	end
130	130
131	131	module SearchkickWithInstrumentation
132		- def multi_search(searches)
	132	+ def multi_search(searches, **options)
133	133	event = {
134	134	name: "Multi Search",
135	135	body: searches.flat_map { \|q\| [q.params.except(:body).to_json, q.body.to_json] }.map { \|v\| "#{v}\n" }.join
...	...
...	...	@@ -0,0 +1,42 @@
	1	+module Searchkick
	2	+ class MultiSearch
	3	+ attr_reader :queries
	4	+
	5	+ def initialize(queries, retry_misspellings: false)
	6	+ @queries = queries
	7	+ @retry_misspellings = retry_misspellings
	8	+ end
	9	+
	10	+ def perform
	11	+ if queries.any?
	12	+ perform_search(queries, retry_misspellings: @retry_misspellings)
	13	+ end
	14	+ end
	15	+
	16	+ private
	17	+
	18	+ def perform_search(queries, retry_misspellings: true)
	19	+ responses = client.msearch(body: queries.flat_map { \|q\| [q.params.except(:body), q.body] })["responses"]
	20	+
	21	+ retry_queries = []
	22	+ queries.each_with_index do \|query, i\|
	23	+ if retry_misspellings && query.retry_misspellings?(responses[i])
	24	+ query.send(:prepare) # okay, since we don't want to expose this method outside Searchkick
	25	+ retry_queries << query
	26	+ else
	27	+ query.handle_response(responses[i])
	28	+ end
	29	+ end
	30	+
	31	+ if retry_misspellings && retry_queries.any?
	32	+ perform_search(retry_queries, retry_misspellings: false)
	33	+ end
	34	+
	35	+ queries
	36	+ end
	37	+
	38	+ def client
	39	+ Searchkick.client
	40	+ end
	41	+ end
	42	+end
...	...
...	...	@@ -79,7 +79,7 @@ module Searchkick
79	79	@execute \|\|= begin
80	80	begin
81	81	response = execute_search
82		- if @misspellings_below && response["hits"]["total"] < @misspellings_below
	82	+ if retry_misspellings?(response)
83	83	prepare
84	84	response = execute_search
85	85	end
...	...	@@ -160,6 +160,10 @@ module Searchkick
160	160	@execute = Searchkick::Results.new(searchkick_klass, response, opts)
161	161	end
162	162
	163	+ def retry_misspellings?(response)
	164	+ @misspellings_below && response["hits"]["total"] < @misspellings_below
	165	+ end
	166	+
163	167	private
164	168
165	169	def handle_error(e)
...	...
...	...	@@ -32,7 +32,7 @@ class HighlightTest < Minitest::Test
32	32
33	33	def test_field_options
34	34	store_names ["Two Door Cinema Club are a Northern Irish indie rock band"]
35		- fragment_size = ENV["MATCH"] == "word_start" ? 26 : 20
	35	+ fragment_size = ENV["MATCH"] == "word_start" ? 26 : 21
36	36	assert_equal "Two Door <em>Cinema</em> Club are", Product.search("cinema", fields: [:name], highlight: {fields: {name: {fragment_size: fragment_size}}}).first.search_highlights[:name]
37	37	end
38	38
...	...