Commit 4c2d92bd534aedcad94d2e0fa45e7252ee709917
Exists in
master
and in
19 other branches
Merge upstream
Showing
11 changed files
with
106 additions
and
20 deletions
Show diff stats
.travis.yml
| @@ -33,6 +33,7 @@ matrix: | @@ -33,6 +33,7 @@ matrix: | ||
| 33 | - gemfile: Gemfile | 33 | - gemfile: Gemfile |
| 34 | env: ELASTICSEARCH_VERSION=5.0.1 | 34 | env: ELASTICSEARCH_VERSION=5.0.1 |
| 35 | jdk: oraclejdk8 | 35 | jdk: oraclejdk8 |
| 36 | + allow_failures: | ||
| 36 | - gemfile: Gemfile | 37 | - gemfile: Gemfile |
| 37 | - env: ELASTICSEARCH_VERSION=6.0.0-alpha2 | 38 | + env: ELASTICSEARCH_VERSION=6.0.0-beta1 |
| 38 | jdk: oraclejdk8 | 39 | jdk: oraclejdk8 |
CHANGELOG.md
| @@ -2,6 +2,8 @@ | @@ -2,6 +2,8 @@ | ||
| 2 | 2 | ||
| 3 | - Added `_all` and `default_fields` options | 3 | - Added `_all` and `default_fields` options |
| 4 | - Added global `index_prefix` option | 4 | - Added global `index_prefix` option |
| 5 | +- Added `wait` option to async reindex | ||
| 6 | +- Raise error for `reindex_status` when Redis not configured | ||
| 5 | 7 | ||
| 6 | ## 2.3.1 | 8 | ## 2.3.1 |
| 7 | 9 |
README.md
| @@ -1214,6 +1214,12 @@ And use: | @@ -1214,6 +1214,12 @@ And use: | ||
| 1214 | Searchkick.reindex_status(index_name) | 1214 | Searchkick.reindex_status(index_name) |
| 1215 | ``` | 1215 | ``` |
| 1216 | 1216 | ||
| 1217 | +You can also have Searchkick wait for reindexing to complete [master] | ||
| 1218 | + | ||
| 1219 | +```ruby | ||
| 1220 | +Searchkick.reindex(async: {wait: true}) | ||
| 1221 | +``` | ||
| 1222 | + | ||
| 1217 | You can use [ActiveJob::TrafficControl](https://github.com/nickelser/activejob-traffic_control) to control concurrency. Install the gem: | 1223 | You can use [ActiveJob::TrafficControl](https://github.com/nickelser/activejob-traffic_control) to control concurrency. Install the gem: |
| 1218 | 1224 | ||
| 1219 | ```ruby | 1225 | ```ruby |
lib/searchkick.rb
| @@ -8,6 +8,7 @@ require "searchkick/indexer" | @@ -8,6 +8,7 @@ require "searchkick/indexer" | ||
| 8 | require "searchkick/reindex_queue" | 8 | require "searchkick/reindex_queue" |
| 9 | require "searchkick/results" | 9 | require "searchkick/results" |
| 10 | require "searchkick/query" | 10 | require "searchkick/query" |
| 11 | +require "searchkick/multi_search" | ||
| 11 | require "searchkick/model" | 12 | require "searchkick/model" |
| 12 | require "searchkick/tasks" | 13 | require "searchkick/tasks" |
| 13 | require "searchkick/middleware" | 14 | require "searchkick/middleware" |
| @@ -101,14 +102,8 @@ module Searchkick | @@ -101,14 +102,8 @@ module Searchkick | ||
| 101 | end | 102 | end |
| 102 | end | 103 | end |
| 103 | 104 | ||
| 104 | - def self.multi_search(queries) | ||
| 105 | - if queries.any? | ||
| 106 | - responses = client.msearch(body: queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"] | ||
| 107 | - queries.each_with_index do |query, i| | ||
| 108 | - query.handle_response(responses[i]) | ||
| 109 | - end | ||
| 110 | - end | ||
| 111 | - queries | 105 | + def self.multi_search(queries, retry_misspellings: false) |
| 106 | + Searchkick::MultiSearch.new(queries, retry_misspellings: retry_misspellings).perform | ||
| 112 | end | 107 | end |
| 113 | 108 | ||
| 114 | # callbacks | 109 | # callbacks |
| @@ -153,6 +148,8 @@ module Searchkick | @@ -153,6 +148,8 @@ module Searchkick | ||
| 153 | completed: batches_left == 0, | 148 | completed: batches_left == 0, |
| 154 | batches_left: batches_left | 149 | batches_left: batches_left |
| 155 | } | 150 | } |
| 151 | + else | ||
| 152 | + raise Searchkick::Error, "Redis not configured" | ||
| 156 | end | 153 | end |
| 157 | end | 154 | end |
| 158 | 155 |
lib/searchkick/index.rb
| @@ -15,7 +15,13 @@ module Searchkick | @@ -15,7 +15,13 @@ module Searchkick | ||
| 15 | end | 15 | end |
| 16 | 16 | ||
| 17 | def delete | 17 | def delete |
| 18 | - client.indices.delete index: name | 18 | + if !Searchkick.server_below?("6.0.0-alpha1") && alias_exists? |
| 19 | + # can't call delete directly on aliases in ES 6 | ||
| 20 | + indices = client.indices.get_alias(name: name).keys | ||
| 21 | + client.indices.delete index: indices | ||
| 22 | + else | ||
| 23 | + client.indices.delete index: name | ||
| 24 | + end | ||
| 19 | end | 25 | end |
| 20 | 26 | ||
| 21 | def exists? | 27 | def exists? |
| @@ -228,7 +234,8 @@ module Searchkick | @@ -228,7 +234,8 @@ module Searchkick | ||
| 228 | end | 234 | end |
| 229 | 235 | ||
| 230 | # check if alias exists | 236 | # check if alias exists |
| 231 | - if alias_exists? | 237 | + alias_exists = alias_exists? |
| 238 | + if alias_exists | ||
| 232 | # import before promotion | 239 | # import before promotion |
| 233 | index.import_scope(scope, resume: resume, async: async, full: true) if import | 240 | index.import_scope(scope, resume: resume, async: async, full: true) if import |
| 234 | 241 | ||
| @@ -246,6 +253,24 @@ module Searchkick | @@ -246,6 +253,24 @@ module Searchkick | ||
| 246 | end | 253 | end |
| 247 | 254 | ||
| 248 | if async | 255 | if async |
| 256 | + if async.is_a?(Hash) && async[:wait] | ||
| 257 | + puts "Created index: #{index.name}" | ||
| 258 | + puts "Jobs queued. Waiting..." | ||
| 259 | + loop do | ||
| 260 | + sleep 3 | ||
| 261 | + status = Searchkick.reindex_status(index.name) | ||
| 262 | + break if status[:completed] | ||
| 263 | + puts "Batches left: #{status[:batches_left]}" | ||
| 264 | + end | ||
| 265 | + # already promoted if alias didn't exist | ||
| 266 | + if alias_exists | ||
| 267 | + puts "Jobs complete. Promoting..." | ||
| 268 | + promote(index.name, update_refresh_interval: !refresh_interval.nil?) | ||
| 269 | + end | ||
| 270 | + clean_indices unless retain | ||
| 271 | + puts "SUCCESS!" | ||
| 272 | + end | ||
| 273 | + | ||
| 249 | {index_name: index.name} | 274 | {index_name: index.name} |
| 250 | else | 275 | else |
| 251 | index.refresh | 276 | index.refresh |
lib/searchkick/index_options.rb
| @@ -162,11 +162,6 @@ module Searchkick | @@ -162,11 +162,6 @@ module Searchkick | ||
| 162 | settings[:similarity] = {default: {type: options[:similarity]}} | 162 | settings[:similarity] = {default: {type: options[:similarity]}} |
| 163 | end | 163 | end |
| 164 | 164 | ||
| 165 | - unless below60 | ||
| 166 | - settings[:mapping] ||= {} | ||
| 167 | - settings[:mapping][:single_type] = false | ||
| 168 | - end | ||
| 169 | - | ||
| 170 | settings.deep_merge!(options[:settings] || {}) | 165 | settings.deep_merge!(options[:settings] || {}) |
| 171 | 166 | ||
| 172 | # synonyms | 167 | # synonyms |
| @@ -188,7 +183,7 @@ module Searchkick | @@ -188,7 +183,7 @@ module Searchkick | ||
| 188 | # - Only apply the synonym expansion at index time | 183 | # - Only apply the synonym expansion at index time |
| 189 | # - Don't have the synonym filter applied search | 184 | # - Don't have the synonym filter applied search |
| 190 | # - Use directional synonyms where appropriate. You want to make sure that you're not injecting terms that are too general. | 185 | # - Use directional synonyms where appropriate. You want to make sure that you're not injecting terms that are too general. |
| 191 | - settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_synonym") | 186 | + settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_synonym") if below60 |
| 192 | settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_synonym" | 187 | settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_synonym" |
| 193 | 188 | ||
| 194 | %w(word_start word_middle word_end).each do |type| | 189 | %w(word_start word_middle word_end).each do |type| |
lib/searchkick/logging.rb
| @@ -129,7 +129,7 @@ module Searchkick | @@ -129,7 +129,7 @@ module Searchkick | ||
| 129 | end | 129 | end |
| 130 | 130 | ||
| 131 | module SearchkickWithInstrumentation | 131 | module SearchkickWithInstrumentation |
| 132 | - def multi_search(searches) | 132 | + def multi_search(searches, **options) |
| 133 | event = { | 133 | event = { |
| 134 | name: "Multi Search", | 134 | name: "Multi Search", |
| 135 | body: searches.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join | 135 | body: searches.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join |
| @@ -0,0 +1,42 @@ | @@ -0,0 +1,42 @@ | ||
| 1 | +module Searchkick | ||
| 2 | + class MultiSearch | ||
| 3 | + attr_reader :queries | ||
| 4 | + | ||
| 5 | + def initialize(queries, retry_misspellings: false) | ||
| 6 | + @queries = queries | ||
| 7 | + @retry_misspellings = retry_misspellings | ||
| 8 | + end | ||
| 9 | + | ||
| 10 | + def perform | ||
| 11 | + if queries.any? | ||
| 12 | + perform_search(queries, retry_misspellings: @retry_misspellings) | ||
| 13 | + end | ||
| 14 | + end | ||
| 15 | + | ||
| 16 | + private | ||
| 17 | + | ||
| 18 | + def perform_search(queries, retry_misspellings: true) | ||
| 19 | + responses = client.msearch(body: queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"] | ||
| 20 | + | ||
| 21 | + retry_queries = [] | ||
| 22 | + queries.each_with_index do |query, i| | ||
| 23 | + if retry_misspellings && query.retry_misspellings?(responses[i]) | ||
| 24 | + query.send(:prepare) # okay, since we don't want to expose this method outside Searchkick | ||
| 25 | + retry_queries << query | ||
| 26 | + else | ||
| 27 | + query.handle_response(responses[i]) | ||
| 28 | + end | ||
| 29 | + end | ||
| 30 | + | ||
| 31 | + if retry_misspellings && retry_queries.any? | ||
| 32 | + perform_search(retry_queries, retry_misspellings: false) | ||
| 33 | + end | ||
| 34 | + | ||
| 35 | + queries | ||
| 36 | + end | ||
| 37 | + | ||
| 38 | + def client | ||
| 39 | + Searchkick.client | ||
| 40 | + end | ||
| 41 | + end | ||
| 42 | +end |
lib/searchkick/query.rb
| @@ -79,7 +79,7 @@ module Searchkick | @@ -79,7 +79,7 @@ module Searchkick | ||
| 79 | @execute ||= begin | 79 | @execute ||= begin |
| 80 | begin | 80 | begin |
| 81 | response = execute_search | 81 | response = execute_search |
| 82 | - if @misspellings_below && response["hits"]["total"] < @misspellings_below | 82 | + if retry_misspellings?(response) |
| 83 | prepare | 83 | prepare |
| 84 | response = execute_search | 84 | response = execute_search |
| 85 | end | 85 | end |
| @@ -160,6 +160,10 @@ module Searchkick | @@ -160,6 +160,10 @@ module Searchkick | ||
| 160 | @execute = Searchkick::Results.new(searchkick_klass, response, opts) | 160 | @execute = Searchkick::Results.new(searchkick_klass, response, opts) |
| 161 | end | 161 | end |
| 162 | 162 | ||
| 163 | + def retry_misspellings?(response) | ||
| 164 | + @misspellings_below && response["hits"]["total"] < @misspellings_below | ||
| 165 | + end | ||
| 166 | + | ||
| 163 | private | 167 | private |
| 164 | 168 | ||
| 165 | def handle_error(e) | 169 | def handle_error(e) |
test/highlight_test.rb
| @@ -32,7 +32,7 @@ class HighlightTest < Minitest::Test | @@ -32,7 +32,7 @@ class HighlightTest < Minitest::Test | ||
| 32 | 32 | ||
| 33 | def test_field_options | 33 | def test_field_options |
| 34 | store_names ["Two Door Cinema Club are a Northern Irish indie rock band"] | 34 | store_names ["Two Door Cinema Club are a Northern Irish indie rock band"] |
| 35 | - fragment_size = ENV["MATCH"] == "word_start" ? 26 : 20 | 35 | + fragment_size = ENV["MATCH"] == "word_start" ? 26 : 21 |
| 36 | assert_equal "Two Door <em>Cinema</em> Club are", Product.search("cinema", fields: [:name], highlight: {fields: {name: {fragment_size: fragment_size}}}).first.search_highlights[:name] | 36 | assert_equal "Two Door <em>Cinema</em> Club are", Product.search("cinema", fields: [:name], highlight: {fields: {name: {fragment_size: fragment_size}}}).first.search_highlights[:name] |
| 37 | end | 37 | end |
| 38 | 38 |
test/multi_search_test.rb
| @@ -19,4 +19,18 @@ class MultiSearchTest < Minitest::Test | @@ -19,4 +19,18 @@ class MultiSearchTest < Minitest::Test | ||
| 19 | assert !products.error | 19 | assert !products.error |
| 20 | assert stores.error | 20 | assert stores.error |
| 21 | end | 21 | end |
| 22 | + | ||
| 23 | + def test_misspellings_below_unmet | ||
| 24 | + store_names ["abc", "abd", "aee"] | ||
| 25 | + products = Product.search("abc", misspellings: {below: 2}, execute: false) | ||
| 26 | + Searchkick.multi_search([products]) | ||
| 27 | + assert_equal ["abc"], products.map(&:name) | ||
| 28 | + end | ||
| 29 | + | ||
| 30 | + def test_misspellings_below_unmet_retry | ||
| 31 | + store_names ["abc", "abd", "aee"] | ||
| 32 | + products = Product.search("abc", misspellings: {below: 2}, execute: false) | ||
| 33 | + Searchkick.multi_search([products], retry_misspellings: true) | ||
| 34 | + assert_equal ["abc", "abd"], products.map(&:name) | ||
| 35 | + end | ||
| 22 | end | 36 | end |