github / searchkick

Download as
Browse Code ยป

Commit d5be48ee83bbd2aa09d9f46680c70bfbaf511187

Authored by Andrew Kane
1 parent a72f5805
Exists in master and in 21 other branches allow_missing, conversions_period, conversions_v2, eager_load, find_in_batches, highlighted, index_name, lazy_index_name, less_deps, nested, reindex_async_threads, reindex_refactor, relation, relation_search, routing_queue, scope_attributes, search_as_you_type, synonyms_ci, test_language, threads, wildcard

Better multi search method, and fixed instrumentation for bulk updates

Showing 7 changed files with 98 additions and 30 deletions   Show diff stats
1 ## 1.2.1 [unreleased] 1 ## 1.2.1 [unreleased]
2 2
  3 +- Added `multi_search` method
3 - Added support for routing for Elasticsearch 2 4 - Added support for routing for Elasticsearch 2
4 - Added support for `search_document_id` and `search_document_type` in models 5 - Added support for `search_document_id` and `search_document_type` in models
  6 +- Fixed instrumentation for bulk updates
5 7
6 ## 1.2.0 8 ## 1.2.0
7 9
@@ -1092,6 +1092,18 @@ products = @@ -1092,6 +1092,18 @@ products =
1092 end 1092 end
1093 ``` 1093 ```
1094 1094
  1095 +### Multi Search [master]
  1096 +
  1097 +To batch search requests for performance, use:
  1098 +
  1099 +```ruby
  1100 +fresh_products = Product.search("fresh", execute: false)
  1101 +frozen_products = Product.search("frozen", execute: false)
  1102 +Searchkick.multi_search([fresh_products, frozen_products])
  1103 +```
  1104 +
  1105 +Then use `fresh_products` and `frozen_products` as typical results.
  1106 +
1095 ## Reference 1107 ## Reference
1096 1108
1097 Reindex one record 1109 Reindex one record
lib/searchkick.rb
  Diff comments   View file @d5be48e
@@ -143,10 +143,10 @@ module Searchkick @@ -143,10 +143,10 @@ module Searchkick
143 if queries.any? 143 if queries.any?
144 responses = client.msearch(body: queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"] 144 responses = client.msearch(body: queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"]
145 queries.each_with_index do |query, i| 145 queries.each_with_index do |query, i|
146 - query.send(:handle_response, responses[i]) 146 + query.handle_response(responses[i])
147 end 147 end
148 end 148 end
149 - true 149 + nil
150 end 150 end
151 end 151 end
152 152
lib/searchkick/logging.rb
  Diff comments   View file @d5be48e
@@ -19,8 +19,12 @@ module Searchkick @@ -19,8 +19,12 @@ module Searchkick
19 name: "#{record.searchkick_klass.name} Store", 19 name: "#{record.searchkick_klass.name} Store",
20 id: search_id(record) 20 id: search_id(record)
21 } 21 }
22 - ActiveSupport::Notifications.instrument("request.searchkick", event) do  
23 - super(record) 22 + if Searchkick.callbacks_value == :bulk
  23 + super
  24 + else
  25 + ActiveSupport::Notifications.instrument("request.searchkick", event) do
  26 + super
  27 + end
24 end 28 end
25 end 29 end
26 30
@@ -29,8 +33,12 @@ module Searchkick @@ -29,8 +33,12 @@ module Searchkick
29 name: "#{record.searchkick_klass.name} Remove", 33 name: "#{record.searchkick_klass.name} Remove",
30 id: search_id(record) 34 id: search_id(record)
31 } 35 }
32 - ActiveSupport::Notifications.instrument("request.searchkick", event) do  
33 - super(record) 36 + if Searchkick.callbacks_value == :bulk
  37 + super
  38 + else
  39 + ActiveSupport::Notifications.instrument("request.searchkick", event) do
  40 + super
  41 + end
34 end 42 end
35 end 43 end
36 44
@@ -47,6 +55,32 @@ module Searchkick @@ -47,6 +55,32 @@ module Searchkick
47 end 55 end
48 end 56 end
49 57
  58 + module SearchkickWithInstrumentation
  59 + def multi_search(searches)
  60 + event = {
  61 + name: "Multi Search",
  62 + count: searches.size
  63 + }
  64 + ActiveSupport::Notifications.instrument("request.searchkick", event) do
  65 + super
  66 + end
  67 + end
  68 +
  69 + def perform_items(items)
  70 + if callbacks_value == :bulk
  71 + event = {
  72 + name: "Bulk",
  73 + count: items.size
  74 + }
  75 + ActiveSupport::Notifications.instrument("request.searchkick", event) do
  76 + super
  77 + end
  78 + else
  79 + super
  80 + end
  81 + end
  82 + end
  83 +
50 # https://github.com/rails/rails/blob/master/activerecord/lib/active_record/log_subscriber.rb 84 # https://github.com/rails/rails/blob/master/activerecord/lib/active_record/log_subscriber.rb
51 class LogSubscriber < ActiveSupport::LogSubscriber 85 class LogSubscriber < ActiveSupport::LogSubscriber
52 def self.runtime=(value) 86 def self.runtime=(value)
@@ -129,6 +163,7 @@ module Searchkick @@ -129,6 +163,7 @@ module Searchkick
129 end 163 end
130 Searchkick::Query.send(:prepend, Searchkick::QueryWithInstrumentation) 164 Searchkick::Query.send(:prepend, Searchkick::QueryWithInstrumentation)
131 Searchkick::Index.send(:prepend, Searchkick::IndexWithInstrumentation) 165 Searchkick::Index.send(:prepend, Searchkick::IndexWithInstrumentation)
  166 +Searchkick.singleton_class.send(:prepend, Searchkick::SearchkickWithInstrumentation)
132 Searchkick::LogSubscriber.attach_to :searchkick 167 Searchkick::LogSubscriber.attach_to :searchkick
133 ActiveSupport.on_load(:action_controller) do 168 ActiveSupport.on_load(:action_controller) do
134 include Searchkick::ControllerRuntime 169 include Searchkick::ControllerRuntime
lib/searchkick/query.rb
  Diff comments   View file @d5be48e
@@ -5,7 +5,13 @@ module Searchkick @@ -5,7 +5,13 @@ module Searchkick
5 attr_reader :klass, :term, :options 5 attr_reader :klass, :term, :options
6 attr_accessor :body 6 attr_accessor :body
7 7
8 - def_delegators :execute, :map, :each, :any?, :empty?, :size, :length, :slice, :[], :to_ary 8 + def_delegators :execute, :map, :each, :any?, :empty?, :size, :length, :slice, :[], :to_ary,
  9 + :records, :results, :suggestions, :each_with_hit, :with_details, :facets, :aggregations, :aggs,
  10 + :took, :error, :model_name, :entry_name, :total_count, :total_entries,
  11 + :current_page, :per_page, :limit_value, :padding, :total_pages, :num_pages,
  12 + :offset_value, :offset, :previous_page, :prev_page, :next_page, :first_page?, :last_page?,
  13 + :out_of_range?, :hits
  14 +
9 15
10 def initialize(klass, term, options = {}) 16 def initialize(klass, term, options = {})
11 if term.is_a?(Hash) 17 if term.is_a?(Hash)
@@ -84,6 +90,29 @@ module Searchkick @@ -84,6 +90,29 @@ module Searchkick
84 "curl #{host[:protocol]}://#{credentials}#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -d '#{query[:body].to_json}'" 90 "curl #{host[:protocol]}://#{credentials}#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -d '#{query[:body].to_json}'"
85 end 91 end
86 92
  93 + def handle_response(response)
  94 + # apply facet limit in client due to
  95 + # https://github.com/elasticsearch/elasticsearch/issues/1305
  96 + @facet_limits.each do |field, limit|
  97 + field = field.to_s
  98 + facet = response["facets"][field]
  99 + response["facets"][field]["terms"] = facet["terms"].first(limit)
  100 + response["facets"][field]["other"] = facet["total"] - facet["terms"].sum { |term| term["count"] }
  101 + end
  102 +
  103 + opts = {
  104 + page: @page,
  105 + per_page: @per_page,
  106 + padding: @padding,
  107 + load: @load,
  108 + includes: options[:include] || options[:includes],
  109 + json: !options[:json].nil?,
  110 + match_suffix: @match_suffix,
  111 + highlighted_fields: @highlighted_fields || []
  112 + }
  113 + @execute = Searchkick::Results.new(searchkick_klass, response, opts)
  114 + end
  115 +
87 private 116 private
88 117
89 def handle_error(e) 118 def handle_error(e)
@@ -109,29 +138,6 @@ module Searchkick @@ -109,29 +138,6 @@ module Searchkick
109 end 138 end
110 end 139 end
111 140
112 - def handle_response(response)  
113 - # apply facet limit in client due to  
114 - # https://github.com/elasticsearch/elasticsearch/issues/1305  
115 - @facet_limits.each do |field, limit|  
116 - field = field.to_s  
117 - facet = response["facets"][field]  
118 - response["facets"][field]["terms"] = facet["terms"].first(limit)  
119 - response["facets"][field]["other"] = facet["total"] - facet["terms"].sum { |term| term["count"] }  
120 - end  
121 -  
122 - opts = {  
123 - page: @page,  
124 - per_page: @per_page,  
125 - padding: @padding,  
126 - load: @load,  
127 - includes: options[:include] || options[:includes],  
128 - json: !options[:json].nil?,  
129 - match_suffix: @match_suffix,  
130 - highlighted_fields: @highlighted_fields || []  
131 - }  
132 - @execute = Searchkick::Results.new(searchkick_klass, response, opts)  
133 - end  
134 -  
135 def reindex_command 141 def reindex_command
136 searchkick_klass ? "#{searchkick_klass.name}.reindex" : "reindex" 142 searchkick_klass ? "#{searchkick_klass.name}.reindex" : "reindex"
137 end 143 end
lib/searchkick/results.rb
  Diff comments   View file @d5be48e
@@ -108,6 +108,10 @@ module Searchkick @@ -108,6 +108,10 @@ module Searchkick
108 response["took"] 108 response["took"]
109 end 109 end
110 110
  111 + def error
  112 + response["error"]
  113 + end
  114 +
111 def model_name 115 def model_name
112 klass.model_name 116 klass.model_name
113 end 117 end
test/multi_search_test.rb
  Diff comments   View file @d5be48e
@@ -10,4 +10,13 @@ class MultiSearchTest &lt; Minitest::Test @@ -10,4 +10,13 @@ class MultiSearchTest &lt; Minitest::Test
10 assert_equal ["Product A"], products.map(&:name) 10 assert_equal ["Product A"], products.map(&:name)
11 assert_equal ["Store A"], stores.map(&:name) 11 assert_equal ["Store A"], stores.map(&:name)
12 end 12 end
  13 +
  14 + def test_error
  15 + store_names ["Product A"]
  16 + products = Product.search("*", execute: false)
  17 + stores = Store.search("*", order: [:bad_field], execute: false)
  18 + Searchkick.multi_search([products, stores])
  19 + assert !products.error
  20 + assert stores.error
  21 + end
13 end 22 end