github / searchkick

Download as
Browse Code ยป

Commit d5be48ee83bbd2aa09d9f46680c70bfbaf511187

Authored by Andrew Kane
1 parent a72f5805
Exists in master and in 21 other branches allow_missing, conversions_period, conversions_v2, eager_load, find_in_batches, highlighted, index_name, lazy_index_name, less_deps, nested, reindex_async_threads, reindex_refactor, relation, relation_search, routing_queue, scope_attributes, search_as_you_type, synonyms_ci, test_language, threads, wildcard

Better multi search method, and fixed instrumentation for bulk updates

Showing 7 changed files with 98 additions and 30 deletions   Show diff stats
CHANGELOG.md
  Diff comments   View file @d5be48e
1 1 ## 1.2.1 [unreleased]
2 2  
  3 +- Added `multi_search` method
3 4 - Added support for routing for Elasticsearch 2
4 5 - Added support for `search_document_id` and `search_document_type` in models
  6 +- Fixed instrumentation for bulk updates
5 7  
6 8 ## 1.2.0
7 9  
... ...
README.md
  Diff comments   View file @d5be48e
... ... @@ -1092,6 +1092,18 @@ products =
1092 1092 end
1093 1093 ```
1094 1094  
  1095 +### Multi Search [master]
  1096 +
  1097 +To batch search requests for performance, use:
  1098 +
  1099 +```ruby
  1100 +fresh_products = Product.search("fresh", execute: false)
  1101 +frozen_products = Product.search("frozen", execute: false)
  1102 +Searchkick.multi_search([fresh_products, frozen_products])
  1103 +```
  1104 +
  1105 +Then use `fresh_products` and `frozen_products` as typical results.
  1106 +
1095 1107 ## Reference
1096 1108  
1097 1109 Reindex one record
... ...
lib/searchkick.rb
  Diff comments   View file @d5be48e
... ... @@ -143,10 +143,10 @@ module Searchkick
143 143 if queries.any?
144 144 responses = client.msearch(body: queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"]
145 145 queries.each_with_index do |query, i|
146   - query.send(:handle_response, responses[i])
  146 + query.handle_response(responses[i])
147 147 end
148 148 end
149   - true
  149 + nil
150 150 end
151 151 end
152 152  
... ...
lib/searchkick/logging.rb
  Diff comments   View file @d5be48e
... ... @@ -19,8 +19,12 @@ module Searchkick
19 19 name: "#{record.searchkick_klass.name} Store",
20 20 id: search_id(record)
21 21 }
22   - ActiveSupport::Notifications.instrument("request.searchkick", event) do
23   - super(record)
  22 + if Searchkick.callbacks_value == :bulk
  23 + super
  24 + else
  25 + ActiveSupport::Notifications.instrument("request.searchkick", event) do
  26 + super
  27 + end
24 28 end
25 29 end
26 30  
... ... @@ -29,8 +33,12 @@ module Searchkick
29 33 name: "#{record.searchkick_klass.name} Remove",
30 34 id: search_id(record)
31 35 }
32   - ActiveSupport::Notifications.instrument("request.searchkick", event) do
33   - super(record)
  36 + if Searchkick.callbacks_value == :bulk
  37 + super
  38 + else
  39 + ActiveSupport::Notifications.instrument("request.searchkick", event) do
  40 + super
  41 + end
34 42 end
35 43 end
36 44  
... ... @@ -47,6 +55,32 @@ module Searchkick
47 55 end
48 56 end
49 57  
  58 + module SearchkickWithInstrumentation
  59 + def multi_search(searches)
  60 + event = {
  61 + name: "Multi Search",
  62 + count: searches.size
  63 + }
  64 + ActiveSupport::Notifications.instrument("request.searchkick", event) do
  65 + super
  66 + end
  67 + end
  68 +
  69 + def perform_items(items)
  70 + if callbacks_value == :bulk
  71 + event = {
  72 + name: "Bulk",
  73 + count: items.size
  74 + }
  75 + ActiveSupport::Notifications.instrument("request.searchkick", event) do
  76 + super
  77 + end
  78 + else
  79 + super
  80 + end
  81 + end
  82 + end
  83 +
50 84 # https://github.com/rails/rails/blob/master/activerecord/lib/active_record/log_subscriber.rb
51 85 class LogSubscriber < ActiveSupport::LogSubscriber
52 86 def self.runtime=(value)
... ... @@ -129,6 +163,7 @@ module Searchkick
129 163 end
130 164 Searchkick::Query.send(:prepend, Searchkick::QueryWithInstrumentation)
131 165 Searchkick::Index.send(:prepend, Searchkick::IndexWithInstrumentation)
  166 +Searchkick.singleton_class.send(:prepend, Searchkick::SearchkickWithInstrumentation)
132 167 Searchkick::LogSubscriber.attach_to :searchkick
133 168 ActiveSupport.on_load(:action_controller) do
134 169 include Searchkick::ControllerRuntime
... ...
lib/searchkick/query.rb
  Diff comments   View file @d5be48e
... ... @@ -5,7 +5,13 @@ module Searchkick
5 5 attr_reader :klass, :term, :options
6 6 attr_accessor :body
7 7  
8   - def_delegators :execute, :map, :each, :any?, :empty?, :size, :length, :slice, :[], :to_ary
  8 + def_delegators :execute, :map, :each, :any?, :empty?, :size, :length, :slice, :[], :to_ary,
  9 + :records, :results, :suggestions, :each_with_hit, :with_details, :facets, :aggregations, :aggs,
  10 + :took, :error, :model_name, :entry_name, :total_count, :total_entries,
  11 + :current_page, :per_page, :limit_value, :padding, :total_pages, :num_pages,
  12 + :offset_value, :offset, :previous_page, :prev_page, :next_page, :first_page?, :last_page?,
  13 + :out_of_range?, :hits
  14 +
9 15  
10 16 def initialize(klass, term, options = {})
11 17 if term.is_a?(Hash)
... ... @@ -84,6 +90,29 @@ module Searchkick
84 90 "curl #{host[:protocol]}://#{credentials}#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -d '#{query[:body].to_json}'"
85 91 end
86 92  
  93 + def handle_response(response)
  94 + # apply facet limit in client due to
  95 + # https://github.com/elasticsearch/elasticsearch/issues/1305
  96 + @facet_limits.each do |field, limit|
  97 + field = field.to_s
  98 + facet = response["facets"][field]
  99 + response["facets"][field]["terms"] = facet["terms"].first(limit)
  100 + response["facets"][field]["other"] = facet["total"] - facet["terms"].sum { |term| term["count"] }
  101 + end
  102 +
  103 + opts = {
  104 + page: @page,
  105 + per_page: @per_page,
  106 + padding: @padding,
  107 + load: @load,
  108 + includes: options[:include] || options[:includes],
  109 + json: !options[:json].nil?,
  110 + match_suffix: @match_suffix,
  111 + highlighted_fields: @highlighted_fields || []
  112 + }
  113 + @execute = Searchkick::Results.new(searchkick_klass, response, opts)
  114 + end
  115 +
87 116 private
88 117  
89 118 def handle_error(e)
... ... @@ -109,29 +138,6 @@ module Searchkick
109 138 end
110 139 end
111 140  
112   - def handle_response(response)
113   - # apply facet limit in client due to
114   - # https://github.com/elasticsearch/elasticsearch/issues/1305
115   - @facet_limits.each do |field, limit|
116   - field = field.to_s
117   - facet = response["facets"][field]
118   - response["facets"][field]["terms"] = facet["terms"].first(limit)
119   - response["facets"][field]["other"] = facet["total"] - facet["terms"].sum { |term| term["count"] }
120   - end
121   -
122   - opts = {
123   - page: @page,
124   - per_page: @per_page,
125   - padding: @padding,
126   - load: @load,
127   - includes: options[:include] || options[:includes],
128   - json: !options[:json].nil?,
129   - match_suffix: @match_suffix,
130   - highlighted_fields: @highlighted_fields || []
131   - }
132   - @execute = Searchkick::Results.new(searchkick_klass, response, opts)
133   - end
134   -
135 141 def reindex_command
136 142 searchkick_klass ? "#{searchkick_klass.name}.reindex" : "reindex"
137 143 end
... ...
lib/searchkick/results.rb
  Diff comments   View file @d5be48e
... ... @@ -108,6 +108,10 @@ module Searchkick
108 108 response["took"]
109 109 end
110 110  
  111 + def error
  112 + response["error"]
  113 + end
  114 +
111 115 def model_name
112 116 klass.model_name
113 117 end
... ...
test/multi_search_test.rb
  Diff comments   View file @d5be48e
... ... @@ -10,4 +10,13 @@ class MultiSearchTest &lt; Minitest::Test
10 10 assert_equal ["Product A"], products.map(&:name)
11 11 assert_equal ["Store A"], stores.map(&:name)
12 12 end
  13 +
  14 + def test_error
  15 + store_names ["Product A"]
  16 + products = Product.search("*", execute: false)
  17 + stores = Store.search("*", order: [:bad_field], execute: false)
  18 + Searchkick.multi_search([products, stores])
  19 + assert !products.error
  20 + assert stores.error
  21 + end
13 22 end
... ...