Commit e5550dd2a48caf1391af51d9e8c184dddbeffaa2
Exists in
reindex_refactor
Merge branch 'master' into reindex_refactor
Showing
9 changed files
with
141 additions
and
102 deletions
Show diff stats
.travis.yml
1 | 1 | dist: bionic |
2 | 2 | language: ruby |
3 | -rvm: 2.6 | |
4 | 3 | gemfile: |
5 | 4 | - Gemfile |
6 | 5 | services: |
... | ... | @@ -18,20 +17,26 @@ cache: |
18 | 17 | env: |
19 | 18 | - ELASTICSEARCH_VERSION=7.6.1 |
20 | 19 | jdk: openjdk10 |
21 | -matrix: | |
20 | +jobs: | |
22 | 21 | include: |
23 | - - gemfile: Gemfile | |
24 | - - gemfile: test/gemfiles/activerecord52.gemfile | |
22 | + - rvm: 2.7 | |
23 | + gemfile: Gemfile | |
24 | + - rvm: 2.6 | |
25 | + gemfile: test/gemfiles/activerecord52.gemfile | |
25 | 26 | env: ELASTICSEARCH_VERSION=7.0.0 |
26 | - - gemfile: test/gemfiles/activerecord51.gemfile | |
27 | + - rvm: 2.5 | |
28 | + gemfile: test/gemfiles/activerecord51.gemfile | |
27 | 29 | env: ELASTICSEARCH_VERSION=6.8.7 |
28 | - - gemfile: test/gemfiles/activerecord50.gemfile | |
30 | + - rvm: 2.4 | |
31 | + gemfile: test/gemfiles/activerecord50.gemfile | |
29 | 32 | env: ELASTICSEARCH_VERSION=6.0.0 |
30 | - - gemfile: test/gemfiles/mongoid7.gemfile | |
33 | + - rvm: 2.7 | |
34 | + gemfile: test/gemfiles/mongoid7.gemfile | |
31 | 35 | services: |
32 | 36 | - mongodb |
33 | 37 | - redis-server |
34 | - - gemfile: test/gemfiles/mongoid6.gemfile | |
38 | + - rvm: 2.6 | |
39 | + gemfile: test/gemfiles/mongoid6.gemfile | |
35 | 40 | services: |
36 | 41 | - mongodb |
37 | 42 | - redis-server | ... | ... |
lib/searchkick.rb
1 | +# dependencies | |
1 | 2 | require "active_support" |
2 | 3 | require "active_support/core_ext/hash/deep_merge" |
3 | 4 | require "elasticsearch" |
4 | 5 | require "hashie" |
5 | 6 | |
7 | +# modules | |
6 | 8 | require "searchkick/bulk_indexer" |
7 | 9 | require "searchkick/index" |
8 | 10 | require "searchkick/indexer" |
... | ... | @@ -17,6 +19,7 @@ require "searchkick/record_indexer" |
17 | 19 | require "searchkick/results" |
18 | 20 | require "searchkick/version" |
19 | 21 | |
22 | +# integrations | |
20 | 23 | require "searchkick/railtie" if defined?(Rails) |
21 | 24 | require "searchkick/logging" if defined?(ActiveSupport::Notifications) |
22 | 25 | |
... | ... | @@ -27,6 +30,7 @@ module Searchkick |
27 | 30 | autoload :ProcessQueueJob, "searchkick/process_queue_job" |
28 | 31 | autoload :ReindexV2Job, "searchkick/reindex_v2_job" |
29 | 32 | |
33 | + # errors | |
30 | 34 | class Error < StandardError; end |
31 | 35 | class MissingIndexError < Error; end |
32 | 36 | class UnsupportedVersionError < Error; end | ... | ... |
lib/searchkick/index_options.rb
... | ... | @@ -27,9 +27,6 @@ module Searchkick |
27 | 27 | default_analyzer = :searchkick_index |
28 | 28 | keyword_mapping = {type: "keyword"} |
29 | 29 | |
30 | - index_true_value = true | |
31 | - index_false_value = false | |
32 | - | |
33 | 30 | keyword_mapping[:ignore_above] = options[:ignore_above] || 30000 |
34 | 31 | |
35 | 32 | settings = { |
... | ... | @@ -356,13 +353,13 @@ module Searchkick |
356 | 353 | |
357 | 354 | mapping_options[:searchable].delete("_all") |
358 | 355 | |
359 | - analyzed_field_options = {type: default_type, index: index_true_value, analyzer: default_analyzer} | |
356 | + analyzed_field_options = {type: default_type, index: true, analyzer: default_analyzer} | |
360 | 357 | |
361 | 358 | mapping_options.values.flatten.uniq.each do |field| |
362 | 359 | fields = {} |
363 | 360 | |
364 | 361 | if options.key?(:filterable) && !mapping_options[:filterable].include?(field) |
365 | - fields[field] = {type: default_type, index: index_false_value} | |
362 | + fields[field] = {type: default_type, index: false} | |
366 | 363 | else |
367 | 364 | fields[field] = keyword_mapping |
368 | 365 | end |
... | ... | @@ -378,7 +375,7 @@ module Searchkick |
378 | 375 | |
379 | 376 | mapping_options.except(:highlight, :searchable, :filterable, :word).each do |type, f| |
380 | 377 | if options[:match] == type || f.include?(field) |
381 | - fields[type] = {type: default_type, index: index_true_value, analyzer: "searchkick_#{type}_index"} | |
378 | + fields[type] = {type: default_type, index: true, analyzer: "searchkick_#{type}_index"} | |
382 | 379 | end |
383 | 380 | end |
384 | 381 | end |
... | ... | @@ -418,12 +415,12 @@ module Searchkick |
418 | 415 | } |
419 | 416 | |
420 | 417 | if options.key?(:filterable) |
421 | - dynamic_fields["{name}"] = {type: default_type, index: index_false_value} | |
418 | + dynamic_fields["{name}"] = {type: default_type, index: false} | |
422 | 419 | end |
423 | 420 | |
424 | 421 | unless options[:searchable] |
425 | 422 | if options[:match] && options[:match] != :word |
426 | - dynamic_fields[options[:match]] = {type: default_type, index: index_true_value, analyzer: "searchkick_#{options[:match]}_index"} | |
423 | + dynamic_fields[options[:match]] = {type: default_type, index: true, analyzer: "searchkick_#{options[:match]}_index"} | |
427 | 424 | end |
428 | 425 | |
429 | 426 | if word | ... | ... |
test/callbacks_test.rb
... | ... | @@ -1,59 +0,0 @@ |
1 | -require_relative "test_helper" | |
2 | - | |
3 | -class CallbacksTest < Minitest::Test | |
4 | - def test_true_create | |
5 | - Searchkick.callbacks(true) do | |
6 | - store_names ["Product A", "Product B"] | |
7 | - end | |
8 | - Product.searchkick_index.refresh | |
9 | - assert_search "product", ["Product A", "Product B"] | |
10 | - end | |
11 | - | |
12 | - def test_false_create | |
13 | - Searchkick.callbacks(false) do | |
14 | - store_names ["Product A", "Product B"] | |
15 | - end | |
16 | - Product.searchkick_index.refresh | |
17 | - assert_search "product", [] | |
18 | - end | |
19 | - | |
20 | - def test_bulk_create | |
21 | - Searchkick.callbacks(:bulk) do | |
22 | - store_names ["Product A", "Product B"] | |
23 | - end | |
24 | - Product.searchkick_index.refresh | |
25 | - assert_search "product", ["Product A", "Product B"] | |
26 | - end | |
27 | - | |
28 | - def test_queue | |
29 | - skip unless defined?(ActiveJob) && defined?(Redis) | |
30 | - | |
31 | - reindex_queue = Product.searchkick_index.reindex_queue | |
32 | - reindex_queue.clear | |
33 | - | |
34 | - Searchkick.callbacks(:queue) do | |
35 | - store_names ["Product A", "Product B"] | |
36 | - end | |
37 | - Product.searchkick_index.refresh | |
38 | - assert_search "product", [], load: false, conversions: false | |
39 | - assert_equal 2, reindex_queue.length | |
40 | - | |
41 | - Searchkick::ProcessQueueJob.perform_later(class_name: "Product") | |
42 | - Product.searchkick_index.refresh | |
43 | - assert_search "product", ["Product A", "Product B"], load: false | |
44 | - assert_equal 0, reindex_queue.length | |
45 | - | |
46 | - Searchkick.callbacks(:queue) do | |
47 | - Product.where(name: "Product B").destroy_all | |
48 | - Product.create!(name: "Product C") | |
49 | - end | |
50 | - Product.searchkick_index.refresh | |
51 | - assert_search "product", ["Product A", "Product B"], load: false | |
52 | - assert_equal 2, reindex_queue.length | |
53 | - | |
54 | - Searchkick::ProcessQueueJob.perform_later(class_name: "Product") | |
55 | - Product.searchkick_index.refresh | |
56 | - assert_search "product", ["Product A", "Product C"], load: false | |
57 | - assert_equal 0, reindex_queue.length | |
58 | - end | |
59 | -end |
test/gemfiles/activerecord50.gemfile
test/gemfiles/activerecord51.gemfile
test/gemfiles/activerecord52.gemfile
test/reindex_test.rb
1 | 1 | require_relative "test_helper" |
2 | 2 | |
3 | 3 | class ReindexTest < Minitest::Test |
4 | - def setup | |
5 | - super | |
6 | - Sku.destroy_all | |
4 | + def test_record_inline | |
5 | + store_names ["Product A", "Product B"], reindex: false | |
6 | + | |
7 | + product = Product.find_by!(name: "Product A") | |
8 | + product.reindex(refresh: true) | |
9 | + assert_search "product", ["Product A"] | |
7 | 10 | end |
8 | 11 | |
9 | - def test_scoped | |
12 | + def test_record_async | |
13 | + store_names ["Product A", "Product B"], reindex: false | |
14 | + | |
15 | + product = Product.find_by!(name: "Product A") | |
16 | + product.reindex(mode: :async) | |
17 | + Product.search_index.refresh | |
18 | + assert_search "product", ["Product A"] | |
19 | + end | |
20 | + | |
21 | + def test_record_queue | |
22 | + skip unless defined?(ActiveJob) && defined?(Redis) | |
23 | + | |
24 | + reindex_queue = Product.searchkick_index.reindex_queue | |
25 | + reindex_queue.clear | |
26 | + | |
27 | + store_names ["Product A", "Product B"], reindex: false | |
28 | + | |
29 | + product = Product.find_by!(name: "Product A") | |
30 | + product.reindex(mode: :queue) | |
31 | + Product.search_index.refresh | |
32 | + assert_search "product", [] | |
33 | + | |
34 | + Searchkick::ProcessQueueJob.perform_now(class_name: "Product") | |
35 | + Product.search_index.refresh | |
36 | + assert_search "product", ["Product A"] | |
37 | + end | |
38 | + | |
39 | + def test_relation_inline | |
10 | 40 | skip if nobrainer? || cequel? |
11 | 41 | |
12 | 42 | store_names ["Product A"] |
13 | - Searchkick.callbacks(false) do | |
14 | - store_names ["Product B", "Product C"] | |
15 | - end | |
43 | + store_names ["Product B", "Product C"], reindex: false | |
16 | 44 | Product.where(name: "Product B").reindex(refresh: true) |
17 | 45 | assert_search "product", ["Product A", "Product B"] |
18 | 46 | end |
19 | 47 | |
20 | - def test_associations | |
48 | + def test_relation_associations | |
21 | 49 | skip if nobrainer? || cequel? |
22 | 50 | |
23 | 51 | store_names ["Product A"] |
... | ... | @@ -27,12 +55,18 @@ class ReindexTest < Minitest::Test |
27 | 55 | assert_search "product", ["Product A", "Product B"] |
28 | 56 | end |
29 | 57 | |
30 | - def test_async | |
58 | + def test_relation_async | |
59 | + skip "Not available yet" | |
60 | + end | |
61 | + | |
62 | + def test_relation_queue | |
63 | + skip "Not available yet" | |
64 | + end | |
65 | + | |
66 | + def test_full_async | |
31 | 67 | skip unless defined?(ActiveJob) |
32 | 68 | |
33 | - Searchkick.callbacks(false) do | |
34 | - store_names ["Product A"] | |
35 | - end | |
69 | + store_names ["Product A"], reindex: false | |
36 | 70 | reindex = Product.reindex(async: true) |
37 | 71 | assert_search "product", [], conversions: false |
38 | 72 | |
... | ... | @@ -48,12 +82,10 @@ class ReindexTest < Minitest::Test |
48 | 82 | assert_search "product", ["Product A"] |
49 | 83 | end |
50 | 84 | |
51 | - def test_async_wait | |
85 | + def test_full_async_wait | |
52 | 86 | skip unless defined?(ActiveJob) && defined?(Redis) |
53 | 87 | |
54 | - Searchkick.callbacks(false) do | |
55 | - store_names ["Product A"] | |
56 | - end | |
88 | + store_names ["Product A"], reindex: false | |
57 | 89 | |
58 | 90 | capture_io do |
59 | 91 | Product.reindex(async: {wait: true}) |
... | ... | @@ -62,7 +94,7 @@ class ReindexTest < Minitest::Test |
62 | 94 | assert_search "product", ["Product A"] |
63 | 95 | end |
64 | 96 | |
65 | - def test_async_non_integer_pk | |
97 | + def test_full_async_non_integer_pk | |
66 | 98 | skip unless defined?(ActiveJob) |
67 | 99 | |
68 | 100 | Sku.create(id: SecureRandom.hex, name: "Test") |
... | ... | @@ -72,9 +104,11 @@ class ReindexTest < Minitest::Test |
72 | 104 | index = Searchkick::Index.new(reindex[:index_name]) |
73 | 105 | index.refresh |
74 | 106 | assert_equal 1, index.total_docs |
107 | + ensure | |
108 | + Sku.destroy_all | |
75 | 109 | end |
76 | 110 | |
77 | - def test_refresh_interval | |
111 | + def test_full_refresh_interval | |
78 | 112 | reindex = Product.reindex(refresh_interval: "30s", async: true, import: false) |
79 | 113 | index = Searchkick::Index.new(reindex[:index_name]) |
80 | 114 | assert_nil Product.search_index.refresh_interval |
... | ... | @@ -85,17 +119,64 @@ class ReindexTest < Minitest::Test |
85 | 119 | assert_equal "1s", Product.search_index.refresh_interval |
86 | 120 | end |
87 | 121 | |
88 | - def test_resume | |
122 | + def test_full_resume | |
89 | 123 | assert Product.reindex(resume: true) |
90 | 124 | end |
91 | 125 | |
92 | - def test_refresh_full_reindex | |
126 | + def test_full_refresh | |
93 | 127 | Product.reindex(refresh: true) |
94 | 128 | end |
95 | 129 | |
96 | - def test_partial_async | |
130 | + def test_full_partial_async | |
97 | 131 | store_names ["Product A"] |
98 | 132 | # warn for now |
99 | 133 | Product.reindex(:search_name, async: true) |
100 | 134 | end |
135 | + | |
136 | + def test_callbacks_false | |
137 | + Searchkick.callbacks(false) do | |
138 | + store_names ["Product A", "Product B"] | |
139 | + end | |
140 | + assert_search "product", [] | |
141 | + end | |
142 | + | |
143 | + def test_callbacks_bulk | |
144 | + Searchkick.callbacks(:bulk) do | |
145 | + store_names ["Product A", "Product B"] | |
146 | + end | |
147 | + Product.search_index.refresh | |
148 | + assert_search "product", ["Product A", "Product B"] | |
149 | + end | |
150 | + | |
151 | + def test_callbacks_queue | |
152 | + skip unless defined?(ActiveJob) && defined?(Redis) | |
153 | + | |
154 | + reindex_queue = Product.searchkick_index.reindex_queue | |
155 | + reindex_queue.clear | |
156 | + | |
157 | + Searchkick.callbacks(:queue) do | |
158 | + store_names ["Product A", "Product B"] | |
159 | + end | |
160 | + Product.searchkick_index.refresh | |
161 | + assert_search "product", [], load: false, conversions: false | |
162 | + assert_equal 2, reindex_queue.length | |
163 | + | |
164 | + Searchkick::ProcessQueueJob.perform_later(class_name: "Product") | |
165 | + Product.searchkick_index.refresh | |
166 | + assert_search "product", ["Product A", "Product B"], load: false | |
167 | + assert_equal 0, reindex_queue.length | |
168 | + | |
169 | + Searchkick.callbacks(:queue) do | |
170 | + Product.where(name: "Product B").destroy_all | |
171 | + Product.create!(name: "Product C") | |
172 | + end | |
173 | + Product.searchkick_index.refresh | |
174 | + assert_search "product", ["Product A", "Product B"], load: false | |
175 | + assert_equal 2, reindex_queue.length | |
176 | + | |
177 | + Searchkick::ProcessQueueJob.perform_later(class_name: "Product") | |
178 | + Product.searchkick_index.refresh | |
179 | + assert_search "product", ["Product A", "Product C"], load: false | |
180 | + assert_equal 0, reindex_queue.length | |
181 | + end | |
101 | 182 | end | ... | ... |
test/test_helper.rb
... | ... | @@ -71,15 +71,23 @@ class Minitest::Test |
71 | 71 | |
72 | 72 | protected |
73 | 73 | |
74 | - def store(documents, klass = Product) | |
75 | - documents.shuffle.each do |document| | |
76 | - klass.create!(document) | |
74 | + def store(documents, klass = Product, reindex: true) | |
75 | + if reindex | |
76 | + documents.shuffle.each do |document| | |
77 | + klass.create!(document) | |
78 | + end | |
79 | + klass.searchkick_index.refresh | |
80 | + else | |
81 | + Searchkick.callbacks(false) do | |
82 | + documents.shuffle.each do |document| | |
83 | + klass.create!(document) | |
84 | + end | |
85 | + end | |
77 | 86 | end |
78 | - klass.searchkick_index.refresh | |
79 | 87 | end |
80 | 88 | |
81 | - def store_names(names, klass = Product) | |
82 | - store names.map { |name| {name: name} }, klass | |
89 | + def store_names(names, klass = Product, reindex: true) | |
90 | + store names.map { |name| {name: name} }, klass, reindex: reindex | |
83 | 91 | end |
84 | 92 | |
85 | 93 | # no order | ... | ... |