Commit e5550dd2a48caf1391af51d9e8c184dddbeffaa2
Exists in
reindex_refactor
Merge branch 'master' into reindex_refactor
Showing
9 changed files
with
141 additions
and
102 deletions
Show diff stats
.travis.yml
1 | dist: bionic | 1 | dist: bionic |
2 | language: ruby | 2 | language: ruby |
3 | -rvm: 2.6 | ||
4 | gemfile: | 3 | gemfile: |
5 | - Gemfile | 4 | - Gemfile |
6 | services: | 5 | services: |
@@ -18,20 +17,26 @@ cache: | @@ -18,20 +17,26 @@ cache: | ||
18 | env: | 17 | env: |
19 | - ELASTICSEARCH_VERSION=7.6.1 | 18 | - ELASTICSEARCH_VERSION=7.6.1 |
20 | jdk: openjdk10 | 19 | jdk: openjdk10 |
21 | -matrix: | 20 | +jobs: |
22 | include: | 21 | include: |
23 | - - gemfile: Gemfile | ||
24 | - - gemfile: test/gemfiles/activerecord52.gemfile | 22 | + - rvm: 2.7 |
23 | + gemfile: Gemfile | ||
24 | + - rvm: 2.6 | ||
25 | + gemfile: test/gemfiles/activerecord52.gemfile | ||
25 | env: ELASTICSEARCH_VERSION=7.0.0 | 26 | env: ELASTICSEARCH_VERSION=7.0.0 |
26 | - - gemfile: test/gemfiles/activerecord51.gemfile | 27 | + - rvm: 2.5 |
28 | + gemfile: test/gemfiles/activerecord51.gemfile | ||
27 | env: ELASTICSEARCH_VERSION=6.8.7 | 29 | env: ELASTICSEARCH_VERSION=6.8.7 |
28 | - - gemfile: test/gemfiles/activerecord50.gemfile | 30 | + - rvm: 2.4 |
31 | + gemfile: test/gemfiles/activerecord50.gemfile | ||
29 | env: ELASTICSEARCH_VERSION=6.0.0 | 32 | env: ELASTICSEARCH_VERSION=6.0.0 |
30 | - - gemfile: test/gemfiles/mongoid7.gemfile | 33 | + - rvm: 2.7 |
34 | + gemfile: test/gemfiles/mongoid7.gemfile | ||
31 | services: | 35 | services: |
32 | - mongodb | 36 | - mongodb |
33 | - redis-server | 37 | - redis-server |
34 | - - gemfile: test/gemfiles/mongoid6.gemfile | 38 | + - rvm: 2.6 |
39 | + gemfile: test/gemfiles/mongoid6.gemfile | ||
35 | services: | 40 | services: |
36 | - mongodb | 41 | - mongodb |
37 | - redis-server | 42 | - redis-server |
lib/searchkick.rb
1 | +# dependencies | ||
1 | require "active_support" | 2 | require "active_support" |
2 | require "active_support/core_ext/hash/deep_merge" | 3 | require "active_support/core_ext/hash/deep_merge" |
3 | require "elasticsearch" | 4 | require "elasticsearch" |
4 | require "hashie" | 5 | require "hashie" |
5 | 6 | ||
7 | +# modules | ||
6 | require "searchkick/bulk_indexer" | 8 | require "searchkick/bulk_indexer" |
7 | require "searchkick/index" | 9 | require "searchkick/index" |
8 | require "searchkick/indexer" | 10 | require "searchkick/indexer" |
@@ -17,6 +19,7 @@ require "searchkick/record_indexer" | @@ -17,6 +19,7 @@ require "searchkick/record_indexer" | ||
17 | require "searchkick/results" | 19 | require "searchkick/results" |
18 | require "searchkick/version" | 20 | require "searchkick/version" |
19 | 21 | ||
22 | +# integrations | ||
20 | require "searchkick/railtie" if defined?(Rails) | 23 | require "searchkick/railtie" if defined?(Rails) |
21 | require "searchkick/logging" if defined?(ActiveSupport::Notifications) | 24 | require "searchkick/logging" if defined?(ActiveSupport::Notifications) |
22 | 25 | ||
@@ -27,6 +30,7 @@ module Searchkick | @@ -27,6 +30,7 @@ module Searchkick | ||
27 | autoload :ProcessQueueJob, "searchkick/process_queue_job" | 30 | autoload :ProcessQueueJob, "searchkick/process_queue_job" |
28 | autoload :ReindexV2Job, "searchkick/reindex_v2_job" | 31 | autoload :ReindexV2Job, "searchkick/reindex_v2_job" |
29 | 32 | ||
33 | + # errors | ||
30 | class Error < StandardError; end | 34 | class Error < StandardError; end |
31 | class MissingIndexError < Error; end | 35 | class MissingIndexError < Error; end |
32 | class UnsupportedVersionError < Error; end | 36 | class UnsupportedVersionError < Error; end |
lib/searchkick/index_options.rb
@@ -27,9 +27,6 @@ module Searchkick | @@ -27,9 +27,6 @@ module Searchkick | ||
27 | default_analyzer = :searchkick_index | 27 | default_analyzer = :searchkick_index |
28 | keyword_mapping = {type: "keyword"} | 28 | keyword_mapping = {type: "keyword"} |
29 | 29 | ||
30 | - index_true_value = true | ||
31 | - index_false_value = false | ||
32 | - | ||
33 | keyword_mapping[:ignore_above] = options[:ignore_above] || 30000 | 30 | keyword_mapping[:ignore_above] = options[:ignore_above] || 30000 |
34 | 31 | ||
35 | settings = { | 32 | settings = { |
@@ -356,13 +353,13 @@ module Searchkick | @@ -356,13 +353,13 @@ module Searchkick | ||
356 | 353 | ||
357 | mapping_options[:searchable].delete("_all") | 354 | mapping_options[:searchable].delete("_all") |
358 | 355 | ||
359 | - analyzed_field_options = {type: default_type, index: index_true_value, analyzer: default_analyzer} | 356 | + analyzed_field_options = {type: default_type, index: true, analyzer: default_analyzer} |
360 | 357 | ||
361 | mapping_options.values.flatten.uniq.each do |field| | 358 | mapping_options.values.flatten.uniq.each do |field| |
362 | fields = {} | 359 | fields = {} |
363 | 360 | ||
364 | if options.key?(:filterable) && !mapping_options[:filterable].include?(field) | 361 | if options.key?(:filterable) && !mapping_options[:filterable].include?(field) |
365 | - fields[field] = {type: default_type, index: index_false_value} | 362 | + fields[field] = {type: default_type, index: false} |
366 | else | 363 | else |
367 | fields[field] = keyword_mapping | 364 | fields[field] = keyword_mapping |
368 | end | 365 | end |
@@ -378,7 +375,7 @@ module Searchkick | @@ -378,7 +375,7 @@ module Searchkick | ||
378 | 375 | ||
379 | mapping_options.except(:highlight, :searchable, :filterable, :word).each do |type, f| | 376 | mapping_options.except(:highlight, :searchable, :filterable, :word).each do |type, f| |
380 | if options[:match] == type || f.include?(field) | 377 | if options[:match] == type || f.include?(field) |
381 | - fields[type] = {type: default_type, index: index_true_value, analyzer: "searchkick_#{type}_index"} | 378 | + fields[type] = {type: default_type, index: true, analyzer: "searchkick_#{type}_index"} |
382 | end | 379 | end |
383 | end | 380 | end |
384 | end | 381 | end |
@@ -418,12 +415,12 @@ module Searchkick | @@ -418,12 +415,12 @@ module Searchkick | ||
418 | } | 415 | } |
419 | 416 | ||
420 | if options.key?(:filterable) | 417 | if options.key?(:filterable) |
421 | - dynamic_fields["{name}"] = {type: default_type, index: index_false_value} | 418 | + dynamic_fields["{name}"] = {type: default_type, index: false} |
422 | end | 419 | end |
423 | 420 | ||
424 | unless options[:searchable] | 421 | unless options[:searchable] |
425 | if options[:match] && options[:match] != :word | 422 | if options[:match] && options[:match] != :word |
426 | - dynamic_fields[options[:match]] = {type: default_type, index: index_true_value, analyzer: "searchkick_#{options[:match]}_index"} | 423 | + dynamic_fields[options[:match]] = {type: default_type, index: true, analyzer: "searchkick_#{options[:match]}_index"} |
427 | end | 424 | end |
428 | 425 | ||
429 | if word | 426 | if word |
test/callbacks_test.rb
@@ -1,59 +0,0 @@ | @@ -1,59 +0,0 @@ | ||
1 | -require_relative "test_helper" | ||
2 | - | ||
3 | -class CallbacksTest < Minitest::Test | ||
4 | - def test_true_create | ||
5 | - Searchkick.callbacks(true) do | ||
6 | - store_names ["Product A", "Product B"] | ||
7 | - end | ||
8 | - Product.searchkick_index.refresh | ||
9 | - assert_search "product", ["Product A", "Product B"] | ||
10 | - end | ||
11 | - | ||
12 | - def test_false_create | ||
13 | - Searchkick.callbacks(false) do | ||
14 | - store_names ["Product A", "Product B"] | ||
15 | - end | ||
16 | - Product.searchkick_index.refresh | ||
17 | - assert_search "product", [] | ||
18 | - end | ||
19 | - | ||
20 | - def test_bulk_create | ||
21 | - Searchkick.callbacks(:bulk) do | ||
22 | - store_names ["Product A", "Product B"] | ||
23 | - end | ||
24 | - Product.searchkick_index.refresh | ||
25 | - assert_search "product", ["Product A", "Product B"] | ||
26 | - end | ||
27 | - | ||
28 | - def test_queue | ||
29 | - skip unless defined?(ActiveJob) && defined?(Redis) | ||
30 | - | ||
31 | - reindex_queue = Product.searchkick_index.reindex_queue | ||
32 | - reindex_queue.clear | ||
33 | - | ||
34 | - Searchkick.callbacks(:queue) do | ||
35 | - store_names ["Product A", "Product B"] | ||
36 | - end | ||
37 | - Product.searchkick_index.refresh | ||
38 | - assert_search "product", [], load: false, conversions: false | ||
39 | - assert_equal 2, reindex_queue.length | ||
40 | - | ||
41 | - Searchkick::ProcessQueueJob.perform_later(class_name: "Product") | ||
42 | - Product.searchkick_index.refresh | ||
43 | - assert_search "product", ["Product A", "Product B"], load: false | ||
44 | - assert_equal 0, reindex_queue.length | ||
45 | - | ||
46 | - Searchkick.callbacks(:queue) do | ||
47 | - Product.where(name: "Product B").destroy_all | ||
48 | - Product.create!(name: "Product C") | ||
49 | - end | ||
50 | - Product.searchkick_index.refresh | ||
51 | - assert_search "product", ["Product A", "Product B"], load: false | ||
52 | - assert_equal 2, reindex_queue.length | ||
53 | - | ||
54 | - Searchkick::ProcessQueueJob.perform_later(class_name: "Product") | ||
55 | - Product.searchkick_index.refresh | ||
56 | - assert_search "product", ["Product A", "Product C"], load: false | ||
57 | - assert_equal 0, reindex_queue.length | ||
58 | - end | ||
59 | -end |
test/gemfiles/activerecord50.gemfile
@@ -6,3 +6,4 @@ gemspec path: "../../" | @@ -6,3 +6,4 @@ gemspec path: "../../" | ||
6 | gem "sqlite3", "~> 1.3.0" | 6 | gem "sqlite3", "~> 1.3.0" |
7 | gem "activerecord", "~> 5.0.0" | 7 | gem "activerecord", "~> 5.0.0" |
8 | gem "actionpack", "~> 5.0.0" | 8 | gem "actionpack", "~> 5.0.0" |
9 | +gem "activejob", "~> 5.0.0", require: "active_job" |
test/gemfiles/activerecord51.gemfile
@@ -6,3 +6,4 @@ gemspec path: "../../" | @@ -6,3 +6,4 @@ gemspec path: "../../" | ||
6 | gem "sqlite3" | 6 | gem "sqlite3" |
7 | gem "activerecord", "~> 5.1.0" | 7 | gem "activerecord", "~> 5.1.0" |
8 | gem "actionpack", "~> 5.1.0" | 8 | gem "actionpack", "~> 5.1.0" |
9 | +gem "activejob", "~> 5.1.0", require: "active_job" |
test/gemfiles/activerecord52.gemfile
@@ -6,3 +6,4 @@ gemspec path: "../../" | @@ -6,3 +6,4 @@ gemspec path: "../../" | ||
6 | gem "sqlite3" | 6 | gem "sqlite3" |
7 | gem "activerecord", "~> 5.2.0" | 7 | gem "activerecord", "~> 5.2.0" |
8 | gem "actionpack", "~> 5.2.0" | 8 | gem "actionpack", "~> 5.2.0" |
9 | +gem "activejob", "~> 5.2.0", require: "active_job" |
test/reindex_test.rb
1 | require_relative "test_helper" | 1 | require_relative "test_helper" |
2 | 2 | ||
3 | class ReindexTest < Minitest::Test | 3 | class ReindexTest < Minitest::Test |
4 | - def setup | ||
5 | - super | ||
6 | - Sku.destroy_all | 4 | + def test_record_inline |
5 | + store_names ["Product A", "Product B"], reindex: false | ||
6 | + | ||
7 | + product = Product.find_by!(name: "Product A") | ||
8 | + product.reindex(refresh: true) | ||
9 | + assert_search "product", ["Product A"] | ||
7 | end | 10 | end |
8 | 11 | ||
9 | - def test_scoped | 12 | + def test_record_async |
13 | + store_names ["Product A", "Product B"], reindex: false | ||
14 | + | ||
15 | + product = Product.find_by!(name: "Product A") | ||
16 | + product.reindex(mode: :async) | ||
17 | + Product.search_index.refresh | ||
18 | + assert_search "product", ["Product A"] | ||
19 | + end | ||
20 | + | ||
21 | + def test_record_queue | ||
22 | + skip unless defined?(ActiveJob) && defined?(Redis) | ||
23 | + | ||
24 | + reindex_queue = Product.searchkick_index.reindex_queue | ||
25 | + reindex_queue.clear | ||
26 | + | ||
27 | + store_names ["Product A", "Product B"], reindex: false | ||
28 | + | ||
29 | + product = Product.find_by!(name: "Product A") | ||
30 | + product.reindex(mode: :queue) | ||
31 | + Product.search_index.refresh | ||
32 | + assert_search "product", [] | ||
33 | + | ||
34 | + Searchkick::ProcessQueueJob.perform_now(class_name: "Product") | ||
35 | + Product.search_index.refresh | ||
36 | + assert_search "product", ["Product A"] | ||
37 | + end | ||
38 | + | ||
39 | + def test_relation_inline | ||
10 | skip if nobrainer? || cequel? | 40 | skip if nobrainer? || cequel? |
11 | 41 | ||
12 | store_names ["Product A"] | 42 | store_names ["Product A"] |
13 | - Searchkick.callbacks(false) do | ||
14 | - store_names ["Product B", "Product C"] | ||
15 | - end | 43 | + store_names ["Product B", "Product C"], reindex: false |
16 | Product.where(name: "Product B").reindex(refresh: true) | 44 | Product.where(name: "Product B").reindex(refresh: true) |
17 | assert_search "product", ["Product A", "Product B"] | 45 | assert_search "product", ["Product A", "Product B"] |
18 | end | 46 | end |
19 | 47 | ||
20 | - def test_associations | 48 | + def test_relation_associations |
21 | skip if nobrainer? || cequel? | 49 | skip if nobrainer? || cequel? |
22 | 50 | ||
23 | store_names ["Product A"] | 51 | store_names ["Product A"] |
@@ -27,12 +55,18 @@ class ReindexTest < Minitest::Test | @@ -27,12 +55,18 @@ class ReindexTest < Minitest::Test | ||
27 | assert_search "product", ["Product A", "Product B"] | 55 | assert_search "product", ["Product A", "Product B"] |
28 | end | 56 | end |
29 | 57 | ||
30 | - def test_async | 58 | + def test_relation_async |
59 | + skip "Not available yet" | ||
60 | + end | ||
61 | + | ||
62 | + def test_relation_queue | ||
63 | + skip "Not available yet" | ||
64 | + end | ||
65 | + | ||
66 | + def test_full_async | ||
31 | skip unless defined?(ActiveJob) | 67 | skip unless defined?(ActiveJob) |
32 | 68 | ||
33 | - Searchkick.callbacks(false) do | ||
34 | - store_names ["Product A"] | ||
35 | - end | 69 | + store_names ["Product A"], reindex: false |
36 | reindex = Product.reindex(async: true) | 70 | reindex = Product.reindex(async: true) |
37 | assert_search "product", [], conversions: false | 71 | assert_search "product", [], conversions: false |
38 | 72 | ||
@@ -48,12 +82,10 @@ class ReindexTest < Minitest::Test | @@ -48,12 +82,10 @@ class ReindexTest < Minitest::Test | ||
48 | assert_search "product", ["Product A"] | 82 | assert_search "product", ["Product A"] |
49 | end | 83 | end |
50 | 84 | ||
51 | - def test_async_wait | 85 | + def test_full_async_wait |
52 | skip unless defined?(ActiveJob) && defined?(Redis) | 86 | skip unless defined?(ActiveJob) && defined?(Redis) |
53 | 87 | ||
54 | - Searchkick.callbacks(false) do | ||
55 | - store_names ["Product A"] | ||
56 | - end | 88 | + store_names ["Product A"], reindex: false |
57 | 89 | ||
58 | capture_io do | 90 | capture_io do |
59 | Product.reindex(async: {wait: true}) | 91 | Product.reindex(async: {wait: true}) |
@@ -62,7 +94,7 @@ class ReindexTest < Minitest::Test | @@ -62,7 +94,7 @@ class ReindexTest < Minitest::Test | ||
62 | assert_search "product", ["Product A"] | 94 | assert_search "product", ["Product A"] |
63 | end | 95 | end |
64 | 96 | ||
65 | - def test_async_non_integer_pk | 97 | + def test_full_async_non_integer_pk |
66 | skip unless defined?(ActiveJob) | 98 | skip unless defined?(ActiveJob) |
67 | 99 | ||
68 | Sku.create(id: SecureRandom.hex, name: "Test") | 100 | Sku.create(id: SecureRandom.hex, name: "Test") |
@@ -72,9 +104,11 @@ class ReindexTest < Minitest::Test | @@ -72,9 +104,11 @@ class ReindexTest < Minitest::Test | ||
72 | index = Searchkick::Index.new(reindex[:index_name]) | 104 | index = Searchkick::Index.new(reindex[:index_name]) |
73 | index.refresh | 105 | index.refresh |
74 | assert_equal 1, index.total_docs | 106 | assert_equal 1, index.total_docs |
107 | + ensure | ||
108 | + Sku.destroy_all | ||
75 | end | 109 | end |
76 | 110 | ||
77 | - def test_refresh_interval | 111 | + def test_full_refresh_interval |
78 | reindex = Product.reindex(refresh_interval: "30s", async: true, import: false) | 112 | reindex = Product.reindex(refresh_interval: "30s", async: true, import: false) |
79 | index = Searchkick::Index.new(reindex[:index_name]) | 113 | index = Searchkick::Index.new(reindex[:index_name]) |
80 | assert_nil Product.search_index.refresh_interval | 114 | assert_nil Product.search_index.refresh_interval |
@@ -85,17 +119,64 @@ class ReindexTest < Minitest::Test | @@ -85,17 +119,64 @@ class ReindexTest < Minitest::Test | ||
85 | assert_equal "1s", Product.search_index.refresh_interval | 119 | assert_equal "1s", Product.search_index.refresh_interval |
86 | end | 120 | end |
87 | 121 | ||
88 | - def test_resume | 122 | + def test_full_resume |
89 | assert Product.reindex(resume: true) | 123 | assert Product.reindex(resume: true) |
90 | end | 124 | end |
91 | 125 | ||
92 | - def test_refresh_full_reindex | 126 | + def test_full_refresh |
93 | Product.reindex(refresh: true) | 127 | Product.reindex(refresh: true) |
94 | end | 128 | end |
95 | 129 | ||
96 | - def test_partial_async | 130 | + def test_full_partial_async |
97 | store_names ["Product A"] | 131 | store_names ["Product A"] |
98 | # warn for now | 132 | # warn for now |
99 | Product.reindex(:search_name, async: true) | 133 | Product.reindex(:search_name, async: true) |
100 | end | 134 | end |
135 | + | ||
136 | + def test_callbacks_false | ||
137 | + Searchkick.callbacks(false) do | ||
138 | + store_names ["Product A", "Product B"] | ||
139 | + end | ||
140 | + assert_search "product", [] | ||
141 | + end | ||
142 | + | ||
143 | + def test_callbacks_bulk | ||
144 | + Searchkick.callbacks(:bulk) do | ||
145 | + store_names ["Product A", "Product B"] | ||
146 | + end | ||
147 | + Product.search_index.refresh | ||
148 | + assert_search "product", ["Product A", "Product B"] | ||
149 | + end | ||
150 | + | ||
151 | + def test_callbacks_queue | ||
152 | + skip unless defined?(ActiveJob) && defined?(Redis) | ||
153 | + | ||
154 | + reindex_queue = Product.searchkick_index.reindex_queue | ||
155 | + reindex_queue.clear | ||
156 | + | ||
157 | + Searchkick.callbacks(:queue) do | ||
158 | + store_names ["Product A", "Product B"] | ||
159 | + end | ||
160 | + Product.searchkick_index.refresh | ||
161 | + assert_search "product", [], load: false, conversions: false | ||
162 | + assert_equal 2, reindex_queue.length | ||
163 | + | ||
164 | + Searchkick::ProcessQueueJob.perform_later(class_name: "Product") | ||
165 | + Product.searchkick_index.refresh | ||
166 | + assert_search "product", ["Product A", "Product B"], load: false | ||
167 | + assert_equal 0, reindex_queue.length | ||
168 | + | ||
169 | + Searchkick.callbacks(:queue) do | ||
170 | + Product.where(name: "Product B").destroy_all | ||
171 | + Product.create!(name: "Product C") | ||
172 | + end | ||
173 | + Product.searchkick_index.refresh | ||
174 | + assert_search "product", ["Product A", "Product B"], load: false | ||
175 | + assert_equal 2, reindex_queue.length | ||
176 | + | ||
177 | + Searchkick::ProcessQueueJob.perform_later(class_name: "Product") | ||
178 | + Product.searchkick_index.refresh | ||
179 | + assert_search "product", ["Product A", "Product C"], load: false | ||
180 | + assert_equal 0, reindex_queue.length | ||
181 | + end | ||
101 | end | 182 | end |
test/test_helper.rb
@@ -71,15 +71,23 @@ class Minitest::Test | @@ -71,15 +71,23 @@ class Minitest::Test | ||
71 | 71 | ||
72 | protected | 72 | protected |
73 | 73 | ||
74 | - def store(documents, klass = Product) | ||
75 | - documents.shuffle.each do |document| | ||
76 | - klass.create!(document) | 74 | + def store(documents, klass = Product, reindex: true) |
75 | + if reindex | ||
76 | + documents.shuffle.each do |document| | ||
77 | + klass.create!(document) | ||
78 | + end | ||
79 | + klass.searchkick_index.refresh | ||
80 | + else | ||
81 | + Searchkick.callbacks(false) do | ||
82 | + documents.shuffle.each do |document| | ||
83 | + klass.create!(document) | ||
84 | + end | ||
85 | + end | ||
77 | end | 86 | end |
78 | - klass.searchkick_index.refresh | ||
79 | end | 87 | end |
80 | 88 | ||
81 | - def store_names(names, klass = Product) | ||
82 | - store names.map { |name| {name: name} }, klass | 89 | + def store_names(names, klass = Product, reindex: true) |
90 | + store names.map { |name| {name: name} }, klass, reindex: reindex | ||
83 | end | 91 | end |
84 | 92 | ||
85 | # no order | 93 | # no order |