Commit e5550dd2a48caf1391af51d9e8c184dddbeffaa2

Authored by Andrew Kane
2 parents 0e3ddf8b ca27d8e4
Exists in reindex_refactor

Merge branch 'master' into reindex_refactor

.travis.yml
1 1 dist: bionic
2 2 language: ruby
3   -rvm: 2.6
4 3 gemfile:
5 4 - Gemfile
6 5 services:
... ... @@ -18,20 +17,26 @@ cache:
18 17 env:
19 18 - ELASTICSEARCH_VERSION=7.6.1
20 19 jdk: openjdk10
21   -matrix:
  20 +jobs:
22 21 include:
23   - - gemfile: Gemfile
24   - - gemfile: test/gemfiles/activerecord52.gemfile
  22 + - rvm: 2.7
  23 + gemfile: Gemfile
  24 + - rvm: 2.6
  25 + gemfile: test/gemfiles/activerecord52.gemfile
25 26 env: ELASTICSEARCH_VERSION=7.0.0
26   - - gemfile: test/gemfiles/activerecord51.gemfile
  27 + - rvm: 2.5
  28 + gemfile: test/gemfiles/activerecord51.gemfile
27 29 env: ELASTICSEARCH_VERSION=6.8.7
28   - - gemfile: test/gemfiles/activerecord50.gemfile
  30 + - rvm: 2.4
  31 + gemfile: test/gemfiles/activerecord50.gemfile
29 32 env: ELASTICSEARCH_VERSION=6.0.0
30   - - gemfile: test/gemfiles/mongoid7.gemfile
  33 + - rvm: 2.7
  34 + gemfile: test/gemfiles/mongoid7.gemfile
31 35 services:
32 36 - mongodb
33 37 - redis-server
34   - - gemfile: test/gemfiles/mongoid6.gemfile
  38 + - rvm: 2.6
  39 + gemfile: test/gemfiles/mongoid6.gemfile
35 40 services:
36 41 - mongodb
37 42 - redis-server
... ...
lib/searchkick.rb
  1 +# dependencies
1 2 require "active_support"
2 3 require "active_support/core_ext/hash/deep_merge"
3 4 require "elasticsearch"
4 5 require "hashie"
5 6  
  7 +# modules
6 8 require "searchkick/bulk_indexer"
7 9 require "searchkick/index"
8 10 require "searchkick/indexer"
... ... @@ -17,6 +19,7 @@ require "searchkick/record_indexer"
17 19 require "searchkick/results"
18 20 require "searchkick/version"
19 21  
  22 +# integrations
20 23 require "searchkick/railtie" if defined?(Rails)
21 24 require "searchkick/logging" if defined?(ActiveSupport::Notifications)
22 25  
... ... @@ -27,6 +30,7 @@ module Searchkick
27 30 autoload :ProcessQueueJob, "searchkick/process_queue_job"
28 31 autoload :ReindexV2Job, "searchkick/reindex_v2_job"
29 32  
  33 + # errors
30 34 class Error < StandardError; end
31 35 class MissingIndexError < Error; end
32 36 class UnsupportedVersionError < Error; end
... ...
lib/searchkick/index_options.rb
... ... @@ -27,9 +27,6 @@ module Searchkick
27 27 default_analyzer = :searchkick_index
28 28 keyword_mapping = {type: "keyword"}
29 29  
30   - index_true_value = true
31   - index_false_value = false
32   -
33 30 keyword_mapping[:ignore_above] = options[:ignore_above] || 30000
34 31  
35 32 settings = {
... ... @@ -356,13 +353,13 @@ module Searchkick
356 353  
357 354 mapping_options[:searchable].delete("_all")
358 355  
359   - analyzed_field_options = {type: default_type, index: index_true_value, analyzer: default_analyzer}
  356 + analyzed_field_options = {type: default_type, index: true, analyzer: default_analyzer}
360 357  
361 358 mapping_options.values.flatten.uniq.each do |field|
362 359 fields = {}
363 360  
364 361 if options.key?(:filterable) && !mapping_options[:filterable].include?(field)
365   - fields[field] = {type: default_type, index: index_false_value}
  362 + fields[field] = {type: default_type, index: false}
366 363 else
367 364 fields[field] = keyword_mapping
368 365 end
... ... @@ -378,7 +375,7 @@ module Searchkick
378 375  
379 376 mapping_options.except(:highlight, :searchable, :filterable, :word).each do |type, f|
380 377 if options[:match] == type || f.include?(field)
381   - fields[type] = {type: default_type, index: index_true_value, analyzer: "searchkick_#{type}_index"}
  378 + fields[type] = {type: default_type, index: true, analyzer: "searchkick_#{type}_index"}
382 379 end
383 380 end
384 381 end
... ... @@ -418,12 +415,12 @@ module Searchkick
418 415 }
419 416  
420 417 if options.key?(:filterable)
421   - dynamic_fields["{name}"] = {type: default_type, index: index_false_value}
  418 + dynamic_fields["{name}"] = {type: default_type, index: false}
422 419 end
423 420  
424 421 unless options[:searchable]
425 422 if options[:match] && options[:match] != :word
426   - dynamic_fields[options[:match]] = {type: default_type, index: index_true_value, analyzer: "searchkick_#{options[:match]}_index"}
  423 + dynamic_fields[options[:match]] = {type: default_type, index: true, analyzer: "searchkick_#{options[:match]}_index"}
427 424 end
428 425  
429 426 if word
... ...
test/callbacks_test.rb
... ... @@ -1,59 +0,0 @@
1   -require_relative "test_helper"
2   -
3   -class CallbacksTest < Minitest::Test
4   - def test_true_create
5   - Searchkick.callbacks(true) do
6   - store_names ["Product A", "Product B"]
7   - end
8   - Product.searchkick_index.refresh
9   - assert_search "product", ["Product A", "Product B"]
10   - end
11   -
12   - def test_false_create
13   - Searchkick.callbacks(false) do
14   - store_names ["Product A", "Product B"]
15   - end
16   - Product.searchkick_index.refresh
17   - assert_search "product", []
18   - end
19   -
20   - def test_bulk_create
21   - Searchkick.callbacks(:bulk) do
22   - store_names ["Product A", "Product B"]
23   - end
24   - Product.searchkick_index.refresh
25   - assert_search "product", ["Product A", "Product B"]
26   - end
27   -
28   - def test_queue
29   - skip unless defined?(ActiveJob) && defined?(Redis)
30   -
31   - reindex_queue = Product.searchkick_index.reindex_queue
32   - reindex_queue.clear
33   -
34   - Searchkick.callbacks(:queue) do
35   - store_names ["Product A", "Product B"]
36   - end
37   - Product.searchkick_index.refresh
38   - assert_search "product", [], load: false, conversions: false
39   - assert_equal 2, reindex_queue.length
40   -
41   - Searchkick::ProcessQueueJob.perform_later(class_name: "Product")
42   - Product.searchkick_index.refresh
43   - assert_search "product", ["Product A", "Product B"], load: false
44   - assert_equal 0, reindex_queue.length
45   -
46   - Searchkick.callbacks(:queue) do
47   - Product.where(name: "Product B").destroy_all
48   - Product.create!(name: "Product C")
49   - end
50   - Product.searchkick_index.refresh
51   - assert_search "product", ["Product A", "Product B"], load: false
52   - assert_equal 2, reindex_queue.length
53   -
54   - Searchkick::ProcessQueueJob.perform_later(class_name: "Product")
55   - Product.searchkick_index.refresh
56   - assert_search "product", ["Product A", "Product C"], load: false
57   - assert_equal 0, reindex_queue.length
58   - end
59   -end
test/gemfiles/activerecord50.gemfile
... ... @@ -6,3 +6,4 @@ gemspec path: &quot;../../&quot;
6 6 gem "sqlite3", "~> 1.3.0"
7 7 gem "activerecord", "~> 5.0.0"
8 8 gem "actionpack", "~> 5.0.0"
  9 +gem "activejob", "~> 5.0.0", require: "active_job"
... ...
test/gemfiles/activerecord51.gemfile
... ... @@ -6,3 +6,4 @@ gemspec path: &quot;../../&quot;
6 6 gem "sqlite3"
7 7 gem "activerecord", "~> 5.1.0"
8 8 gem "actionpack", "~> 5.1.0"
  9 +gem "activejob", "~> 5.1.0", require: "active_job"
... ...
test/gemfiles/activerecord52.gemfile
... ... @@ -6,3 +6,4 @@ gemspec path: &quot;../../&quot;
6 6 gem "sqlite3"
7 7 gem "activerecord", "~> 5.2.0"
8 8 gem "actionpack", "~> 5.2.0"
  9 +gem "activejob", "~> 5.2.0", require: "active_job"
... ...
test/reindex_test.rb
1 1 require_relative "test_helper"
2 2  
3 3 class ReindexTest < Minitest::Test
4   - def setup
5   - super
6   - Sku.destroy_all
  4 + def test_record_inline
  5 + store_names ["Product A", "Product B"], reindex: false
  6 +
  7 + product = Product.find_by!(name: "Product A")
  8 + product.reindex(refresh: true)
  9 + assert_search "product", ["Product A"]
7 10 end
8 11  
9   - def test_scoped
  12 + def test_record_async
  13 + store_names ["Product A", "Product B"], reindex: false
  14 +
  15 + product = Product.find_by!(name: "Product A")
  16 + product.reindex(mode: :async)
  17 + Product.search_index.refresh
  18 + assert_search "product", ["Product A"]
  19 + end
  20 +
  21 + def test_record_queue
  22 + skip unless defined?(ActiveJob) && defined?(Redis)
  23 +
  24 + reindex_queue = Product.searchkick_index.reindex_queue
  25 + reindex_queue.clear
  26 +
  27 + store_names ["Product A", "Product B"], reindex: false
  28 +
  29 + product = Product.find_by!(name: "Product A")
  30 + product.reindex(mode: :queue)
  31 + Product.search_index.refresh
  32 + assert_search "product", []
  33 +
  34 + Searchkick::ProcessQueueJob.perform_now(class_name: "Product")
  35 + Product.search_index.refresh
  36 + assert_search "product", ["Product A"]
  37 + end
  38 +
  39 + def test_relation_inline
10 40 skip if nobrainer? || cequel?
11 41  
12 42 store_names ["Product A"]
13   - Searchkick.callbacks(false) do
14   - store_names ["Product B", "Product C"]
15   - end
  43 + store_names ["Product B", "Product C"], reindex: false
16 44 Product.where(name: "Product B").reindex(refresh: true)
17 45 assert_search "product", ["Product A", "Product B"]
18 46 end
19 47  
20   - def test_associations
  48 + def test_relation_associations
21 49 skip if nobrainer? || cequel?
22 50  
23 51 store_names ["Product A"]
... ... @@ -27,12 +55,18 @@ class ReindexTest &lt; Minitest::Test
27 55 assert_search "product", ["Product A", "Product B"]
28 56 end
29 57  
30   - def test_async
  58 + def test_relation_async
  59 + skip "Not available yet"
  60 + end
  61 +
  62 + def test_relation_queue
  63 + skip "Not available yet"
  64 + end
  65 +
  66 + def test_full_async
31 67 skip unless defined?(ActiveJob)
32 68  
33   - Searchkick.callbacks(false) do
34   - store_names ["Product A"]
35   - end
  69 + store_names ["Product A"], reindex: false
36 70 reindex = Product.reindex(async: true)
37 71 assert_search "product", [], conversions: false
38 72  
... ... @@ -48,12 +82,10 @@ class ReindexTest &lt; Minitest::Test
48 82 assert_search "product", ["Product A"]
49 83 end
50 84  
51   - def test_async_wait
  85 + def test_full_async_wait
52 86 skip unless defined?(ActiveJob) && defined?(Redis)
53 87  
54   - Searchkick.callbacks(false) do
55   - store_names ["Product A"]
56   - end
  88 + store_names ["Product A"], reindex: false
57 89  
58 90 capture_io do
59 91 Product.reindex(async: {wait: true})
... ... @@ -62,7 +94,7 @@ class ReindexTest &lt; Minitest::Test
62 94 assert_search "product", ["Product A"]
63 95 end
64 96  
65   - def test_async_non_integer_pk
  97 + def test_full_async_non_integer_pk
66 98 skip unless defined?(ActiveJob)
67 99  
68 100 Sku.create(id: SecureRandom.hex, name: "Test")
... ... @@ -72,9 +104,11 @@ class ReindexTest &lt; Minitest::Test
72 104 index = Searchkick::Index.new(reindex[:index_name])
73 105 index.refresh
74 106 assert_equal 1, index.total_docs
  107 + ensure
  108 + Sku.destroy_all
75 109 end
76 110  
77   - def test_refresh_interval
  111 + def test_full_refresh_interval
78 112 reindex = Product.reindex(refresh_interval: "30s", async: true, import: false)
79 113 index = Searchkick::Index.new(reindex[:index_name])
80 114 assert_nil Product.search_index.refresh_interval
... ... @@ -85,17 +119,64 @@ class ReindexTest &lt; Minitest::Test
85 119 assert_equal "1s", Product.search_index.refresh_interval
86 120 end
87 121  
88   - def test_resume
  122 + def test_full_resume
89 123 assert Product.reindex(resume: true)
90 124 end
91 125  
92   - def test_refresh_full_reindex
  126 + def test_full_refresh
93 127 Product.reindex(refresh: true)
94 128 end
95 129  
96   - def test_partial_async
  130 + def test_full_partial_async
97 131 store_names ["Product A"]
98 132 # warn for now
99 133 Product.reindex(:search_name, async: true)
100 134 end
  135 +
  136 + def test_callbacks_false
  137 + Searchkick.callbacks(false) do
  138 + store_names ["Product A", "Product B"]
  139 + end
  140 + assert_search "product", []
  141 + end
  142 +
  143 + def test_callbacks_bulk
  144 + Searchkick.callbacks(:bulk) do
  145 + store_names ["Product A", "Product B"]
  146 + end
  147 + Product.search_index.refresh
  148 + assert_search "product", ["Product A", "Product B"]
  149 + end
  150 +
  151 + def test_callbacks_queue
  152 + skip unless defined?(ActiveJob) && defined?(Redis)
  153 +
  154 + reindex_queue = Product.searchkick_index.reindex_queue
  155 + reindex_queue.clear
  156 +
  157 + Searchkick.callbacks(:queue) do
  158 + store_names ["Product A", "Product B"]
  159 + end
  160 + Product.searchkick_index.refresh
  161 + assert_search "product", [], load: false, conversions: false
  162 + assert_equal 2, reindex_queue.length
  163 +
  164 + Searchkick::ProcessQueueJob.perform_later(class_name: "Product")
  165 + Product.searchkick_index.refresh
  166 + assert_search "product", ["Product A", "Product B"], load: false
  167 + assert_equal 0, reindex_queue.length
  168 +
  169 + Searchkick.callbacks(:queue) do
  170 + Product.where(name: "Product B").destroy_all
  171 + Product.create!(name: "Product C")
  172 + end
  173 + Product.searchkick_index.refresh
  174 + assert_search "product", ["Product A", "Product B"], load: false
  175 + assert_equal 2, reindex_queue.length
  176 +
  177 + Searchkick::ProcessQueueJob.perform_later(class_name: "Product")
  178 + Product.searchkick_index.refresh
  179 + assert_search "product", ["Product A", "Product C"], load: false
  180 + assert_equal 0, reindex_queue.length
  181 + end
101 182 end
... ...
test/test_helper.rb
... ... @@ -71,15 +71,23 @@ class Minitest::Test
71 71  
72 72 protected
73 73  
74   - def store(documents, klass = Product)
75   - documents.shuffle.each do |document|
76   - klass.create!(document)
  74 + def store(documents, klass = Product, reindex: true)
  75 + if reindex
  76 + documents.shuffle.each do |document|
  77 + klass.create!(document)
  78 + end
  79 + klass.searchkick_index.refresh
  80 + else
  81 + Searchkick.callbacks(false) do
  82 + documents.shuffle.each do |document|
  83 + klass.create!(document)
  84 + end
  85 + end
77 86 end
78   - klass.searchkick_index.refresh
79 87 end
80 88  
81   - def store_names(names, klass = Product)
82   - store names.map { |name| {name: name} }, klass
  89 + def store_names(names, klass = Product, reindex: true)
  90 + store names.map { |name| {name: name} }, klass, reindex: reindex
83 91 end
84 92  
85 93 # no order
... ...