Commit e5550dd2a48caf1391af51d9e8c184dddbeffaa2

Authored by Andrew Kane
2 parents 0e3ddf8b ca27d8e4
Exists in reindex_refactor

Merge branch 'master' into reindex_refactor

1 dist: bionic 1 dist: bionic
2 language: ruby 2 language: ruby
3 -rvm: 2.6  
4 gemfile: 3 gemfile:
5 - Gemfile 4 - Gemfile
6 services: 5 services:
@@ -18,20 +17,26 @@ cache: @@ -18,20 +17,26 @@ cache:
18 env: 17 env:
19 - ELASTICSEARCH_VERSION=7.6.1 18 - ELASTICSEARCH_VERSION=7.6.1
20 jdk: openjdk10 19 jdk: openjdk10
21 -matrix: 20 +jobs:
22 include: 21 include:
23 - - gemfile: Gemfile  
24 - - gemfile: test/gemfiles/activerecord52.gemfile 22 + - rvm: 2.7
  23 + gemfile: Gemfile
  24 + - rvm: 2.6
  25 + gemfile: test/gemfiles/activerecord52.gemfile
25 env: ELASTICSEARCH_VERSION=7.0.0 26 env: ELASTICSEARCH_VERSION=7.0.0
26 - - gemfile: test/gemfiles/activerecord51.gemfile 27 + - rvm: 2.5
  28 + gemfile: test/gemfiles/activerecord51.gemfile
27 env: ELASTICSEARCH_VERSION=6.8.7 29 env: ELASTICSEARCH_VERSION=6.8.7
28 - - gemfile: test/gemfiles/activerecord50.gemfile 30 + - rvm: 2.4
  31 + gemfile: test/gemfiles/activerecord50.gemfile
29 env: ELASTICSEARCH_VERSION=6.0.0 32 env: ELASTICSEARCH_VERSION=6.0.0
30 - - gemfile: test/gemfiles/mongoid7.gemfile 33 + - rvm: 2.7
  34 + gemfile: test/gemfiles/mongoid7.gemfile
31 services: 35 services:
32 - mongodb 36 - mongodb
33 - redis-server 37 - redis-server
34 - - gemfile: test/gemfiles/mongoid6.gemfile 38 + - rvm: 2.6
  39 + gemfile: test/gemfiles/mongoid6.gemfile
35 services: 40 services:
36 - mongodb 41 - mongodb
37 - redis-server 42 - redis-server
lib/searchkick.rb
  1 +# dependencies
1 require "active_support" 2 require "active_support"
2 require "active_support/core_ext/hash/deep_merge" 3 require "active_support/core_ext/hash/deep_merge"
3 require "elasticsearch" 4 require "elasticsearch"
4 require "hashie" 5 require "hashie"
5 6
  7 +# modules
6 require "searchkick/bulk_indexer" 8 require "searchkick/bulk_indexer"
7 require "searchkick/index" 9 require "searchkick/index"
8 require "searchkick/indexer" 10 require "searchkick/indexer"
@@ -17,6 +19,7 @@ require "searchkick/record_indexer" @@ -17,6 +19,7 @@ require "searchkick/record_indexer"
17 require "searchkick/results" 19 require "searchkick/results"
18 require "searchkick/version" 20 require "searchkick/version"
19 21
  22 +# integrations
20 require "searchkick/railtie" if defined?(Rails) 23 require "searchkick/railtie" if defined?(Rails)
21 require "searchkick/logging" if defined?(ActiveSupport::Notifications) 24 require "searchkick/logging" if defined?(ActiveSupport::Notifications)
22 25
@@ -27,6 +30,7 @@ module Searchkick @@ -27,6 +30,7 @@ module Searchkick
27 autoload :ProcessQueueJob, "searchkick/process_queue_job" 30 autoload :ProcessQueueJob, "searchkick/process_queue_job"
28 autoload :ReindexV2Job, "searchkick/reindex_v2_job" 31 autoload :ReindexV2Job, "searchkick/reindex_v2_job"
29 32
  33 + # errors
30 class Error < StandardError; end 34 class Error < StandardError; end
31 class MissingIndexError < Error; end 35 class MissingIndexError < Error; end
32 class UnsupportedVersionError < Error; end 36 class UnsupportedVersionError < Error; end
lib/searchkick/index_options.rb
@@ -27,9 +27,6 @@ module Searchkick @@ -27,9 +27,6 @@ module Searchkick
27 default_analyzer = :searchkick_index 27 default_analyzer = :searchkick_index
28 keyword_mapping = {type: "keyword"} 28 keyword_mapping = {type: "keyword"}
29 29
30 - index_true_value = true  
31 - index_false_value = false  
32 -  
33 keyword_mapping[:ignore_above] = options[:ignore_above] || 30000 30 keyword_mapping[:ignore_above] = options[:ignore_above] || 30000
34 31
35 settings = { 32 settings = {
@@ -356,13 +353,13 @@ module Searchkick @@ -356,13 +353,13 @@ module Searchkick
356 353
357 mapping_options[:searchable].delete("_all") 354 mapping_options[:searchable].delete("_all")
358 355
359 - analyzed_field_options = {type: default_type, index: index_true_value, analyzer: default_analyzer} 356 + analyzed_field_options = {type: default_type, index: true, analyzer: default_analyzer}
360 357
361 mapping_options.values.flatten.uniq.each do |field| 358 mapping_options.values.flatten.uniq.each do |field|
362 fields = {} 359 fields = {}
363 360
364 if options.key?(:filterable) && !mapping_options[:filterable].include?(field) 361 if options.key?(:filterable) && !mapping_options[:filterable].include?(field)
365 - fields[field] = {type: default_type, index: index_false_value} 362 + fields[field] = {type: default_type, index: false}
366 else 363 else
367 fields[field] = keyword_mapping 364 fields[field] = keyword_mapping
368 end 365 end
@@ -378,7 +375,7 @@ module Searchkick @@ -378,7 +375,7 @@ module Searchkick
378 375
379 mapping_options.except(:highlight, :searchable, :filterable, :word).each do |type, f| 376 mapping_options.except(:highlight, :searchable, :filterable, :word).each do |type, f|
380 if options[:match] == type || f.include?(field) 377 if options[:match] == type || f.include?(field)
381 - fields[type] = {type: default_type, index: index_true_value, analyzer: "searchkick_#{type}_index"} 378 + fields[type] = {type: default_type, index: true, analyzer: "searchkick_#{type}_index"}
382 end 379 end
383 end 380 end
384 end 381 end
@@ -418,12 +415,12 @@ module Searchkick @@ -418,12 +415,12 @@ module Searchkick
418 } 415 }
419 416
420 if options.key?(:filterable) 417 if options.key?(:filterable)
421 - dynamic_fields["{name}"] = {type: default_type, index: index_false_value} 418 + dynamic_fields["{name}"] = {type: default_type, index: false}
422 end 419 end
423 420
424 unless options[:searchable] 421 unless options[:searchable]
425 if options[:match] && options[:match] != :word 422 if options[:match] && options[:match] != :word
426 - dynamic_fields[options[:match]] = {type: default_type, index: index_true_value, analyzer: "searchkick_#{options[:match]}_index"} 423 + dynamic_fields[options[:match]] = {type: default_type, index: true, analyzer: "searchkick_#{options[:match]}_index"}
427 end 424 end
428 425
429 if word 426 if word
test/callbacks_test.rb
@@ -1,59 +0,0 @@ @@ -1,59 +0,0 @@
1 -require_relative "test_helper"  
2 -  
3 -class CallbacksTest < Minitest::Test  
4 - def test_true_create  
5 - Searchkick.callbacks(true) do  
6 - store_names ["Product A", "Product B"]  
7 - end  
8 - Product.searchkick_index.refresh  
9 - assert_search "product", ["Product A", "Product B"]  
10 - end  
11 -  
12 - def test_false_create  
13 - Searchkick.callbacks(false) do  
14 - store_names ["Product A", "Product B"]  
15 - end  
16 - Product.searchkick_index.refresh  
17 - assert_search "product", []  
18 - end  
19 -  
20 - def test_bulk_create  
21 - Searchkick.callbacks(:bulk) do  
22 - store_names ["Product A", "Product B"]  
23 - end  
24 - Product.searchkick_index.refresh  
25 - assert_search "product", ["Product A", "Product B"]  
26 - end  
27 -  
28 - def test_queue  
29 - skip unless defined?(ActiveJob) && defined?(Redis)  
30 -  
31 - reindex_queue = Product.searchkick_index.reindex_queue  
32 - reindex_queue.clear  
33 -  
34 - Searchkick.callbacks(:queue) do  
35 - store_names ["Product A", "Product B"]  
36 - end  
37 - Product.searchkick_index.refresh  
38 - assert_search "product", [], load: false, conversions: false  
39 - assert_equal 2, reindex_queue.length  
40 -  
41 - Searchkick::ProcessQueueJob.perform_later(class_name: "Product")  
42 - Product.searchkick_index.refresh  
43 - assert_search "product", ["Product A", "Product B"], load: false  
44 - assert_equal 0, reindex_queue.length  
45 -  
46 - Searchkick.callbacks(:queue) do  
47 - Product.where(name: "Product B").destroy_all  
48 - Product.create!(name: "Product C")  
49 - end  
50 - Product.searchkick_index.refresh  
51 - assert_search "product", ["Product A", "Product B"], load: false  
52 - assert_equal 2, reindex_queue.length  
53 -  
54 - Searchkick::ProcessQueueJob.perform_later(class_name: "Product")  
55 - Product.searchkick_index.refresh  
56 - assert_search "product", ["Product A", "Product C"], load: false  
57 - assert_equal 0, reindex_queue.length  
58 - end  
59 -end  
test/gemfiles/activerecord50.gemfile
@@ -6,3 +6,4 @@ gemspec path: &quot;../../&quot; @@ -6,3 +6,4 @@ gemspec path: &quot;../../&quot;
6 gem "sqlite3", "~> 1.3.0" 6 gem "sqlite3", "~> 1.3.0"
7 gem "activerecord", "~> 5.0.0" 7 gem "activerecord", "~> 5.0.0"
8 gem "actionpack", "~> 5.0.0" 8 gem "actionpack", "~> 5.0.0"
  9 +gem "activejob", "~> 5.0.0", require: "active_job"
test/gemfiles/activerecord51.gemfile
@@ -6,3 +6,4 @@ gemspec path: &quot;../../&quot; @@ -6,3 +6,4 @@ gemspec path: &quot;../../&quot;
6 gem "sqlite3" 6 gem "sqlite3"
7 gem "activerecord", "~> 5.1.0" 7 gem "activerecord", "~> 5.1.0"
8 gem "actionpack", "~> 5.1.0" 8 gem "actionpack", "~> 5.1.0"
  9 +gem "activejob", "~> 5.1.0", require: "active_job"
test/gemfiles/activerecord52.gemfile
@@ -6,3 +6,4 @@ gemspec path: &quot;../../&quot; @@ -6,3 +6,4 @@ gemspec path: &quot;../../&quot;
6 gem "sqlite3" 6 gem "sqlite3"
7 gem "activerecord", "~> 5.2.0" 7 gem "activerecord", "~> 5.2.0"
8 gem "actionpack", "~> 5.2.0" 8 gem "actionpack", "~> 5.2.0"
  9 +gem "activejob", "~> 5.2.0", require: "active_job"
test/reindex_test.rb
1 require_relative "test_helper" 1 require_relative "test_helper"
2 2
3 class ReindexTest < Minitest::Test 3 class ReindexTest < Minitest::Test
4 - def setup  
5 - super  
6 - Sku.destroy_all 4 + def test_record_inline
  5 + store_names ["Product A", "Product B"], reindex: false
  6 +
  7 + product = Product.find_by!(name: "Product A")
  8 + product.reindex(refresh: true)
  9 + assert_search "product", ["Product A"]
7 end 10 end
8 11
9 - def test_scoped 12 + def test_record_async
  13 + store_names ["Product A", "Product B"], reindex: false
  14 +
  15 + product = Product.find_by!(name: "Product A")
  16 + product.reindex(mode: :async)
  17 + Product.search_index.refresh
  18 + assert_search "product", ["Product A"]
  19 + end
  20 +
  21 + def test_record_queue
  22 + skip unless defined?(ActiveJob) && defined?(Redis)
  23 +
  24 + reindex_queue = Product.searchkick_index.reindex_queue
  25 + reindex_queue.clear
  26 +
  27 + store_names ["Product A", "Product B"], reindex: false
  28 +
  29 + product = Product.find_by!(name: "Product A")
  30 + product.reindex(mode: :queue)
  31 + Product.search_index.refresh
  32 + assert_search "product", []
  33 +
  34 + Searchkick::ProcessQueueJob.perform_now(class_name: "Product")
  35 + Product.search_index.refresh
  36 + assert_search "product", ["Product A"]
  37 + end
  38 +
  39 + def test_relation_inline
10 skip if nobrainer? || cequel? 40 skip if nobrainer? || cequel?
11 41
12 store_names ["Product A"] 42 store_names ["Product A"]
13 - Searchkick.callbacks(false) do  
14 - store_names ["Product B", "Product C"]  
15 - end 43 + store_names ["Product B", "Product C"], reindex: false
16 Product.where(name: "Product B").reindex(refresh: true) 44 Product.where(name: "Product B").reindex(refresh: true)
17 assert_search "product", ["Product A", "Product B"] 45 assert_search "product", ["Product A", "Product B"]
18 end 46 end
19 47
20 - def test_associations 48 + def test_relation_associations
21 skip if nobrainer? || cequel? 49 skip if nobrainer? || cequel?
22 50
23 store_names ["Product A"] 51 store_names ["Product A"]
@@ -27,12 +55,18 @@ class ReindexTest &lt; Minitest::Test @@ -27,12 +55,18 @@ class ReindexTest &lt; Minitest::Test
27 assert_search "product", ["Product A", "Product B"] 55 assert_search "product", ["Product A", "Product B"]
28 end 56 end
29 57
30 - def test_async 58 + def test_relation_async
  59 + skip "Not available yet"
  60 + end
  61 +
  62 + def test_relation_queue
  63 + skip "Not available yet"
  64 + end
  65 +
  66 + def test_full_async
31 skip unless defined?(ActiveJob) 67 skip unless defined?(ActiveJob)
32 68
33 - Searchkick.callbacks(false) do  
34 - store_names ["Product A"]  
35 - end 69 + store_names ["Product A"], reindex: false
36 reindex = Product.reindex(async: true) 70 reindex = Product.reindex(async: true)
37 assert_search "product", [], conversions: false 71 assert_search "product", [], conversions: false
38 72
@@ -48,12 +82,10 @@ class ReindexTest &lt; Minitest::Test @@ -48,12 +82,10 @@ class ReindexTest &lt; Minitest::Test
48 assert_search "product", ["Product A"] 82 assert_search "product", ["Product A"]
49 end 83 end
50 84
51 - def test_async_wait 85 + def test_full_async_wait
52 skip unless defined?(ActiveJob) && defined?(Redis) 86 skip unless defined?(ActiveJob) && defined?(Redis)
53 87
54 - Searchkick.callbacks(false) do  
55 - store_names ["Product A"]  
56 - end 88 + store_names ["Product A"], reindex: false
57 89
58 capture_io do 90 capture_io do
59 Product.reindex(async: {wait: true}) 91 Product.reindex(async: {wait: true})
@@ -62,7 +94,7 @@ class ReindexTest &lt; Minitest::Test @@ -62,7 +94,7 @@ class ReindexTest &lt; Minitest::Test
62 assert_search "product", ["Product A"] 94 assert_search "product", ["Product A"]
63 end 95 end
64 96
65 - def test_async_non_integer_pk 97 + def test_full_async_non_integer_pk
66 skip unless defined?(ActiveJob) 98 skip unless defined?(ActiveJob)
67 99
68 Sku.create(id: SecureRandom.hex, name: "Test") 100 Sku.create(id: SecureRandom.hex, name: "Test")
@@ -72,9 +104,11 @@ class ReindexTest &lt; Minitest::Test @@ -72,9 +104,11 @@ class ReindexTest &lt; Minitest::Test
72 index = Searchkick::Index.new(reindex[:index_name]) 104 index = Searchkick::Index.new(reindex[:index_name])
73 index.refresh 105 index.refresh
74 assert_equal 1, index.total_docs 106 assert_equal 1, index.total_docs
  107 + ensure
  108 + Sku.destroy_all
75 end 109 end
76 110
77 - def test_refresh_interval 111 + def test_full_refresh_interval
78 reindex = Product.reindex(refresh_interval: "30s", async: true, import: false) 112 reindex = Product.reindex(refresh_interval: "30s", async: true, import: false)
79 index = Searchkick::Index.new(reindex[:index_name]) 113 index = Searchkick::Index.new(reindex[:index_name])
80 assert_nil Product.search_index.refresh_interval 114 assert_nil Product.search_index.refresh_interval
@@ -85,17 +119,64 @@ class ReindexTest &lt; Minitest::Test @@ -85,17 +119,64 @@ class ReindexTest &lt; Minitest::Test
85 assert_equal "1s", Product.search_index.refresh_interval 119 assert_equal "1s", Product.search_index.refresh_interval
86 end 120 end
87 121
88 - def test_resume 122 + def test_full_resume
89 assert Product.reindex(resume: true) 123 assert Product.reindex(resume: true)
90 end 124 end
91 125
92 - def test_refresh_full_reindex 126 + def test_full_refresh
93 Product.reindex(refresh: true) 127 Product.reindex(refresh: true)
94 end 128 end
95 129
96 - def test_partial_async 130 + def test_full_partial_async
97 store_names ["Product A"] 131 store_names ["Product A"]
98 # warn for now 132 # warn for now
99 Product.reindex(:search_name, async: true) 133 Product.reindex(:search_name, async: true)
100 end 134 end
  135 +
  136 + def test_callbacks_false
  137 + Searchkick.callbacks(false) do
  138 + store_names ["Product A", "Product B"]
  139 + end
  140 + assert_search "product", []
  141 + end
  142 +
  143 + def test_callbacks_bulk
  144 + Searchkick.callbacks(:bulk) do
  145 + store_names ["Product A", "Product B"]
  146 + end
  147 + Product.search_index.refresh
  148 + assert_search "product", ["Product A", "Product B"]
  149 + end
  150 +
  151 + def test_callbacks_queue
  152 + skip unless defined?(ActiveJob) && defined?(Redis)
  153 +
  154 + reindex_queue = Product.searchkick_index.reindex_queue
  155 + reindex_queue.clear
  156 +
  157 + Searchkick.callbacks(:queue) do
  158 + store_names ["Product A", "Product B"]
  159 + end
  160 + Product.searchkick_index.refresh
  161 + assert_search "product", [], load: false, conversions: false
  162 + assert_equal 2, reindex_queue.length
  163 +
  164 + Searchkick::ProcessQueueJob.perform_later(class_name: "Product")
  165 + Product.searchkick_index.refresh
  166 + assert_search "product", ["Product A", "Product B"], load: false
  167 + assert_equal 0, reindex_queue.length
  168 +
  169 + Searchkick.callbacks(:queue) do
  170 + Product.where(name: "Product B").destroy_all
  171 + Product.create!(name: "Product C")
  172 + end
  173 + Product.searchkick_index.refresh
  174 + assert_search "product", ["Product A", "Product B"], load: false
  175 + assert_equal 2, reindex_queue.length
  176 +
  177 + Searchkick::ProcessQueueJob.perform_later(class_name: "Product")
  178 + Product.searchkick_index.refresh
  179 + assert_search "product", ["Product A", "Product C"], load: false
  180 + assert_equal 0, reindex_queue.length
  181 + end
101 end 182 end
test/test_helper.rb
@@ -71,15 +71,23 @@ class Minitest::Test @@ -71,15 +71,23 @@ class Minitest::Test
71 71
72 protected 72 protected
73 73
74 - def store(documents, klass = Product)  
75 - documents.shuffle.each do |document|  
76 - klass.create!(document) 74 + def store(documents, klass = Product, reindex: true)
  75 + if reindex
  76 + documents.shuffle.each do |document|
  77 + klass.create!(document)
  78 + end
  79 + klass.searchkick_index.refresh
  80 + else
  81 + Searchkick.callbacks(false) do
  82 + documents.shuffle.each do |document|
  83 + klass.create!(document)
  84 + end
  85 + end
77 end 86 end
78 - klass.searchkick_index.refresh  
79 end 87 end
80 88
81 - def store_names(names, klass = Product)  
82 - store names.map { |name| {name: name} }, klass 89 + def store_names(names, klass = Product, reindex: true)
  90 + store names.map { |name| {name: name} }, klass, reindex: reindex
83 end 91 end
84 92
85 # no order 93 # no order