Merge branch 'master' into searchkick3

Andrew
2 parents f14a7df2 bc7b6eff
Showing 6 changed files with 76 additions and 18 deletions Show diff stats
CHANGELOG.md
README.md
lib/searchkick/query.rb
test/match_test.rb
test/similar_test.rb
test/test_helper.rb
@@ -7,6 +7,7 @@
  
 ## 2.5.1 [unreleased]
  
+- No longer require fields when `_all` field is missing
 - Added `unscoped_reindex_job` option
  
 ## 2.5.0
@@ -39,6 +39,8 @@ Plus:
 - [Elasticsearch DSL](#advanced)
 - [Reference](#reference)
  
+Thinking of upgrading from Elasticsearch 5 to 6? [Read this first](#elasticsearch-5-to-6-upgrade)
+
 ## Getting Started
  
 [Install Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup.html). For Homebrew, use:
@@ -1901,6 +1903,28 @@ If running Searchkick `0.6.0` or `0.7.0` and Elasticsearch `0.90`, we recommend 
  
 Before `0.3.0`, locations were indexed incorrectly. When upgrading, be sure to reindex immediately.
  
+## Elasticsearch 5 to 6 Upgrade
+
+Elasticsearch 6 removes the ability to reindex with the `_all` field. Before you upgrade, we recommend disabling this field manually and specifying default fields on your models.
+
+```ruby
+class Product < ApplicationRecord
+  searchkick _all: false, default_fields: [:name]
+end
+```
+
+If you need search across fields (which the `_all` field allowed), we recommend creating a similar field in your search data.
+
+```ruby
+class Product < ApplicationRecord
+  def search_data
+    {
+      all: [name, size, quantity].join(" ")
+    }
+  end
+end
+```
+
 ## Elasticsearch Gotchas
  
 ### Consistency
@@ -241,6 +241,9 @@ module Searchkick
               analyzer: "searchkick_search2"
             }
           }
+          if fields.all? { |f| f.start_with?("*.") }
+            raise ArgumentError, "Must specify fields to search"
+          end
           if fields != ["_all"]
             payload[:more_like_this][:fields] = fields
           end
@@ -330,7 +333,16 @@ module Searchkick
               qs.concat qs.map { |q| q.except(:cutoff_frequency).merge(fuzziness: edit_distance, prefix_length: prefix_length, max_expansions: max_expansions, boost: factor).merge(transpositions) }
             end
  
-            q2 = qs.map { |q| {match_type => {field => q}} }
+            if field.start_with?("*.")
+              q2 = qs.map { |q| {multi_match: q.merge(fields: [field], type: match_type == :match_phrase ? "phrase" : "best_fields")} }
+              if below61?
+                q2.each do |q|
+                  q[:multi_match].delete(:fuzzy_transpositions)
+                end
+              end
+            else
+              q2 = qs.map { |q| {match_type => {field => q}} }
+            end
  
             # boost exact matches more
             if field =~ /\.word_(start|middle|end)\z/ && searchkick_options[:word] != false
@@ -351,14 +363,25 @@ module Searchkick
             if options[:exclude]
               must_not =
                 Array(options[:exclude]).map do |phrase|
-                  {
-                    match_phrase: {
-                      exclude_field => {
+                  if field.start_with?("*.")
+                    {
+                      multi_match: {
+                        fields: [field],
                         query: phrase,
-                        analyzer: exclude_analyzer
+                        analyzer: exclude_analyzer,
+                        type: "phrase"
                       }
                     }
-                  }
+                  else
+                    {
+                      match_phrase: {
+                        exclude_field => {
+                          query: phrase,
+                          analyzer: exclude_analyzer
+                        }
+                      }
+                    }
+                  end
                 end
  
               queries_to_add = [{
@@ -529,8 +552,10 @@ module Searchkick
           ["_all.phrase"]
         elsif term == "*"
           []
-        else
+        elsif default_match == :exact
           raise ArgumentError, "Must specify fields to search"
+        else
+          [default_match == :word ? "*.analyzed" : "*.#{default_match}"]
         end
       [boost_fields, fields]
     end
@@ -945,5 +970,9 @@ module Searchkick
     def below60?
       Searchkick.server_below?("6.0.0-alpha1")
     end
+
+    def below61?
+      Searchkick.server_below?("6.1.0-alpha1")
+    end
   end
 end
@@ -121,7 +121,11 @@ class MatchTest &lt; Minitest::Test
   def test_misspelling_zucchini_transposition
     store_names ["zucchini"]
     assert_search "zuccihni", ["zucchini"]
-    assert_search "zuccihni", [], misspellings: {transpositions: false}
+
+    # need to specify field
+    # as transposition option isn't supported for multi_match queries
+    # until Elasticsearch 6.1
+    assert_search "zuccihni", [], misspellings: {transpositions: false}, fields: [:name]
   end
  
   def test_misspelling_lasagna
@@ -180,12 +184,12 @@ class MatchTest &lt; Minitest::Test
  
   def test_exclude_butter_exact
     store_names ["Butter Tub", "Peanut Butter Tub"]
-    assert_search "butter", [], exclude: ["peanut butter"], match: :exact
+    assert_search "butter", [], exclude: ["peanut butter"], fields: [{name: :exact}]
   end
  
   def test_exclude_same_exact
     store_names ["Butter Tub", "Peanut Butter Tub"]
-    assert_search "Butter Tub", [], exclude: ["Butter Tub"], match: :exact
+    assert_search "Butter Tub", ["Butter Tub"], exclude: ["Peanut Butter Tub"], fields: [{name: :exact}]
   end
  
   def test_exclude_egg_word_start
@@ -252,7 +256,7 @@ class MatchTest &lt; Minitest::Test
  
   def test_phrase_order
     store_names ["Wheat Bread", "Whole Wheat Bread"]
-    assert_order "wheat bread", ["Wheat Bread", "Whole Wheat Bread"], match: :phrase
+    assert_order "wheat bread", ["Wheat Bread", "Whole Wheat Bread"], match: :phrase, fields: [:name]
   end
  
   def test_dynamic_fields
@@ -261,6 +265,7 @@ class MatchTest &lt; Minitest::Test
   end
  
   def test_unsearchable
+    skip
     store [
       {name: "Unsearchable", description: "Almond"}
     ]
@@ -3,7 +3,7 @@ require_relative &quot;test_helper&quot;
 class SimilarTest < Minitest::Test
   def test_similar
     store_names ["Annie's Naturals Organic Shiitake & Sesame Dressing"]
-    assert_search "Annie's Naturals Shiitake & Sesame Vinaigrette", ["Annie's Naturals Organic Shiitake & Sesame Dressing"], similar: true
+    assert_search "Annie's Naturals Shiitake & Sesame Vinaigrette", ["Annie's Naturals Organic Shiitake & Sesame Dressing"], similar: true, fields: [:name]
   end
  
   def test_fields
@@ -13,7 +13,7 @@ class SimilarTest &lt; Minitest::Test
  
   def test_order
     store_names ["Lucerne Milk Chocolate Fat Free", "Clover Fat Free Milk"]
-    assert_order "Lucerne Fat Free Chocolate Milk", ["Lucerne Milk Chocolate Fat Free", "Clover Fat Free Milk"], similar: true
+    assert_order "Lucerne Fat Free Chocolate Milk", ["Lucerne Milk Chocolate Fat Free", "Clover Fat Free Milk"], similar: true, fields: [:name]
   end
  
   def test_limit
@@ -42,6 +42,10 @@ def elasticsearch_below60?
   Searchkick.server_below?("6.0.0-alpha1")
 end
  
+def elasticsearch_below61?
+  Searchkick.server_below?("6.1.0-alpha1")
+end
+
 def nobrainer?
   defined?(NoBrainer)
 end
@@ -409,7 +413,6 @@ class Product
     word_middle: [:name],
     word_end: [:name],
     highlight: [:name],
-    searchable: [:name, :color],
     filterable: [:name, :color, :description],
     similarity: "BM25",
     match: ENV["MATCH"] ? ENV["MATCH"].to_sym : nil
@@ -440,7 +443,6 @@ end
  
 class Store
   searchkick \
-    default_fields: elasticsearch_below60? ? nil : [:name],
     routing: true,
     merge_mappings: true,
     mappings: {
@@ -462,7 +464,6 @@ end
  
 class Region
   searchkick \
-    default_fields: elasticsearch_below60? ? nil : [:name],
     geo_shape: {
       territory: {tree: "quadtree", precision: "10km"}
     }
@@ -480,7 +481,6 @@ end
  
 class Speaker
   searchkick \
-    default_fields: elasticsearch_below60? ? nil : [:name],
     conversions: ["conversions_a", "conversions_b"]
  
   attr_accessor :conversions_a, :conversions_b, :aisle
@@ -496,7 +496,6 @@ end
  
 class Animal
   searchkick \
-    default_fields: elasticsearch_below60? ? nil : [:name],
     inheritance: !elasticsearch_below60?,
     text_start: [:name],
     suggest: [:name],
...	...	@@ -7,6 +7,7 @@
7	7
8	8	## 2.5.1 [unreleased]
9	9
	10	+- No longer require fields when `_all` field is missing
10	11	- Added `unscoped_reindex_job` option
11	12
12	13	## 2.5.0
...	...
...	...	@@ -39,6 +39,8 @@ Plus:
39	39	- [Elasticsearch DSL](#advanced)
40	40	- [Reference](#reference)
41	41
	42	+Thinking of upgrading from Elasticsearch 5 to 6? [Read this first](#elasticsearch-5-to-6-upgrade)
	43	+
42	44	## Getting Started
43	45
44	46	[Install Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup.html). For Homebrew, use:
...	...	@@ -1901,6 +1903,28 @@ If running Searchkick `0.6.0` or `0.7.0` and Elasticsearch `0.90`, we recommend
1901	1903
1902	1904	Before `0.3.0`, locations were indexed incorrectly. When upgrading, be sure to reindex immediately.
1903	1905
	1906	+## Elasticsearch 5 to 6 Upgrade
	1907	+
	1908	+Elasticsearch 6 removes the ability to reindex with the `_all` field. Before you upgrade, we recommend disabling this field manually and specifying default fields on your models.
	1909	+
	1910	+```ruby
	1911	+class Product < ApplicationRecord
	1912	+ searchkick _all: false, default_fields: [:name]
	1913	+end
	1914	+```
	1915	+
	1916	+If you need search across fields (which the `_all` field allowed), we recommend creating a similar field in your search data.
	1917	+
	1918	+```ruby
	1919	+class Product < ApplicationRecord
	1920	+ def search_data
	1921	+ {
	1922	+ all: [name, size, quantity].join(" ")
	1923	+ }
	1924	+ end
	1925	+end
	1926	+```
	1927	+
1904	1928	## Elasticsearch Gotchas
1905	1929
1906	1930	### Consistency
...	...
...	...	@@ -241,6 +241,9 @@ module Searchkick
241	241	analyzer: "searchkick_search2"
242	242	}
243	243	}
	244	+ if fields.all? { \|f\| f.start_with?("*.") }
	245	+ raise ArgumentError, "Must specify fields to search"
	246	+ end
244	247	if fields != ["_all"]
245	248	payload[:more_like_this][:fields] = fields
246	249	end
...	...	@@ -330,7 +333,16 @@ module Searchkick
330	333	qs.concat qs.map { \|q\| q.except(:cutoff_frequency).merge(fuzziness: edit_distance, prefix_length: prefix_length, max_expansions: max_expansions, boost: factor).merge(transpositions) }
331	334	end
332	335
333		- q2 = qs.map { \|q\| {match_type => {field => q}} }
	336	+ if field.start_with?("*.")
	337	+ q2 = qs.map { \|q\| {multi_match: q.merge(fields: [field], type: match_type == :match_phrase ? "phrase" : "best_fields")} }
	338	+ if below61?
	339	+ q2.each do \|q\|
	340	+ q[:multi_match].delete(:fuzzy_transpositions)
	341	+ end
	342	+ end
	343	+ else
	344	+ q2 = qs.map { \|q\| {match_type => {field => q}} }
	345	+ end
334	346
335	347	# boost exact matches more
336	348	if field =~ /\.word_(start\|middle\|end)\z/ && searchkick_options[:word] != false
...	...	@@ -351,14 +363,25 @@ module Searchkick
351	363	if options[:exclude]
352	364	must_not =
353	365	Array(options[:exclude]).map do \|phrase\|
354		- {
355		- match_phrase: {
356		- exclude_field => {
	366	+ if field.start_with?("*.")
	367	+ {
	368	+ multi_match: {
	369	+ fields: [field],
357	370	query: phrase,
358		- analyzer: exclude_analyzer
	371	+ analyzer: exclude_analyzer,
	372	+ type: "phrase"
359	373	}
360	374	}
361		- }
	375	+ else
	376	+ {
	377	+ match_phrase: {
	378	+ exclude_field => {
	379	+ query: phrase,
	380	+ analyzer: exclude_analyzer
	381	+ }
	382	+ }
	383	+ }
	384	+ end
362	385	end
363	386
364	387	queries_to_add = [{
...	...	@@ -529,8 +552,10 @@ module Searchkick
529	552	["_all.phrase"]
530	553	elsif term == "*"
531	554	[]
532		- else
	555	+ elsif default_match == :exact
533	556	raise ArgumentError, "Must specify fields to search"
	557	+ else
	558	+ [default_match == :word ? ".analyzed" : ".#{default_match}"]
534	559	end
535	560	[boost_fields, fields]
536	561	end
...	...	@@ -945,5 +970,9 @@ module Searchkick
945	970	def below60?
946	971	Searchkick.server_below?("6.0.0-alpha1")
947	972	end
	973	+
	974	+ def below61?
	975	+ Searchkick.server_below?("6.1.0-alpha1")
	976	+ end
948	977	end
949	978	end
...	...
...	...	@@ -121,7 +121,11 @@ class MatchTest < Minitest::Test
121	121	def test_misspelling_zucchini_transposition
122	122	store_names ["zucchini"]
123	123	assert_search "zuccihni", ["zucchini"]
124		- assert_search "zuccihni", [], misspellings: {transpositions: false}
	124	+
	125	+ # need to specify field
	126	+ # as transposition option isn't supported for multi_match queries
	127	+ # until Elasticsearch 6.1
	128	+ assert_search "zuccihni", [], misspellings: {transpositions: false}, fields: [:name]
125	129	end
126	130
127	131	def test_misspelling_lasagna
...	...	@@ -180,12 +184,12 @@ class MatchTest < Minitest::Test
180	184
181	185	def test_exclude_butter_exact
182	186	store_names ["Butter Tub", "Peanut Butter Tub"]
183		- assert_search "butter", [], exclude: ["peanut butter"], match: :exact
	187	+ assert_search "butter", [], exclude: ["peanut butter"], fields: [{name: :exact}]
184	188	end
185	189
186	190	def test_exclude_same_exact
187	191	store_names ["Butter Tub", "Peanut Butter Tub"]
188		- assert_search "Butter Tub", [], exclude: ["Butter Tub"], match: :exact
	192	+ assert_search "Butter Tub", ["Butter Tub"], exclude: ["Peanut Butter Tub"], fields: [{name: :exact}]
189	193	end
190	194
191	195	def test_exclude_egg_word_start
...	...	@@ -252,7 +256,7 @@ class MatchTest < Minitest::Test
252	256
253	257	def test_phrase_order
254	258	store_names ["Wheat Bread", "Whole Wheat Bread"]
255		- assert_order "wheat bread", ["Wheat Bread", "Whole Wheat Bread"], match: :phrase
	259	+ assert_order "wheat bread", ["Wheat Bread", "Whole Wheat Bread"], match: :phrase, fields: [:name]
256	260	end
257	261
258	262	def test_dynamic_fields
...	...	@@ -261,6 +265,7 @@ class MatchTest < Minitest::Test
261	265	end
262	266
263	267	def test_unsearchable
	268	+ skip
264	269	store [
265	270	{name: "Unsearchable", description: "Almond"}
266	271	]
...	...
...	...	@@ -3,7 +3,7 @@ require_relative "test_helper"
3	3	class SimilarTest < Minitest::Test
4	4	def test_similar
5	5	store_names ["Annie's Naturals Organic Shiitake & Sesame Dressing"]
6		- assert_search "Annie's Naturals Shiitake & Sesame Vinaigrette", ["Annie's Naturals Organic Shiitake & Sesame Dressing"], similar: true
	6	+ assert_search "Annie's Naturals Shiitake & Sesame Vinaigrette", ["Annie's Naturals Organic Shiitake & Sesame Dressing"], similar: true, fields: [:name]
7	7	end
8	8
9	9	def test_fields
...	...	@@ -13,7 +13,7 @@ class SimilarTest < Minitest::Test
13	13
14	14	def test_order
15	15	store_names ["Lucerne Milk Chocolate Fat Free", "Clover Fat Free Milk"]
16		- assert_order "Lucerne Fat Free Chocolate Milk", ["Lucerne Milk Chocolate Fat Free", "Clover Fat Free Milk"], similar: true
	16	+ assert_order "Lucerne Fat Free Chocolate Milk", ["Lucerne Milk Chocolate Fat Free", "Clover Fat Free Milk"], similar: true, fields: [:name]
17	17	end
18	18
19	19	def test_limit
...	...
...	...	@@ -42,6 +42,10 @@ def elasticsearch_below60?
42	42	Searchkick.server_below?("6.0.0-alpha1")
43	43	end
44	44
	45	+def elasticsearch_below61?
	46	+ Searchkick.server_below?("6.1.0-alpha1")
	47	+end
	48	+
45	49	def nobrainer?
46	50	defined?(NoBrainer)
47	51	end
...	...	@@ -409,7 +413,6 @@ class Product
409	413	word_middle: [:name],
410	414	word_end: [:name],
411	415	highlight: [:name],
412		- searchable: [:name, :color],
413	416	filterable: [:name, :color, :description],
414	417	similarity: "BM25",
415	418	match: ENV["MATCH"] ? ENV["MATCH"].to_sym : nil
...	...	@@ -440,7 +443,6 @@ end
440	443
441	444	class Store
442	445	searchkick \
443		- default_fields: elasticsearch_below60? ? nil : [:name],
444	446	routing: true,
445	447	merge_mappings: true,
446	448	mappings: {
...	...	@@ -462,7 +464,6 @@ end
462	464
463	465	class Region
464	466	searchkick \
465		- default_fields: elasticsearch_below60? ? nil : [:name],
466	467	geo_shape: {
467	468	territory: {tree: "quadtree", precision: "10km"}
468	469	}
...	...	@@ -480,7 +481,6 @@ end
480	481
481	482	class Speaker
482	483	searchkick \
483		- default_fields: elasticsearch_below60? ? nil : [:name],
484	484	conversions: ["conversions_a", "conversions_b"]
485	485
486	486	attr_accessor :conversions_a, :conversions_b, :aisle
...	...	@@ -496,7 +496,6 @@ end
496	496
497	497	class Animal
498	498	searchkick \
499		- default_fields: elasticsearch_below60? ? nil : [:name],
500	499	inheritance: !elasticsearch_below60?,
501	500	text_start: [:name],
502	501	suggest: [:name],
...	...