Allow substring searches

Andrew Kane
2 parents 22a3645c 19e08d6a
Showing 5 changed files with 157 additions and 30 deletions Show diff stats
README.md
lib/searchkick/reindex.rb
lib/searchkick/search.rb
test/autocomplete_test.rb
test/test_helper.rb
@@ -148,6 +148,32 @@ To change this, use:
 Product.search "fresh honey", partial: true # fresh OR honey
 ```
  
+By default, results must match the entire word - `back` will not match `backpack`. You can change this behavior with:
+
+```ruby
+class Product < ActiveRecord::Base
+  searchkick word_start: [:name]
+end
+```
+
+And to search:
+
+```ruby
+Product.search "back", fields: [{name: :word_start}]
+```
+
+Available options are:
+
+```ruby
+:word # default
+:word_start
+:word_middle
+:word_end
+:text_start
+:text_middle
+:text_end
+```
+
 ### Synonyms
  
 ```ruby
@@ -293,14 +319,14 @@ First, specify which fields use this feature.  This is necessary since autocompl
  
 ```ruby
 class City < ActiveRecord::Base
-  searchkick autocomplete: ["name"]
+  searchkick text_start: [:name]
 end
 ```
  
 Reindex and search with:
  
 ```ruby
-City.search "san fr", autocomplete: true
+City.search "san fr", fields: [{name: :text_start}]
 ```
  
 Typically, you want to use a Javascript library like [typeahead.js](http://twitter.github.io/typeahead.js/) or [jQuery UI](http://jqueryui.com/autocomplete/).
@@ -314,7 +340,7 @@ First, add a controller action.
 class CitiesController < ApplicationController
  
   def autocomplete
-    render json: City.search(params[:query], autocomplete: true, limit: 10).map(&:name)
+    render json: City.search(params[:query], fields: [{name: :text_start}], limit: 10).map(&:name)
   end
  
 end
@@ -128,6 +128,41 @@ module Searchkick
                 type: "custom",
                 tokenizer: "standard",
                 filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
+              },
+              searchkick_suggest_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
+              },
+              searchkick_text_start_index: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
+              },
+              searchkick_text_middle_index: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
+              },
+              searchkick_text_end_index: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
+              },
+              searchkick_word_start_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
+              },
+              searchkick_word_middle_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
+              },
+              searchkick_word_end_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
               }
             },
             filter: {
@@ -145,6 +180,16 @@ module Searchkick
               searchkick_suggest_shingle: {
                 type: "shingle",
                 max_shingle_size: 5
+              },
+              searchkick_edge_ngram: {
+                type: "edgeNGram",
+                min_gram: 1,
+                max_gram: 50
+              },
+              searchkick_ngram: {
+                type: "nGram",
+                min_gram: 1,
+                max_gram: 50
               }
             },
             tokenizer: {
@@ -202,10 +247,12 @@ module Searchkick
           }
         end
  
-        # autocomplete and suggest
-        autocomplete = (options[:autocomplete] || []).map(&:to_s)
-        suggest = (options[:suggest] || []).map(&:to_s)
-        (autocomplete + suggest).uniq.each do |field|
+        mapping_options = Hash[
+          [:autocomplete, :suggest, :text_start, :text_middle, :text_end, :word_start, :word_middle, :word_end]
+            .map{|type| [type, (options[type] || []).map(&:to_s)] }
+        ]
+
+        mapping_options.values.flatten.uniq.each do |field|
           field_mapping = {
             type: "multi_field",
             fields: {
@@ -215,12 +262,13 @@ module Searchkick
               # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-highlighting.html#_fast_vector_highlighter
             }
           }
-          if autocomplete.include?(field)
-            field_mapping[:fields]["autocomplete"] = {type: "string", index: "analyzed", analyzer: "searchkick_autocomplete_index"}
-          end
-          if suggest.include?(field)
-            field_mapping[:fields]["suggest"] = {type: "string", index: "analyzed", analyzer: "searchkick_suggest_index"}
+
+          mapping_options.each do |type, fields|
+            if fields.include?(field)
+              field_mapping[:fields][type] = {type: "string", index: "analyzed", analyzer: "searchkick_#{type}_index"}
+            end
           end
+
           mapping[field] = field_mapping
         end
  
@@ -14,7 +14,10 @@ module Searchkick
           if options[:autocomplete]
             options[:fields].map{|f| "#{f}.autocomplete" }
           else
-            options[:fields].map{|f| "#{f}.analyzed" }
+            options[:fields].map do |value|
+              k, v = value.is_a?(Hash) ? value.to_a.first : [value, :word]
+              "#{k}.#{v == :word ? "analyzed" : v}"
+            end
           end
         else
           if options[:autocomplete]
@@ -67,23 +70,37 @@ module Searchkick
             }
           }
         else
-          shared_options = {
-            fields: fields,
-            query: term,
-            use_dis_max: false,
-            operator: operator
-          }
-          queries = [
-            {multi_match: shared_options.merge(boost: 10, analyzer: "searchkick_search")},
-            {multi_match: shared_options.merge(boost: 10, analyzer: "searchkick_search2")}
-          ]
-          if options[:misspellings] != false
-            distance = (options[:misspellings].is_a?(Hash) && options[:misspellings][:distance]) || 1
-            queries.concat [
-              {multi_match: shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search")},
-              {multi_match: shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search2")}
-            ]
+          queries = []
+          fields.each do |field|
+            if field == "_all" or field.end_with?(".analyzed")
+              shared_options = {
+                fields: [field],
+                query: term,
+                use_dis_max: false,
+                operator: operator
+              }
+              queries.concat [
+                {multi_match: shared_options.merge(boost: 10, analyzer: "searchkick_search")},
+                {multi_match: shared_options.merge(boost: 10, analyzer: "searchkick_search2")}
+              ]
+              if options[:misspellings] != false
+                distance = (options[:misspellings].is_a?(Hash) && options[:misspellings][:distance]) || 1
+                queries.concat [
+                  {multi_match: shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search")},
+                  {multi_match: shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search2")}
+                ]
+              end
+            else
+              queries << {
+                multi_match: {
+                  fields: [field],
+                  query: term,
+                  analyzer: "searchkick_autocomplete_search"
+                }
+              }
+            end
           end
+
           payload = {
             dis_max: {
               queries: queries
@@ -17,4 +17,34 @@ class TestAutocomplete &lt; Minitest::Unit::TestCase
     assert_search "hum", ["Hummus"], autocomplete: true, fields: [:name]
   end
  
+  def test_text_start
+    store_names ["Where in the World is Carmen San Diego?"]
+    assert_search "whe", ["Where in the World is Carmen San Diego?"], fields: [{name: :text_start}]
+  end
+
+  def test_text_middle
+    store_names ["Where in the World is Carmen San Diego?"]
+    assert_search "n the wor", ["Where in the World is Carmen San Diego?"], fields: [{name: :text_middle}]
+  end
+
+  def test_text_end
+    store_names ["Where in the World is Carmen San Diego?"]
+    assert_search "ego?", ["Where in the World is Carmen San Diego?"], fields: [{name: :text_end}]
+  end
+
+  def test_word_start
+    store_names ["Where in the World is Carmen San Diego?"]
+    assert_search "car", ["Where in the World is Carmen San Diego?"], fields: [{name: :word_start}]
+  end
+
+  def test_word_middle
+    store_names ["Where in the World is Carmen San Diego?"]
+    assert_search "orl", ["Where in the World is Carmen San Diego?"], fields: [{name: :word_middle}]
+  end
+
+  def test_word_end
+    store_names ["Where in the World is Carmen San Diego?"]
+    assert_search "men", ["Where in the World is Carmen San Diego?"], fields: [{name: :word_end}]
+  end
+
 end
@@ -115,7 +115,13 @@ class Product
     suggest: [:name, :color],
     conversions: "conversions",
     personalize: "user_ids",
-    locations: ["location", "multiple_locations"]
+    locations: ["location", "multiple_locations"],
+    text_start: [:name],
+    text_middle: [:name],
+    text_end: [:name],
+    word_start: [:name],
+    word_middle: [:name],
+    word_end: [:name]
  
   attr_accessor :conversions, :user_ids
...	...	@@ -148,6 +148,32 @@ To change this, use:
148	148	Product.search "fresh honey", partial: true # fresh OR honey
149	149	```
150	150
	151	+By default, results must match the entire word - `back` will not match `backpack`. You can change this behavior with:
	152	+
	153	+```ruby
	154	+class Product < ActiveRecord::Base
	155	+ searchkick word_start: [:name]
	156	+end
	157	+```
	158	+
	159	+And to search:
	160	+
	161	+```ruby
	162	+Product.search "back", fields: [{name: :word_start}]
	163	+```
	164	+
	165	+Available options are:
	166	+
	167	+```ruby
	168	+:word # default
	169	+:word_start
	170	+:word_middle
	171	+:word_end
	172	+:text_start
	173	+:text_middle
	174	+:text_end
	175	+```
	176	+
151	177	### Synonyms
152	178
153	179	```ruby
...	...	@@ -293,14 +319,14 @@ First, specify which fields use this feature. This is necessary since autocompl
293	319
294	320	```ruby
295	321	class City < ActiveRecord::Base
296		- searchkick autocomplete: ["name"]
	322	+ searchkick text_start: [:name]
297	323	end
298	324	```
299	325
300	326	Reindex and search with:
301	327
302	328	```ruby
303		-City.search "san fr", autocomplete: true
	329	+City.search "san fr", fields: [{name: :text_start}]
304	330	```
305	331
306	332	Typically, you want to use a Javascript library like [typeahead.js](http://twitter.github.io/typeahead.js/) or [jQuery UI](http://jqueryui.com/autocomplete/).
...	...	@@ -314,7 +340,7 @@ First, add a controller action.
314	340	class CitiesController < ApplicationController
315	341
316	342	def autocomplete
317		- render json: City.search(params[:query], autocomplete: true, limit: 10).map(&:name)
	343	+ render json: City.search(params[:query], fields: [{name: :text_start}], limit: 10).map(&:name)
318	344	end
319	345
320	346	end
...	...
...	...	@@ -128,6 +128,41 @@ module Searchkick
128	128	type: "custom",
129	129	tokenizer: "standard",
130	130	filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
	131	+ },
	132	+ searchkick_suggest_index: {
	133	+ type: "custom",
	134	+ tokenizer: "standard",
	135	+ filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
	136	+ },
	137	+ searchkick_text_start_index: {
	138	+ type: "custom",
	139	+ tokenizer: "keyword",
	140	+ filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
	141	+ },
	142	+ searchkick_text_middle_index: {
	143	+ type: "custom",
	144	+ tokenizer: "keyword",
	145	+ filter: ["lowercase", "asciifolding", "searchkick_ngram"]
	146	+ },
	147	+ searchkick_text_end_index: {
	148	+ type: "custom",
	149	+ tokenizer: "keyword",
	150	+ filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
	151	+ },
	152	+ searchkick_word_start_index: {
	153	+ type: "custom",
	154	+ tokenizer: "standard",
	155	+ filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
	156	+ },
	157	+ searchkick_word_middle_index: {
	158	+ type: "custom",
	159	+ tokenizer: "standard",
	160	+ filter: ["lowercase", "asciifolding", "searchkick_ngram"]
	161	+ },
	162	+ searchkick_word_end_index: {
	163	+ type: "custom",
	164	+ tokenizer: "standard",
	165	+ filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
131	166	}
132	167	},
133	168	filter: {
...	...	@@ -145,6 +180,16 @@ module Searchkick
145	180	searchkick_suggest_shingle: {
146	181	type: "shingle",
147	182	max_shingle_size: 5
	183	+ },
	184	+ searchkick_edge_ngram: {
	185	+ type: "edgeNGram",
	186	+ min_gram: 1,
	187	+ max_gram: 50
	188	+ },
	189	+ searchkick_ngram: {
	190	+ type: "nGram",
	191	+ min_gram: 1,
	192	+ max_gram: 50
148	193	}
149	194	},
150	195	tokenizer: {
...	...	@@ -202,10 +247,12 @@ module Searchkick
202	247	}
203	248	end
204	249
205		- # autocomplete and suggest
206		- autocomplete = (options[:autocomplete] \|\| []).map(&:to_s)
207		- suggest = (options[:suggest] \|\| []).map(&:to_s)
208		- (autocomplete + suggest).uniq.each do \|field\|
	250	+ mapping_options = Hash[
	251	+ [:autocomplete, :suggest, :text_start, :text_middle, :text_end, :word_start, :word_middle, :word_end]
	252	+ .map{\|type\| [type, (options[type] \|\| []).map(&:to_s)] }
	253	+ ]
	254	+
	255	+ mapping_options.values.flatten.uniq.each do \|field\|
209	256	field_mapping = {
210	257	type: "multi_field",
211	258	fields: {
...	...	@@ -215,12 +262,13 @@ module Searchkick
215	262	# http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-highlighting.html#_fast_vector_highlighter
216	263	}
217	264	}
218		- if autocomplete.include?(field)
219		- field_mapping[:fields]["autocomplete"] = {type: "string", index: "analyzed", analyzer: "searchkick_autocomplete_index"}
220		- end
221		- if suggest.include?(field)
222		- field_mapping[:fields]["suggest"] = {type: "string", index: "analyzed", analyzer: "searchkick_suggest_index"}
	265	+
	266	+ mapping_options.each do \|type, fields\|
	267	+ if fields.include?(field)
	268	+ field_mapping[:fields][type] = {type: "string", index: "analyzed", analyzer: "searchkick_#{type}_index"}
	269	+ end
223	270	end
	271	+
224	272	mapping[field] = field_mapping
225	273	end
226	274
...	...
...	...	@@ -14,7 +14,10 @@ module Searchkick
14	14	if options[:autocomplete]
15	15	options[:fields].map{\|f\| "#{f}.autocomplete" }
16	16	else
17		- options[:fields].map{\|f\| "#{f}.analyzed" }
	17	+ options[:fields].map do \|value\|
	18	+ k, v = value.is_a?(Hash) ? value.to_a.first : [value, :word]
	19	+ "#{k}.#{v == :word ? "analyzed" : v}"
	20	+ end
18	21	end
19	22	else
20	23	if options[:autocomplete]
...	...	@@ -67,23 +70,37 @@ module Searchkick
67	70	}
68	71	}
69	72	else
70		- shared_options = {
71		- fields: fields,
72		- query: term,
73		- use_dis_max: false,
74		- operator: operator
75		- }
76		- queries = [
77		- {multi_match: shared_options.merge(boost: 10, analyzer: "searchkick_search")},
78		- {multi_match: shared_options.merge(boost: 10, analyzer: "searchkick_search2")}
79		- ]
80		- if options[:misspellings] != false
81		- distance = (options[:misspellings].is_a?(Hash) && options[:misspellings][:distance]) \|\| 1
82		- queries.concat [
83		- {multi_match: shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search")},
84		- {multi_match: shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search2")}
85		- ]
	73	+ queries = []
	74	+ fields.each do \|field\|
	75	+ if field == "_all" or field.end_with?(".analyzed")
	76	+ shared_options = {
	77	+ fields: [field],
	78	+ query: term,
	79	+ use_dis_max: false,
	80	+ operator: operator
	81	+ }
	82	+ queries.concat [
	83	+ {multi_match: shared_options.merge(boost: 10, analyzer: "searchkick_search")},
	84	+ {multi_match: shared_options.merge(boost: 10, analyzer: "searchkick_search2")}
	85	+ ]
	86	+ if options[:misspellings] != false
	87	+ distance = (options[:misspellings].is_a?(Hash) && options[:misspellings][:distance]) \|\| 1
	88	+ queries.concat [
	89	+ {multi_match: shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search")},
	90	+ {multi_match: shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search2")}
	91	+ ]
	92	+ end
	93	+ else
	94	+ queries << {
	95	+ multi_match: {
	96	+ fields: [field],
	97	+ query: term,
	98	+ analyzer: "searchkick_autocomplete_search"
	99	+ }
	100	+ }
	101	+ end
86	102	end
	103	+
87	104	payload = {
88	105	dis_max: {
89	106	queries: queries
...	...
...	...	@@ -17,4 +17,34 @@ class TestAutocomplete < Minitest::Unit::TestCase
17	17	assert_search "hum", ["Hummus"], autocomplete: true, fields: [:name]
18	18	end
19	19
	20	+ def test_text_start
	21	+ store_names ["Where in the World is Carmen San Diego?"]
	22	+ assert_search "whe", ["Where in the World is Carmen San Diego?"], fields: [{name: :text_start}]
	23	+ end
	24	+
	25	+ def test_text_middle
	26	+ store_names ["Where in the World is Carmen San Diego?"]
	27	+ assert_search "n the wor", ["Where in the World is Carmen San Diego?"], fields: [{name: :text_middle}]
	28	+ end
	29	+
	30	+ def test_text_end
	31	+ store_names ["Where in the World is Carmen San Diego?"]
	32	+ assert_search "ego?", ["Where in the World is Carmen San Diego?"], fields: [{name: :text_end}]
	33	+ end
	34	+
	35	+ def test_word_start
	36	+ store_names ["Where in the World is Carmen San Diego?"]
	37	+ assert_search "car", ["Where in the World is Carmen San Diego?"], fields: [{name: :word_start}]
	38	+ end
	39	+
	40	+ def test_word_middle
	41	+ store_names ["Where in the World is Carmen San Diego?"]
	42	+ assert_search "orl", ["Where in the World is Carmen San Diego?"], fields: [{name: :word_middle}]
	43	+ end
	44	+
	45	+ def test_word_end
	46	+ store_names ["Where in the World is Carmen San Diego?"]
	47	+ assert_search "men", ["Where in the World is Carmen San Diego?"], fields: [{name: :word_end}]
	48	+ end
	49	+
20	50	end
...	...
...	...	@@ -115,7 +115,13 @@ class Product
115	115	suggest: [:name, :color],
116	116	conversions: "conversions",
117	117	personalize: "user_ids",
118		- locations: ["location", "multiple_locations"]
	118	+ locations: ["location", "multiple_locations"],
	119	+ text_start: [:name],
	120	+ text_middle: [:name],
	121	+ text_end: [:name],
	122	+ word_start: [:name],
	123	+ word_middle: [:name],
	124	+ word_end: [:name]
119	125
120	126	attr_accessor :conversions, :user_ids
121	127
...	...