First version of typeahead - closes #5

Andrew Kane
1 parent 26a57398
Showing 5 changed files with 101 additions and 16 deletions Show diff stats
README.md
lib/searchkick/reindex.rb
lib/searchkick/search.rb
test/match_test.rb
test/test_helper.rb
@@ -17,6 +17,7 @@ Plus:
 - query like SQL - no need to learn a new query language
 - reindex without downtime
 - easily personalize results for each user [master branch]
+- typeahead / autocomplete [master branch]
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com)
@@ -130,6 +131,22 @@ To change this, use:
 Product.search "fresh honey", partial: true # fresh OR honey
 ```
+### Typeahead / Autocomplete [master branch]
+
+You must specify which fields use this feature since this can increase the index size significantly.  Don’t worry - this gives you blazing faster queries.
+
+```ruby
+class Product < ActiveRecord::Base
+  searchkick typeahead: [:name]
+end
+```
+
+Reindex and search with:
+
+```ruby
+Product.search "puddi", typeahead: true
+```
+
 ### Synonyms
 ```ruby
@@ -349,11 +366,10 @@ end
 ## Thanks
-Thanks to Karel Minarik for [Tire](https://github.com/karmi/tire) and Jaroslav Kalistsuk for [zero downtime reindexing](https://gist.github.com/jarosan/3124884).
+Thanks to Karel Minarik for [Tire](https://github.com/karmi/tire), Jaroslav Kalistsuk for [zero downtime reindexing](https://gist.github.com/jarosan/3124884), and Alex Leschenko for [Elasticsearch autocomplete](https://github.com/leschenko/elasticsearch_autocomplete).
 ## TODO
-- Custom results for each user
 - Make Searchkick work with any language
 - Built-in synonyms from WordNet
 - [Did you mean?](http://www.elasticsearch.org/guide/reference/api/search/suggest/)
@@ -68,6 +68,17 @@ module Searchkick
               type: "custom",
               tokenizer: "standard",
               filter: ["standard", "lowercase", "asciifolding", "stop", "snowball"]
+            },
+            # https://github.com/leschenko/elasticsearch_autocomplete/blob/master/lib/elasticsearch_autocomplete/analyzers.rb
+            searchkick_typeahead_index: {
+              type: "custom",
+              tokenizer: "searchkick_typeahead_ngram",
+              filter: ["lowercase", "asciifolding"]
+            },
+            searchkick_typeahead_search: {
+              type: "custom",
+              tokenizer: "keyword",
+              filter: ["lowercase", "asciifolding"]
             }
           },
           filter: {
@@ -82,9 +93,18 @@ module Searchkick
               output_unigrams: false,
               output_unigrams_if_no_shingles: true
             }
+          },
+          tokenizer: {
+            searchkick_typeahead_ngram: {
+              type: "edgeNGram",
+              min_gram: 1,
+              max_gram: 50
+            }
           }
         }
       }.merge(options[:settings] || {})
+
+      # synonyms
       synonyms = options[:synonyms] || []
       if synonyms.any?
         settings[:analysis][:filter][:searchkick_synonym] = {
@@ -103,6 +123,8 @@ module Searchkick
       end
       mapping = {}
+
+      # conversions
       if options[:conversions]
         mapping[:conversions] = {
           type: "nested",
@@ -113,6 +135,18 @@ module Searchkick
         }
       end
+      # typeahead
+      (options[:typeahead] || []).each do |field|
+        mapping[field] = {
+          type: "multi_field",
+          fields: {
+            field => {type: "string", index: "not_analyzed"},
+            "analyzed" => {type: "string", index: "analyzed"},
+            "typeahead" => {type: "string", index: "analyzed", analyzer: "searchkick_typeahead_index"}
+          }
+        }
+      end
+
       mappings = {
         document_type.to_sym => {
           properties: mapping,
@@ -3,7 +3,20 @@ module Searchkick
     def search(term, options = {})
       term = term.to_s
-      fields = options[:fields] ? options[:fields].map{|f| "#{f}.analyzed" } : ["_all"]
+      fields =
+        if options[:fields]
+          if options[:typeahead]
+            options[:fields].map{|f| "#{f}.typeahead" }
+          else
+            options[:fields].map{|f| "#{f}.analyzed" }
+          end
+        else
+          if options[:typeahead]
+            (@searchkick_options[:typeahead] || []).map{|f| "#{f}.typeahead" }
+          else
+            ["_all"]
+          end
+        end
       operator = options[:partial] ? "or" : "and"
       load = options[:load].nil? ? true : options[:load]
       load = (options[:include] ? {include: options[:include]} : true) if load
@@ -22,18 +35,22 @@ module Searchkick
               query do
                 boolean do
                   must do
-                    dis_max do
-                      query do
-                        match fields, term, boost: 10, operator: operator, analyzer: "searchkick_search"
-                      end
-                      query do
-                        match fields, term, boost: 10, operator: operator, analyzer: "searchkick_search2"
-                      end
-                      query do
-                        match fields, term, use_dis_max: false, fuzziness: 1, max_expansions: 1, operator: operator, analyzer: "searchkick_search"
-                      end
-                      query do
-                        match fields, term, use_dis_max: false, fuzziness: 1, max_expansions: 1, operator: operator, analyzer: "searchkick_search2"
+                    if options[:typeahead]
+                      match fields, term, analyzer: "searchkick_typeahead_search"
+                    else
+                      dis_max do
+                        query do
+                          match fields, term, boost: 10, operator: operator, analyzer: "searchkick_search"
+                        end
+                        query do
+                          match fields, term, boost: 10, operator: operator, analyzer: "searchkick_search2"
+                        end
+                        query do
+                          match fields, term, use_dis_max: false, fuzziness: 1, max_expansions: 1, operator: operator, analyzer: "searchkick_search"
+                        end
+                        query do
+                          match fields, term, use_dis_max: false, fuzziness: 1, max_expansions: 1, operator: operator, analyzer: "searchkick_search2"
+                        end
                       end
                     end
                   end
@@ -110,4 +110,21 @@ class TestMatch &lt; Minitest::Unit::TestCase
     assert_search "almondmilks", ["Almond Milk"]
   end
+  # typeahead
+
+  def test_typeahead
+    store_names ["Hummus"]
+    assert_search "hum", ["Hummus"], typeahead: true
+  end
+
+  def test_typeahead_two_words
+    store_names ["Organic Hummus"]
+    assert_search "hum", [], typeahead: true
+  end
+
+  def test_typeahead_fields
+    store_names ["Hummus"]
+    assert_search "hum", ["Hummus"], typeahead: true, fields: [:name]
+  end
+
 end
@@ -47,7 +47,8 @@ class Product &lt; ActiveRecord::Base
       ["qtip", "cotton swab"],
       ["burger", "hamburger"],
       ["bandaid", "bandag"]
-    ]
+    ],
+    typeahead: [:name]
   attr_accessor :conversions, :user_ids
	@@ -17,6 +17,7 @@ Plus:		@@ -17,6 +17,7 @@ Plus:
17	- query like SQL - no need to learn a new query language	17	- query like SQL - no need to learn a new query language
18	- reindex without downtime	18	- reindex without downtime
19	- easily personalize results for each user [master branch]	19	- easily personalize results for each user [master branch]
		20	+- typeahead / autocomplete [master branch]
20		21
21	:tangerine: Battle-tested at [Instacart](https://www.instacart.com)	22	:tangerine: Battle-tested at [Instacart](https://www.instacart.com)
22		23
	@@ -130,6 +131,22 @@ To change this, use:		@@ -130,6 +131,22 @@ To change this, use:
130	Product.search "fresh honey", partial: true # fresh OR honey	131	Product.search "fresh honey", partial: true # fresh OR honey
131	```	132	```
132		133
		134	+### Typeahead / Autocomplete [master branch]
		135	+
		136	+You must specify which fields use this feature since this can increase the index size significantly. Don’t worry - this gives you blazing faster queries.
		137	+
		138	+```ruby
		139	+class Product < ActiveRecord::Base
		140	+ searchkick typeahead: [:name]
		141	+end
		142	+```
		143	+
		144	+Reindex and search with:
		145	+
		146	+```ruby
		147	+Product.search "puddi", typeahead: true
		148	+```
		149	+
133	### Synonyms	150	### Synonyms
134		151
135	```ruby	152	```ruby
	@@ -349,11 +366,10 @@ end		@@ -349,11 +366,10 @@ end
349		366
350	## Thanks	367	## Thanks
351		368
352	-Thanks to Karel Minarik for [Tire](https://github.com/karmi/tire) and Jaroslav Kalistsuk for [zero downtime reindexing](https://gist.github.com/jarosan/3124884).	369	+Thanks to Karel Minarik for [Tire](https://github.com/karmi/tire), Jaroslav Kalistsuk for [zero downtime reindexing](https://gist.github.com/jarosan/3124884), and Alex Leschenko for [Elasticsearch autocomplete](https://github.com/leschenko/elasticsearch_autocomplete).
353		370
354	## TODO	371	## TODO
355		372
356	-- Custom results for each user
357	- Make Searchkick work with any language	373	- Make Searchkick work with any language
358	- Built-in synonyms from WordNet	374	- Built-in synonyms from WordNet
359	- [Did you mean?](http://www.elasticsearch.org/guide/reference/api/search/suggest/)	375	- [Did you mean?](http://www.elasticsearch.org/guide/reference/api/search/suggest/)
	@@ -68,6 +68,17 @@ module Searchkick		@@ -68,6 +68,17 @@ module Searchkick
68	type: "custom",	68	type: "custom",
69	tokenizer: "standard",	69	tokenizer: "standard",
70	filter: ["standard", "lowercase", "asciifolding", "stop", "snowball"]	70	filter: ["standard", "lowercase", "asciifolding", "stop", "snowball"]
		71	+ },
		72	+ # https://github.com/leschenko/elasticsearch_autocomplete/blob/master/lib/elasticsearch_autocomplete/analyzers.rb
		73	+ searchkick_typeahead_index: {
		74	+ type: "custom",
		75	+ tokenizer: "searchkick_typeahead_ngram",
		76	+ filter: ["lowercase", "asciifolding"]
		77	+ },
		78	+ searchkick_typeahead_search: {
		79	+ type: "custom",
		80	+ tokenizer: "keyword",
		81	+ filter: ["lowercase", "asciifolding"]
71	}	82	}
72	},	83	},
73	filter: {	84	filter: {
	@@ -82,9 +93,18 @@ module Searchkick		@@ -82,9 +93,18 @@ module Searchkick
82	output_unigrams: false,	93	output_unigrams: false,
83	output_unigrams_if_no_shingles: true	94	output_unigrams_if_no_shingles: true
84	}	95	}
		96	+ },
		97	+ tokenizer: {
		98	+ searchkick_typeahead_ngram: {
		99	+ type: "edgeNGram",
		100	+ min_gram: 1,
		101	+ max_gram: 50
		102	+ }
85	}	103	}
86	}	104	}
87	}.merge(options[:settings] \|\| {})	105	}.merge(options[:settings] \|\| {})
		106	+
		107	+ # synonyms
88	synonyms = options[:synonyms] \|\| []	108	synonyms = options[:synonyms] \|\| []
89	if synonyms.any?	109	if synonyms.any?
90	settings[:analysis][:filter][:searchkick_synonym] = {	110	settings[:analysis][:filter][:searchkick_synonym] = {
	@@ -103,6 +123,8 @@ module Searchkick		@@ -103,6 +123,8 @@ module Searchkick
103	end	123	end
104		124
105	mapping = {}	125	mapping = {}
		126	+
		127	+ # conversions
106	if options[:conversions]	128	if options[:conversions]
107	mapping[:conversions] = {	129	mapping[:conversions] = {
108	type: "nested",	130	type: "nested",
	@@ -113,6 +135,18 @@ module Searchkick		@@ -113,6 +135,18 @@ module Searchkick
113	}	135	}
114	end	136	end
115		137
		138	+ # typeahead
		139	+ (options[:typeahead] \|\| []).each do \|field\|
		140	+ mapping[field] = {
		141	+ type: "multi_field",
		142	+ fields: {
		143	+ field => {type: "string", index: "not_analyzed"},
		144	+ "analyzed" => {type: "string", index: "analyzed"},
		145	+ "typeahead" => {type: "string", index: "analyzed", analyzer: "searchkick_typeahead_index"}
		146	+ }
		147	+ }
		148	+ end
		149	+
116	mappings = {	150	mappings = {
117	document_type.to_sym => {	151	document_type.to_sym => {
118	properties: mapping,	152	properties: mapping,
	@@ -3,7 +3,20 @@ module Searchkick		@@ -3,7 +3,20 @@ module Searchkick
3		3
4	def search(term, options = {})	4	def search(term, options = {})
5	term = term.to_s	5	term = term.to_s
6	- fields = options[:fields] ? options[:fields].map{\|f\| "#{f}.analyzed" } : ["_all"]	6	+ fields =
		7	+ if options[:fields]
		8	+ if options[:typeahead]
		9	+ options[:fields].map{\|f\| "#{f}.typeahead" }
		10	+ else
		11	+ options[:fields].map{\|f\| "#{f}.analyzed" }
		12	+ end
		13	+ else
		14	+ if options[:typeahead]
		15	+ (@searchkick_options[:typeahead] \|\| []).map{\|f\| "#{f}.typeahead" }
		16	+ else
		17	+ ["_all"]
		18	+ end
		19	+ end
7	operator = options[:partial] ? "or" : "and"	20	operator = options[:partial] ? "or" : "and"
8	load = options[:load].nil? ? true : options[:load]	21	load = options[:load].nil? ? true : options[:load]
9	load = (options[:include] ? {include: options[:include]} : true) if load	22	load = (options[:include] ? {include: options[:include]} : true) if load
	@@ -22,18 +35,22 @@ module Searchkick		@@ -22,18 +35,22 @@ module Searchkick
22	query do	35	query do
23	boolean do	36	boolean do
24	must do	37	must do
25	- dis_max do
26	- query do
27	- match fields, term, boost: 10, operator: operator, analyzer: "searchkick_search"
28	- end
29	- query do
30	- match fields, term, boost: 10, operator: operator, analyzer: "searchkick_search2"
31	- end
32	- query do
33	- match fields, term, use_dis_max: false, fuzziness: 1, max_expansions: 1, operator: operator, analyzer: "searchkick_search"
34	- end
35	- query do
36	- match fields, term, use_dis_max: false, fuzziness: 1, max_expansions: 1, operator: operator, analyzer: "searchkick_search2"	38	+ if options[:typeahead]
		39	+ match fields, term, analyzer: "searchkick_typeahead_search"
		40	+ else
		41	+ dis_max do
		42	+ query do
		43	+ match fields, term, boost: 10, operator: operator, analyzer: "searchkick_search"
		44	+ end
		45	+ query do
		46	+ match fields, term, boost: 10, operator: operator, analyzer: "searchkick_search2"
		47	+ end
		48	+ query do
		49	+ match fields, term, use_dis_max: false, fuzziness: 1, max_expansions: 1, operator: operator, analyzer: "searchkick_search"
		50	+ end
		51	+ query do
		52	+ match fields, term, use_dis_max: false, fuzziness: 1, max_expansions: 1, operator: operator, analyzer: "searchkick_search2"
		53	+ end
37	end	54	end
38	end	55	end
39	end	56	end
	@@ -110,4 +110,21 @@ class TestMatch < Minitest::Unit::TestCase		@@ -110,4 +110,21 @@ class TestMatch < Minitest::Unit::TestCase
110	assert_search "almondmilks", ["Almond Milk"]	110	assert_search "almondmilks", ["Almond Milk"]
111	end	111	end
112		112
		113	+ # typeahead
		114	+
		115	+ def test_typeahead
		116	+ store_names ["Hummus"]
		117	+ assert_search "hum", ["Hummus"], typeahead: true
		118	+ end
		119	+
		120	+ def test_typeahead_two_words
		121	+ store_names ["Organic Hummus"]
		122	+ assert_search "hum", [], typeahead: true
		123	+ end
		124	+
		125	+ def test_typeahead_fields
		126	+ store_names ["Hummus"]
		127	+ assert_search "hum", ["Hummus"], typeahead: true, fields: [:name]
		128	+ end
		129	+
113	end	130	end
	@@ -47,7 +47,8 @@ class Product < ActiveRecord::Base		@@ -47,7 +47,8 @@ class Product < ActiveRecord::Base
47	["qtip", "cotton swab"],	47	["qtip", "cotton swab"],
48	["burger", "hamburger"],	48	["burger", "hamburger"],
49	["bandaid", "bandag"]	49	["bandaid", "bandag"]
50	- ]	50	+ ],
		51	+ typeahead: [:name]
51		52
52	attr_accessor :conversions, :user_ids	53	attr_accessor :conversions, :user_ids
53		54