Added support for Chinese - closes #924

Andrew
1 parent 7ebb36f5
Showing 5 changed files with 73 additions and 1 deletions Show diff stats
CHANGELOG.md
README.md
lib/searchkick/index_options.rb
test/language_test.rb
test/test_helper.rb
 ## 3.0.0 [unreleased]
  
 - Better performance
+- Added support for Chinese
  
 Breaking changes
  
@@ -291,6 +291,14 @@ end
  
 [See the list of stemmers](https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-stemmer-tokenfilter.html)
  
+For Chinese, install the [elasticsearch-analysis-ik plugin](https://github.com/medcl/elasticsearch-analysis-ik) and use:
+
+```ruby
+class Product < ApplicationRecord
+  searchkick language: "chinese"
+end
+```
+
 ### Synonyms
  
 ```ruby
@@ -143,6 +143,22 @@ module Searchkick
           }
         }
  
+        if language == "chinese"
+          settings[:analysis][:analyzer].merge!(
+            default_analyzer => {
+              type: "ik_smart"
+            },
+            searchkick_search: {
+              type: "ik_smart"
+            },
+            searchkick_search2: {
+              type: "ik_max_word"
+            }
+          )
+
+          settings[:analysis][:filter].delete(:searchkick_stemmer)
+        end
+
         if Searchkick.env == "test"
           settings[:number_of_shards] = 1
           settings[:number_of_replicas] = 0
@@ -0,0 +1,15 @@
+require_relative "test_helper"
+
+class LanguageTest < Minitest::Test
+  def setup
+    Song.destroy_all
+  end
+
+  def test_chinese
+    skip unless ENV["CHINESE"]
+    store_names ["中华人民共和国国歌"], Song
+    assert_search "中华人民共和国", ["中华人民共和国国歌"], {}, Song
+    assert_search "国歌", ["中华人民共和国国歌"], {}, Song
+    assert_search "人", [], {}, Song
+  end
+end
@@ -117,6 +117,12 @@ if defined?(Mongoid)
  
     field :name
   end
+
+  class Song
+    include Mongoid::Document
+
+    field :name
+  end
 elsif defined?(NoBrainer)
   NoBrainer.configure do |config|
     config.app_name = :searchkick
@@ -184,6 +190,13 @@ elsif defined?(NoBrainer)
     field :id,   type: String
     field :name, type: String
   end
+
+  class Song
+    include NoBrainer::Document
+
+    field :id,   type: Object
+    field :name, type: String
+  end
 elsif defined?(Cequel)
   cequel =
     Cequel.connect(
@@ -272,7 +285,14 @@ elsif defined?(Cequel)
     column :name, :text
   end
  
-  [Product, Store, Region, Speaker, Animal].each(&:synchronize_schema)
+  class Song
+    include Cequel::Record
+
+    key :id, :timeuuid, auto: true
+    column :name, :text
+  end
+
+  [Product, Store, Region, Speaker, Animal, Sku, Song].each(&:synchronize_schema)
 else
   require "active_record"
  
@@ -363,6 +383,10 @@ else
     t.string :name
   end
  
+  ActiveRecord::Migration.create_table :songs do |t|
+    t.string :name
+  end
+
   class Product < ActiveRecord::Base
     belongs_to :store
   end
@@ -388,6 +412,9 @@ else
  
   class Sku < ActiveRecord::Base
   end
+
+  class Song < ActiveRecord::Base
+  end
 end
  
 class Product
@@ -508,6 +535,10 @@ class Sku
   searchkick callbacks: defined?(ActiveJob) ? :async : true
 end
  
+class Song
+  searchkick language: "chinese"
+end
+
 Product.searchkick_index.delete if Product.searchkick_index.exists?
 Product.reindex
 Product.reindex # run twice for both index paths
@@ -517,6 +548,7 @@ Store.reindex
 Animal.reindex
 Speaker.reindex
 Region.reindex
+Song.reindex
  
 class Minitest::Test
   def setup
1	1	## 3.0.0 [unreleased]
2	2
3	3	- Better performance
	4	+- Added support for Chinese
4	5
5	6	Breaking changes
6	7
...	...
...	...	@@ -291,6 +291,14 @@ end
291	291
292	292	[See the list of stemmers](https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-stemmer-tokenfilter.html)
293	293
	294	+For Chinese, install the [elasticsearch-analysis-ik plugin](https://github.com/medcl/elasticsearch-analysis-ik) and use:
	295	+
	296	+```ruby
	297	+class Product < ApplicationRecord
	298	+ searchkick language: "chinese"
	299	+end
	300	+```
	301	+
294	302	### Synonyms
295	303
296	304	```ruby
...	...
...	...	@@ -143,6 +143,22 @@ module Searchkick
143	143	}
144	144	}
145	145
	146	+ if language == "chinese"
	147	+ settings[:analysis][:analyzer].merge!(
	148	+ default_analyzer => {
	149	+ type: "ik_smart"
	150	+ },
	151	+ searchkick_search: {
	152	+ type: "ik_smart"
	153	+ },
	154	+ searchkick_search2: {
	155	+ type: "ik_max_word"
	156	+ }
	157	+ )
	158	+
	159	+ settings[:analysis][:filter].delete(:searchkick_stemmer)
	160	+ end
	161	+
146	162	if Searchkick.env == "test"
147	163	settings[:number_of_shards] = 1
148	164	settings[:number_of_replicas] = 0
...	...
...	...	@@ -0,0 +1,15 @@
	1	+require_relative "test_helper"
	2	+
	3	+class LanguageTest < Minitest::Test
	4	+ def setup
	5	+ Song.destroy_all
	6	+ end
	7	+
	8	+ def test_chinese
	9	+ skip unless ENV["CHINESE"]
	10	+ store_names ["中华人民共和国国歌"], Song
	11	+ assert_search "中华人民共和国", ["中华人民共和国国歌"], {}, Song
	12	+ assert_search "国歌", ["中华人民共和国国歌"], {}, Song
	13	+ assert_search "人", [], {}, Song
	14	+ end
	15	+end
...	...
...	...	@@ -117,6 +117,12 @@ if defined?(Mongoid)
117	117
118	118	field :name
119	119	end
	120	+
	121	+ class Song
	122	+ include Mongoid::Document
	123	+
	124	+ field :name
	125	+ end
120	126	elsif defined?(NoBrainer)
121	127	NoBrainer.configure do \|config\|
122	128	config.app_name = :searchkick
...	...	@@ -184,6 +190,13 @@ elsif defined?(NoBrainer)
184	190	field :id, type: String
185	191	field :name, type: String
186	192	end
	193	+
	194	+ class Song
	195	+ include NoBrainer::Document
	196	+
	197	+ field :id, type: Object
	198	+ field :name, type: String
	199	+ end
187	200	elsif defined?(Cequel)
188	201	cequel =
189	202	Cequel.connect(
...	...	@@ -272,7 +285,14 @@ elsif defined?(Cequel)
272	285	column :name, :text
273	286	end
274	287
275		- [Product, Store, Region, Speaker, Animal].each(&:synchronize_schema)
	288	+ class Song
	289	+ include Cequel::Record
	290	+
	291	+ key :id, :timeuuid, auto: true
	292	+ column :name, :text
	293	+ end
	294	+
	295	+ [Product, Store, Region, Speaker, Animal, Sku, Song].each(&:synchronize_schema)
276	296	else
277	297	require "active_record"
278	298
...	...	@@ -363,6 +383,10 @@ else
363	383	t.string :name
364	384	end
365	385
	386	+ ActiveRecord::Migration.create_table :songs do \|t\|
	387	+ t.string :name
	388	+ end
	389	+
366	390	class Product < ActiveRecord::Base
367	391	belongs_to :store
368	392	end
...	...	@@ -388,6 +412,9 @@ else
388	412
389	413	class Sku < ActiveRecord::Base
390	414	end
	415	+
	416	+ class Song < ActiveRecord::Base
	417	+ end
391	418	end
392	419
393	420	class Product
...	...	@@ -508,6 +535,10 @@ class Sku
508	535	searchkick callbacks: defined?(ActiveJob) ? :async : true
509	536	end
510	537
	538	+class Song
	539	+ searchkick language: "chinese"
	540	+end
	541	+
511	542	Product.searchkick_index.delete if Product.searchkick_index.exists?
512	543	Product.reindex
513	544	Product.reindex # run twice for both index paths
...	...	@@ -517,6 +548,7 @@ Store.reindex
517	548	Animal.reindex
518	549	Speaker.reindex
519	550	Region.reindex
	551	+Song.reindex
520	552
521	553	class Minitest::Test
522	554	def setup
...	...