Commit 2b2682d3690acb721a85c9cac50fe6ad9fcdc8b9
1 parent
41dd14bb
Exists in
master
and in
21 other branches
No more tire
Showing
6 changed files
with
138 additions
and
128 deletions
Show diff stats
lib/searchkick.rb
| @@ -5,65 +5,78 @@ require "searchkick/tasks" | @@ -5,65 +5,78 @@ require "searchkick/tasks" | ||
| 5 | require "tire" | 5 | require "tire" |
| 6 | 6 | ||
| 7 | module Searchkick | 7 | module Searchkick |
| 8 | - # TODO fix this monstrosity | ||
| 9 | - # TODO add custom synonyms | ||
| 10 | - def self.settings(options = {}) | ||
| 11 | - synonyms = options[:synonyms] || [] | ||
| 12 | - settings = { | ||
| 13 | - analysis: { | ||
| 14 | - analyzer: { | ||
| 15 | - searchkick_keyword: { | ||
| 16 | - type: "custom", | ||
| 17 | - tokenizer: "keyword", | ||
| 18 | - filter: ["lowercase", "snowball"] | 8 | + module Model |
| 9 | + def searchkick(field, options = {}) | ||
| 10 | + custom_settings = { | ||
| 11 | + analysis: { | ||
| 12 | + analyzer: { | ||
| 13 | + searchkick_keyword: { | ||
| 14 | + type: "custom", | ||
| 15 | + tokenizer: "keyword", | ||
| 16 | + filter: ["lowercase", "snowball"] | ||
| 17 | + }, | ||
| 18 | + searchkick: { | ||
| 19 | + type: "custom", | ||
| 20 | + tokenizer: "standard", | ||
| 21 | + # synonym should come last, after stemming and shingle | ||
| 22 | + # shingle must come before snowball | ||
| 23 | + filter: ["standard", "lowercase", "asciifolding", "stop", "snowball", "searchkick_index_shingle"] | ||
| 24 | + }, | ||
| 25 | + searchkick_search: { | ||
| 26 | + type: "custom", | ||
| 27 | + tokenizer: "standard", | ||
| 28 | + filter: ["standard", "lowercase", "asciifolding", "stop", "snowball", "searchkick_search_shingle"] | ||
| 29 | + }, | ||
| 30 | + searchkick_search2: { | ||
| 31 | + type: "custom", | ||
| 32 | + tokenizer: "standard", | ||
| 33 | + filter: ["standard", "lowercase", "asciifolding", "stop", "snowball"] #, "searchkick_search_shingle"] | ||
| 34 | + } | ||
| 19 | }, | 35 | }, |
| 20 | - searchkick: { | ||
| 21 | - type: "custom", | ||
| 22 | - tokenizer: "standard", | ||
| 23 | - # synonym should come last, after stemming and shingle | ||
| 24 | - # shingle must come before snowball | ||
| 25 | - filter: ["standard", "lowercase", "asciifolding", "stop", "snowball", "searchkick_index_shingle"] | ||
| 26 | - }, | ||
| 27 | - searchkick_search: { | ||
| 28 | - type: "custom", | ||
| 29 | - tokenizer: "standard", | ||
| 30 | - filter: ["standard", "lowercase", "asciifolding", "stop", "snowball", "searchkick_search_shingle"] | ||
| 31 | - }, | ||
| 32 | - searchkick_search2: { | ||
| 33 | - type: "custom", | ||
| 34 | - tokenizer: "standard", | ||
| 35 | - filter: ["standard", "lowercase", "asciifolding", "stop", "snowball"] #, "searchkick_search_shingle"] | ||
| 36 | - } | ||
| 37 | - }, | ||
| 38 | - filter: { | ||
| 39 | - searchkick_index_shingle: { | ||
| 40 | - type: "shingle", | ||
| 41 | - token_separator: "" | ||
| 42 | - }, | ||
| 43 | - # lucky find http://web.archiveorange.com/archive/v/AAfXfQ17f57FcRINsof7 | ||
| 44 | - searchkick_search_shingle: { | ||
| 45 | - type: "shingle", | ||
| 46 | - token_separator: "", | ||
| 47 | - output_unigrams: false, | ||
| 48 | - output_unigrams_if_no_shingles: true | 36 | + filter: { |
| 37 | + searchkick_index_shingle: { | ||
| 38 | + type: "shingle", | ||
| 39 | + token_separator: "" | ||
| 40 | + }, | ||
| 41 | + # lucky find http://web.archiveorange.com/archive/v/AAfXfQ17f57FcRINsof7 | ||
| 42 | + searchkick_search_shingle: { | ||
| 43 | + type: "shingle", | ||
| 44 | + token_separator: "", | ||
| 45 | + output_unigrams: false, | ||
| 46 | + output_unigrams_if_no_shingles: true | ||
| 47 | + } | ||
| 49 | } | 48 | } |
| 50 | } | 49 | } |
| 51 | - } | ||
| 52 | - } | ||
| 53 | - if synonyms.any? | ||
| 54 | - settings[:analysis][:filter][:searchkick_synonym] = { | ||
| 55 | - type: "synonym", | ||
| 56 | - ignore_case: true, | ||
| 57 | - synonyms: synonyms | ||
| 58 | - } | ||
| 59 | - settings[:analysis][:analyzer][:searchkick][:filter] << "searchkick_synonym" | ||
| 60 | - settings[:analysis][:analyzer][:searchkick_search][:filter].insert(-2, "searchkick_synonym") | ||
| 61 | - settings[:analysis][:analyzer][:searchkick_search][:filter] << "searchkick_synonym" | ||
| 62 | - settings[:analysis][:analyzer][:searchkick_search2][:filter] << "searchkick_synonym" | 50 | + }.merge(options[:settings] || {}) |
| 51 | + synonyms = options[:synonyms] || [] | ||
| 52 | + if synonyms.any? | ||
| 53 | + custom_settings[:analysis][:filter][:searchkick_synonym] = { | ||
| 54 | + type: "synonym", | ||
| 55 | + ignore_case: true, | ||
| 56 | + synonyms: synonyms | ||
| 57 | + } | ||
| 58 | + custom_settings[:analysis][:analyzer][:searchkick][:filter] << "searchkick_synonym" | ||
| 59 | + custom_settings[:analysis][:analyzer][:searchkick_search][:filter].insert(-2, "searchkick_synonym") | ||
| 60 | + custom_settings[:analysis][:analyzer][:searchkick_search][:filter] << "searchkick_synonym" | ||
| 61 | + custom_settings[:analysis][:analyzer][:searchkick_search2][:filter] << "searchkick_synonym" | ||
| 62 | + end | ||
| 63 | + | ||
| 64 | + class_eval do | ||
| 65 | + extend Searchkick::Search | ||
| 66 | + extend Searchkick::Reindex | ||
| 67 | + include Tire::Model::Search | ||
| 68 | + include Tire::Model::Callbacks | ||
| 69 | + | ||
| 70 | + tire do | ||
| 71 | + settings custom_settings | ||
| 72 | + mapping do | ||
| 73 | + indexes field, analyzer: "searchkick" | ||
| 74 | + end | ||
| 75 | + end | ||
| 76 | + end | ||
| 63 | end | 77 | end |
| 64 | - settings | ||
| 65 | end | 78 | end |
| 66 | end | 79 | end |
| 67 | 80 | ||
| 68 | -Tire::Model::Search::ClassMethodsProxy.send :include, Searchkick::Reindex | ||
| 69 | -Tire::Search::Search.send :include, Searchkick::Search | 81 | +require "active_record" |
| 82 | +ActiveRecord::Base.send(:extend, Searchkick::Model) if defined?(ActiveRecord) |
lib/searchkick/reindex.rb
| @@ -3,13 +3,13 @@ module Searchkick | @@ -3,13 +3,13 @@ module Searchkick | ||
| 3 | 3 | ||
| 4 | # https://gist.github.com/jarosan/3124884 | 4 | # https://gist.github.com/jarosan/3124884 |
| 5 | def reindex | 5 | def reindex |
| 6 | - alias_name = klass.tire.index.name | 6 | + alias_name = tire.index.name |
| 7 | new_index = alias_name + "_" + Time.now.strftime("%Y%m%d%H%M%S") | 7 | new_index = alias_name + "_" + Time.now.strftime("%Y%m%d%H%M%S") |
| 8 | 8 | ||
| 9 | # Rake::Task["tire:import"].invoke | 9 | # Rake::Task["tire:import"].invoke |
| 10 | index = Tire::Index.new(new_index) | 10 | index = Tire::Index.new(new_index) |
| 11 | - Tire::Tasks::Import.create_index(index, klass) | ||
| 12 | - scope = klass.respond_to?(:tire_import) ? klass.tire_import : klass | 11 | + Tire::Tasks::Import.create_index(index, self) |
| 12 | + scope = respond_to?(:searchkick_import) ? searchkick_import : self | ||
| 13 | scope.find_in_batches do |batch| | 13 | scope.find_in_batches do |batch| |
| 14 | index.import batch | 14 | index.import batch |
| 15 | end | 15 | end |
| @@ -37,6 +37,7 @@ module Searchkick | @@ -37,6 +37,7 @@ module Searchkick | ||
| 37 | end | 37 | end |
| 38 | 38 | ||
| 39 | puts "[IMPORT] Saved alias #{alias_name} pointing to #{new_index}" | 39 | puts "[IMPORT] Saved alias #{alias_name} pointing to #{new_index}" |
| 40 | + true | ||
| 40 | end | 41 | end |
| 41 | 42 | ||
| 42 | end | 43 | end |
lib/searchkick/search.rb
| 1 | module Searchkick | 1 | module Searchkick |
| 2 | # can't check mapping for conversions since the new index may not be built | 2 | # can't check mapping for conversions since the new index may not be built |
| 3 | module Search | 3 | module Search |
| 4 | - def searchkick_query(fields, term, conversions = false) | ||
| 5 | - query do | ||
| 6 | - boolean do | ||
| 7 | - must do | ||
| 8 | - dis_max do | ||
| 9 | - query do | ||
| 10 | - match fields, term, boost: 10, operator: "and", analyzer: "searchkick_search" | ||
| 11 | - end | ||
| 12 | - query do | ||
| 13 | - match fields, term, boost: 10, operator: "and", analyzer: "searchkick_search2" | ||
| 14 | - end | ||
| 15 | - query do | ||
| 16 | - match fields, term, use_dis_max: false, fuzziness: 0.7, max_expansions: 1, prefix_length: 1, operator: "and", analyzer: "searchkick_search" | ||
| 17 | - end | ||
| 18 | - query do | ||
| 19 | - match fields, term, use_dis_max: false, fuzziness: 0.7, max_expansions: 1, prefix_length: 1, operator: "and", analyzer: "searchkick_search2" | ||
| 20 | - end | ||
| 21 | - end | ||
| 22 | - end | ||
| 23 | - if conversions | ||
| 24 | - should do | ||
| 25 | - nested path: "conversions", score_mode: "total" do | 4 | + def search(term, conversions = false) |
| 5 | + fields = ["name"] | ||
| 6 | + tire.search do | ||
| 7 | + query do | ||
| 8 | + boolean do | ||
| 9 | + must do | ||
| 10 | + dis_max do | ||
| 11 | + query do | ||
| 12 | + match fields, term, boost: 10, operator: "and", analyzer: "searchkick_search" | ||
| 13 | + end | ||
| 14 | + query do | ||
| 15 | + match fields, term, boost: 10, operator: "and", analyzer: "searchkick_search2" | ||
| 16 | + end | ||
| 17 | + query do | ||
| 18 | + match fields, term, use_dis_max: false, fuzziness: 0.7, max_expansions: 1, prefix_length: 1, operator: "and", analyzer: "searchkick_search" | ||
| 19 | + end | ||
| 26 | query do | 20 | query do |
| 27 | - custom_score script: "log(doc['count'].value)" do | ||
| 28 | - match "query", term | ||
| 29 | - end | 21 | + match fields, term, use_dis_max: false, fuzziness: 0.7, max_expansions: 1, prefix_length: 1, operator: "and", analyzer: "searchkick_search2" |
| 30 | end | 22 | end |
| 31 | end | 23 | end |
| 32 | end | 24 | end |
| 25 | + # if conversions | ||
| 26 | + # should do | ||
| 27 | + # nested path: "conversions", score_mode: "total" do | ||
| 28 | + # query do | ||
| 29 | + # custom_score script: "log(doc['count'].value)" do | ||
| 30 | + # match "query", term | ||
| 31 | + # end | ||
| 32 | + # end | ||
| 33 | + # end | ||
| 34 | + # end | ||
| 35 | + # end | ||
| 33 | end | 36 | end |
| 34 | end | 37 | end |
| 35 | end | 38 | end |
searchkick.gemspec
| @@ -23,4 +23,6 @@ Gem::Specification.new do |spec| | @@ -23,4 +23,6 @@ Gem::Specification.new do |spec| | ||
| 23 | spec.add_development_dependency "bundler", "~> 1.3" | 23 | spec.add_development_dependency "bundler", "~> 1.3" |
| 24 | spec.add_development_dependency "rake" | 24 | spec.add_development_dependency "rake" |
| 25 | spec.add_development_dependency "minitest" | 25 | spec.add_development_dependency "minitest" |
| 26 | + spec.add_development_dependency "activerecord" | ||
| 27 | + spec.add_development_dependency "pg" | ||
| 26 | end | 28 | end |
test/searchkick_test.rb
| 1 | require "test_helper" | 1 | require "test_helper" |
| 2 | 2 | ||
| 3 | +class Product < ActiveRecord::Base | ||
| 4 | + searchkick :name, synonyms: [ | ||
| 5 | + "clorox => bleach", | ||
| 6 | + "saranwrap => plastic wrap", | ||
| 7 | + "scallion => green onion", | ||
| 8 | + "qtip => cotton swab", | ||
| 9 | + "burger => hamburger", | ||
| 10 | + "bandaid => bandag" | ||
| 11 | + ], settings: {number_of_shards: 1} | ||
| 12 | +end | ||
| 13 | + | ||
| 3 | class TestSearchkick < Minitest::Unit::TestCase | 14 | class TestSearchkick < Minitest::Unit::TestCase |
| 4 | 15 | ||
| 5 | def setup | 16 | def setup |
| 6 | - $index = Tire::Index.new("products") | ||
| 7 | - $index.delete | ||
| 8 | - synonyms = [ | ||
| 9 | - "clorox => bleach", | ||
| 10 | - "saranwrap => plastic wrap", | ||
| 11 | - "scallion => green onion", | ||
| 12 | - "qtip => cotton swab", | ||
| 13 | - "burger => hamburger", | ||
| 14 | - "bandaid => bandag" | ||
| 15 | - ] | ||
| 16 | - index_options = { | ||
| 17 | - settings: Searchkick.settings(synonyms: synonyms).merge(number_of_shards: 1), | ||
| 18 | - mappings: { | ||
| 19 | - document: { | ||
| 20 | - properties: { | ||
| 21 | - name: { | ||
| 22 | - type: "string", | ||
| 23 | - analyzer: "searchkick" | ||
| 24 | - }, | ||
| 25 | - conversions: { | ||
| 26 | - type: "nested", | ||
| 27 | - properties: { | ||
| 28 | - query: { | ||
| 29 | - type: "string", | ||
| 30 | - analyzer: "searchkick_keyword" | ||
| 31 | - }, | ||
| 32 | - count: { | ||
| 33 | - type: "integer" | ||
| 34 | - } | ||
| 35 | - } | ||
| 36 | - } | ||
| 37 | - } | ||
| 38 | - } | ||
| 39 | - } | ||
| 40 | - } | ||
| 41 | - $index.create index_options | 17 | + Product.index.delete |
| 18 | + Product.create_elasticsearch_index | ||
| 19 | + end | ||
| 20 | + | ||
| 21 | + def test_reindex | ||
| 22 | + assert Product.reindex | ||
| 42 | end | 23 | end |
| 43 | 24 | ||
| 44 | # exact | 25 | # exact |
| @@ -206,9 +187,9 @@ class TestSearchkick < Minitest::Unit::TestCase | @@ -206,9 +187,9 @@ class TestSearchkick < Minitest::Unit::TestCase | ||
| 206 | 187 | ||
| 207 | def store(documents) | 188 | def store(documents) |
| 208 | documents.each do |document| | 189 | documents.each do |document| |
| 209 | - $index.store document | 190 | + Product.index.store document.merge(_type: "product") |
| 210 | end | 191 | end |
| 211 | - $index.refresh | 192 | + Product.index.refresh |
| 212 | end | 193 | end |
| 213 | 194 | ||
| 214 | def store_names(names) | 195 | def store_names(names) |
| @@ -216,13 +197,7 @@ class TestSearchkick < Minitest::Unit::TestCase | @@ -216,13 +197,7 @@ class TestSearchkick < Minitest::Unit::TestCase | ||
| 216 | end | 197 | end |
| 217 | 198 | ||
| 218 | def assert_search(term, expected) | 199 | def assert_search(term, expected) |
| 219 | - search = | ||
| 220 | - Tire.search "products", type: "document" do | ||
| 221 | - searchkick_query ["name"], term, true | ||
| 222 | - explain true | ||
| 223 | - end | ||
| 224 | - | ||
| 225 | - assert_equal expected, search.results.map(&:name) | 200 | + assert_equal expected, Product.search(term).map(&:name) |
| 226 | end | 201 | end |
| 227 | 202 | ||
| 228 | end | 203 | end |
test/test_helper.rb
| @@ -2,6 +2,22 @@ require "bundler/setup" | @@ -2,6 +2,22 @@ require "bundler/setup" | ||
| 2 | Bundler.require(:default) | 2 | Bundler.require(:default) |
| 3 | require "minitest/autorun" | 3 | require "minitest/autorun" |
| 4 | require "minitest/pride" | 4 | require "minitest/pride" |
| 5 | +require "active_record" | ||
| 6 | + | ||
| 7 | +# for debugging | ||
| 8 | +# ActiveRecord::Base.logger = Logger.new(STDOUT) | ||
| 9 | + | ||
| 10 | +# rails does this in activerecord/lib/active_record/railtie.rb | ||
| 11 | +ActiveRecord::Base.default_timezone = :utc | ||
| 12 | +ActiveRecord::Base.time_zone_aware_attributes = true | ||
| 13 | + | ||
| 14 | +# migrations | ||
| 15 | +ActiveRecord::Base.establish_connection :adapter => "postgresql", :database => "searchkick_test" | ||
| 16 | + | ||
| 17 | +ActiveRecord::Migration.create_table :products, :force => true do |t| | ||
| 18 | + t.string :name | ||
| 19 | + t.timestamps | ||
| 20 | +end | ||
| 5 | 21 | ||
| 6 | File.delete("elasticsearch.log") if File.exists?("elasticsearch.log") | 22 | File.delete("elasticsearch.log") if File.exists?("elasticsearch.log") |
| 7 | Tire.configure { logger "elasticsearch.log", :level => "debug" } | 23 | Tire.configure { logger "elasticsearch.log", :level => "debug" } |