Test mongoid release

Andrew Kane
2 parents 6a71a686 2a43d660
Showing 14 changed files with 225 additions and 83 deletions Show diff stats
README.md
gemfiles/mongoid4.gemfile
lib/searchkick.rb
lib/searchkick/index.rb
lib/searchkick/logger.rb
lib/searchkick/model.rb
lib/searchkick/query.rb
lib/searchkick/reindex.rb
lib/searchkick/results.rb
lib/searchkick/similar.rb
searchkick.gemspec
test/index_test.rb
test/sql_test.rb
test/test_helper.rb
@@ -568,7 +568,13 @@ end
 And use the `query` option to search:
 ```ruby
-Product.search query: {match: {name: "milk"}}
+products = Product.search query: {match: {name: "milk"}}
+```
+
+View the response with:
+
+```ruby
+products.response
 ```
 To keep the mappings and settings generated by Searchkick, use:
@@ -579,7 +585,7 @@ class Product &lt; ActiveRecord::Base
 end
 ```
-## Experimental [master]
+## Experimental
 Modify the query generated by Searchkick.
@@ -3,4 +3,4 @@ source &#39;https://rubygems.org&#39;
 # Specify your gem's dependencies in searchkick.gemspec
 gemspec path: "../"
-gem "mongoid", github: "mongoid/mongoid"
+gem "mongoid", "4.0.0.beta1"
-require "tire"
+require "active_model"
+require "patron"
+require "elasticsearch"
 require "searchkick/version"
+require "searchkick/index"
 require "searchkick/reindex"
 require "searchkick/results"
 require "searchkick/query"
@@ -7,9 +10,15 @@ require &quot;searchkick/search&quot;
 require "searchkick/similar"
 require "searchkick/model"
 require "searchkick/tasks"
-require "searchkick/logger" if defined?(Rails)
+# TODO add logger
+# require "searchkick/logger" if defined?(Rails)
 module Searchkick
+
+  def self.client
+    @client ||= Elasticsearch::Client.new(url: ENV["ELASTICSEARCH_URL"])
+  end
+
   @callbacks = true
   def self.enable_callbacks
@@ -0,0 +1,67 @@
+module Searchkick
+  class Index
+    attr_reader :name
+
+    def initialize(name)
+      @name = name
+    end
+
+    def create(options = {})
+      client.indices.create index: name, body: options
+    end
+
+    def delete
+      client.indices.delete index: name
+    end
+
+    def exists?
+      client.indices.exists index: name
+    end
+
+    def refresh
+      client.indices.refresh index: name
+    end
+
+    def store(record)
+      client.index(
+        index: name,
+        type: record.document_type,
+        id: record.id,
+        body: record.as_indexed_json
+      )
+    end
+
+    def remove(record)
+      client.delete(
+        index: name,
+        type: record.document_type,
+        id: record.id
+      )
+    end
+
+    def import(records)
+      if records.any?
+        client.bulk(
+          index: name,
+          type: records.first.document_type,
+          body: records.map{|r| data = r.as_indexed_json; {index: {_id: data["_id"] || data["id"] || r.id, data: data}} }
+        )
+      end
+    end
+
+    def retrieve(document_type, id)
+      client.get_source(
+        index: name,
+        type: document_type,
+        id: id
+      )
+    end
+
+    protected
+
+    def client
+      Searchkick.client
+    end
+
+  end
+end
@@ -1,19 +0,0 @@
-require "tire/rails/logger"
-require "tire/rails/logger/log_subscriber"
-
-class Tire::Rails::LogSubscriber
-
-  # better output format
-  def search(event)
-    self.class.runtime += event.duration
-    return unless logger.debug?
-
-    payload = event.payload
-
-    name    = "%s (%.1fms)" % [payload[:name], event.duration]
-    query   = payload[:search].to_s
-
-    debug "  #{color(name, YELLOW, true)}  #{query}"
-  end
-
-end
@@ -13,7 +13,7 @@ module Searchkick
         # set index name
         # TODO support proc
         index_name = options[:index_name] || [options[:index_prefix], model_name.plural, searchkick_env].compact.join("_")
-        class_variable_set :@@searchkick_index, Tire::Index.new(index_name)
+        class_variable_set :@@searchkick_index, Searchkick::Index.new(index_name)
         extend Searchkick::Search
         extend Searchkick::Reindex
@@ -45,7 +45,11 @@ module Searchkick
         def reindex
           index = self.class.searchkick_index
           if destroyed? or !should_index?
-            index.remove self
+            begin
+              index.remove self
+            rescue Elasticsearch::Transport::Transport::Errors::NotFound
+              # do nothing
+            end
           else
             index.store self
           end
@@ -55,7 +59,7 @@ module Searchkick
           respond_to?(:to_hash) ? to_hash : serializable_hash
         end
-        def to_indexed_json
+        def as_indexed_json
           source = search_data
           # stringify fields
@@ -115,7 +119,7 @@ module Searchkick
           # p search_data
-          source.to_json
+          source.as_json
         end
         # TODO remove
@@ -35,10 +35,6 @@ module Searchkick
       operator = options[:operator] || (options[:partial] ? "or" : "and")
-      # model and eagar loading
-      load = options[:load].nil? ? true : options[:load]
-      load = (options[:include] ? {include: options[:include]} : true) if load
-
       # pagination
       page = [options[:page].to_i, 1].max
       per_page = (options[:limit] || options[:per_page] || 100000).to_i
@@ -278,18 +274,22 @@ module Searchkick
         end
       end
+      # model and eagar loading
+      load = options[:load].nil? ? true : options[:load]
+
       # An empty array will cause only the _id and _type for each hit to be returned
       # http://www.elasticsearch.org/guide/reference/api/search/fields/
       payload[:fields] = [] if load
-      tire_options = {load: load, size: per_page, from: offset}
       if options[:type] or klass != searchkick_klass
-        tire_options[:type] = [options[:type] || klass].flatten.map(&:document_type)
+        @type = [options[:type] || klass].flatten.map(&:document_type)
       end
-      @search = Tire::Search::Search.new(index_name, tire_options)
       @body = payload
       @facet_limits = facet_limits
+      @page = page
+      @per_page = per_page
+      @load = load
     end
     def searchkick_index
@@ -309,14 +309,18 @@ module Searchkick
     end
     def execute
-      @search.options[:payload] = body
+      params = {
+        index: searchkick_index.name,
+        body: body
+      }
+      params.merge!(type: @type) if @type
       begin
-        response = @search.json
-      rescue Tire::Search::SearchRequestFailed => e
+        response = Searchkick.client.search(params)
+      rescue => e # TODO rescue type
         status_code = e.message[1..3].to_i
         if status_code == 404
           raise "Index missing - run #{searchkick_klass.name}.reindex"
-        elsif status_code == 500 and (e.message.include?("IllegalArgumentException[minimumSimilarity >= 1]") or e.message.include?("No query registered for [multi_match]"))
+        elsif status_code == 500 and (e.message.include?("IllegalArgumentException[minimumSimilarity >= 1]") or e.message.include?("No query registered for [multi_match]") or e.message.include?("[match] query does not support [cutoff_frequency]]"))
           raise "Upgrade Elasticsearch to 0.90.0 or greater"
         else
           raise e
@@ -332,7 +336,13 @@ module Searchkick
         response["facets"][field]["other"] = facet["total"] - facet["terms"].sum{|term| term["count"] }
       end
-      Searchkick::Results.new(response, @search.options.merge(term: term, model_name: searchkick_klass.model_name))
+      opts = {
+        page: @page,
+        per_page: @per_page,
+        load: @load,
+        includes: options[:include] || options[:includes]
+      }
+      Searchkick::Results.new(searchkick_klass, response, opts)
     end
     private
@@ -2,36 +2,28 @@ module Searchkick
   module Reindex
     # https://gist.github.com/jarosan/3124884
+    # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
     def reindex
       alias_name = searchkick_index.name
-      new_index = alias_name + "_" + Time.now.strftime("%Y%m%d%H%M%S%L")
-      index = Tire::Index.new(new_index)
+      new_name = alias_name + "_" + Time.now.strftime("%Y%m%d%H%M%S%L")
+      index = Searchkick::Index.new(new_name)
       clean_indices
-      success = index.create searchkick_index_options
-      raise index.response.to_s if !success
+      index.create searchkick_index_options
-      if a = Tire::Alias.find(alias_name)
+      # check if alias exists
+      if Searchkick.client.indices.exists_alias(name: alias_name)
         searchkick_import(index) # import before swap
-        a.indices.each do |i|
-          a.indices.delete i
-        end
-
-        a.indices.add new_index
-        response = a.save
-
-        if response.success?
-          clean_indices
-        else
-          raise response.to_s
-        end
+        # get existing indices to remove
+        old_indices = Searchkick.client.indices.get_alias(name: alias_name).keys
+        actions = old_indices.map{|name| {remove: {index: name, alias: alias_name}} } + [{add: {index: new_name, alias: alias_name}}]
+        Searchkick.client.indices.update_aliases body: {actions: actions}
+        clean_indices
       else
         searchkick_index.delete if searchkick_index.exists?
-        response = Tire::Alias.create(name: alias_name, indices: [new_index])
-        raise response.to_s if !response.success?
-
+        Searchkick.client.indices.update_aliases body: {actions: [{add: {index: new_name, alias: alias_name}}]}
         searchkick_import(index) # import after swap
       end
@@ -42,10 +34,10 @@ module Searchkick
     # remove old indices that start w/ index_name
     def clean_indices
-      all_indices = JSON.parse(Tire::Configuration.client.get("#{Tire::Configuration.url}/_aliases").body)
+      all_indices = Searchkick.client.indices.get_aliases
       indices = all_indices.select{|k, v| v["aliases"].empty? && k =~ /\A#{Regexp.escape(searchkick_index.name)}_\d{14,17}\z/ }.keys
       indices.each do |index|
-        Tire::Index.new(index).delete
+        Searchkick::Index.new(index).delete
       end
       indices
     end
@@ -73,7 +65,7 @@ module Searchkick
         items = []
         scope.all.each do |item|
           items << item if item.should_index?
-          if items.length % batch_size == 0
+          if items.length == batch_size
             index.import items
             items = []
           end
 module Searchkick
-  class Results < Tire::Results::Collection
-    attr_reader :response
+  class Results
+    include Enumerable
+    extend Forwardable
+
+    attr_reader :klass, :response, :options
+
+    def_delegators :results, :each, :empty?, :size, :slice, :[], :to_ary
+
+    def initialize(klass, response, options = {})
+      @klass = klass
+      @response = response
+      @options = options
+    end
+
+    def results
+      @results ||= begin
+        if options[:load]
+          hit_ids = hits.map{|hit| hit["_id"] }
+          records = klass
+          if options[:includes]
+            records = records.includes(options[:includes])
+          end
+          records = records.find(hit_ids)
+          hit_ids = hit_ids.map(&:to_s)
+          records.sort_by{|r| hit_ids.index(r.id.to_s)  }
+        else
+          hits
+        end
+      end
+    end
     def suggestions
-      if @response["suggest"]
-        @response["suggest"].values.flat_map{|v| v.first["options"] }.sort_by{|o| -o["score"] }.map{|o| o["text"] }.uniq
+      if response["suggest"]
+        response["suggest"].values.flat_map{|v| v.first["options"] }.sort_by{|o| -o["score"] }.map{|o| o["text"] }.uniq
       else
         raise "Pass `suggest: true` to the search method for suggestions"
       end
     end
+    def each_with_hit(&block)
+      results.zip(hits).each(&block)
+    end
+
     def with_details
       each_with_hit.map do |model, hit|
         details = {}
@@ -20,13 +52,50 @@ module Searchkick
       end
     end
-    # fixes deprecation warning
-    def __find_records_by_ids(klass, ids)
-      @options[:load] === true ? klass.find(ids) : klass.includes(@options[:load][:include]).find(ids)
+    def facets
+      response["facets"]
     end
     def model_name
-      @options[:model_name]
+      klass.model_name
+    end
+
+    def total_count
+      response["hits"]["total"]
+    end
+
+    def current_page
+      options[:page]
+    end
+
+    def per_page
+      options[:per_page]
+    end
+
+    def total_pages
+      (total_count / per_page.to_f).ceil
+    end
+
+    def limit_value
+      per_page
+    end
+
+    def offset_value
+      current_page * per_page
+    end
+
+    def previous_page
+      current_page > 1 ? (current_page - 1) : nil
+    end
+
+    def next_page
+      current_page < total_pages ? (current_page + 1) : nil
+    end
+
+    protected
+
+    def hits
+      @response["hits"]["hits"]
     end
   end
@@ -3,7 +3,7 @@ module Searchkick
     def similar(options = {})
       like_text = self.class.searchkick_index.retrieve(document_type, id).to_hash
-        .keep_if{|k,v| k[0] != "_" and (!options[:fields] or options[:fields].map(&:to_sym).include?(k)) }
+        .keep_if{|k,v| !options[:fields] || options[:fields].map(&:to_s).include?(k) }
         .values.compact.join(" ")
       # TODO deep merge method
@@ -18,8 +18,9 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
-  spec.add_dependency "tire"
-  spec.add_dependency "tire-contrib"
+  spec.add_dependency "activemodel"
+  spec.add_dependency "elasticsearch"
+  spec.add_dependency "patron" # persistent http connections for performance
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
@@ -3,8 +3,11 @@ require_relative &quot;test_helper&quot;
 class TestIndex < Minitest::Unit::TestCase
   def test_clean_indices
-    old_index = Tire::Index.new("products_test_20130801000000000")
-    different_index = Tire::Index.new("items_test_20130801000000000")
+    old_index = Searchkick::Index.new("products_test_20130801000000000")
+    different_index = Searchkick::Index.new("items_test_20130801000000000")
+
+    old_index.delete if old_index.exists?
+    different_index.delete if different_index.exists?
     # create indexes
     old_index.create
@@ -18,7 +21,7 @@ class TestIndex &lt; Minitest::Unit::TestCase
   end
   def test_clean_indices_old_format
-    old_index = Tire::Index.new("products_test_20130801000000")
+    old_index = Searchkick::Index.new("products_test_20130801000000")
     old_index.create
     Product.clean_indices
@@ -26,6 +26,8 @@ class TestSql &lt; Minitest::Unit::TestCase
     assert_equal 2, products.per_page
     assert_equal 3, products.total_pages
     assert_equal 5, products.total_count
+    assert_equal 2, products.limit_value
+    assert_equal 4, products.offset_value
   end
   def test_pagination_nil_page
@@ -245,12 +247,12 @@ class TestSql &lt; Minitest::Unit::TestCase
   def test_load_false
     store_names ["Product A"]
-    assert_kind_of Tire::Results::Item, Product.search("product", load: false).first
+    assert_kind_of Hash, Product.search("product", load: false).first
   end
   def test_load_false_with_include
     store_names ["Product A"]
-    assert_kind_of Tire::Results::Item, Product.search("product", load: false, include: [:store]).first
+    assert_kind_of Hash, Product.search("product", load: false, include: [:store]).first
   end
   # TODO see if Mongoid is loaded
@@ -2,14 +2,12 @@ require &quot;bundler/setup&quot;
 Bundler.require(:default)
 require "minitest/autorun"
 require "minitest/pride"
+require "logger"
 ENV["RACK_ENV"] = "test"
 File.delete("elasticsearch.log") if File.exists?("elasticsearch.log")
-Tire.configure do
-  logger "elasticsearch.log", :level => "debug"
-  pretty true
-end
+Searchkick.client.transport.logger = Logger.new("elasticsearch.log")
 if defined?(Mongoid)
   Mongoid.configure do |config|
	@@ -568,7 +568,13 @@ end		@@ -568,7 +568,13 @@ end
568	And use the `query` option to search:	568	And use the `query` option to search:
569		569
570	```ruby	570	```ruby
571	-Product.search query: {match: {name: "milk"}}	571	+products = Product.search query: {match: {name: "milk"}}
		572	+```
		573	+
		574	+View the response with:
		575	+
		576	+```ruby
		577	+products.response
572	```	578	```
573		579
574	To keep the mappings and settings generated by Searchkick, use:	580	To keep the mappings and settings generated by Searchkick, use:
	@@ -579,7 +585,7 @@ class Product < ActiveRecord::Base		@@ -579,7 +585,7 @@ class Product < ActiveRecord::Base
579	end	585	end
580	```	586	```
581		587
582	-## Experimental [master]	588	+## Experimental
583		589
584	Modify the query generated by Searchkick.	590	Modify the query generated by Searchkick.
585		591
	@@ -3,4 +3,4 @@ source 'https://rubygems.org'		@@ -3,4 +3,4 @@ source 'https://rubygems.org'
3	# Specify your gem's dependencies in searchkick.gemspec	3	# Specify your gem's dependencies in searchkick.gemspec
4	gemspec path: "../"	4	gemspec path: "../"
5		5
6	-gem "mongoid", github: "mongoid/mongoid"	6	+gem "mongoid", "4.0.0.beta1"
@@ -0,0 +1,67 @@		@@ -0,0 +1,67 @@
	1	+module Searchkick
	2	+ class Index
	3	+ attr_reader :name
	4	+
	5	+ def initialize(name)
	6	+ @name = name
	7	+ end
	8	+
	9	+ def create(options = {})
	10	+ client.indices.create index: name, body: options
	11	+ end
	12	+
	13	+ def delete
	14	+ client.indices.delete index: name
	15	+ end
	16	+
	17	+ def exists?
	18	+ client.indices.exists index: name
	19	+ end
	20	+
	21	+ def refresh
	22	+ client.indices.refresh index: name
	23	+ end
	24	+
	25	+ def store(record)
	26	+ client.index(
	27	+ index: name,
	28	+ type: record.document_type,
	29	+ id: record.id,
	30	+ body: record.as_indexed_json
	31	+ )
	32	+ end
	33	+
	34	+ def remove(record)
	35	+ client.delete(
	36	+ index: name,
	37	+ type: record.document_type,
	38	+ id: record.id
	39	+ )
	40	+ end
	41	+
	42	+ def import(records)
	43	+ if records.any?
	44	+ client.bulk(
	45	+ index: name,
	46	+ type: records.first.document_type,
	47	+ body: records.map{\|r\| data = r.as_indexed_json; {index: {_id: data["_id"] \|\| data["id"] \|\| r.id, data: data}} }
	48	+ )
	49	+ end
	50	+ end
	51	+
	52	+ def retrieve(document_type, id)
	53	+ client.get_source(
	54	+ index: name,
	55	+ type: document_type,
	56	+ id: id
	57	+ )
	58	+ end
	59	+
	60	+ protected
	61	+
	62	+ def client
	63	+ Searchkick.client
	64	+ end
	65	+
	66	+ end
	67	+end
	@@ -1,19 +0,0 @@	@@ -1,19 +0,0 @@
1	-require "tire/rails/logger"
2	-require "tire/rails/logger/log_subscriber"
3	-
4	-class Tire::Rails::LogSubscriber
5	-
6	- # better output format
7	- def search(event)
8	- self.class.runtime += event.duration
9	- return unless logger.debug?
10	-
11	- payload = event.payload
12	-
13	- name = "%s (%.1fms)" % [payload[:name], event.duration]
14	- query = payload[:search].to_s
15	-
16	- debug " #{color(name, YELLOW, true)} #{query}"
17	- end
18	-
19	-end
1	module Searchkick	1	module Searchkick
2	- class Results < Tire::Results::Collection
3	- attr_reader :response	2	+ class Results
		3	+ include Enumerable
		4	+ extend Forwardable
		5	+
		6	+ attr_reader :klass, :response, :options
		7	+
		8	+ def_delegators :results, :each, :empty?, :size, :slice, :[], :to_ary
		9	+
		10	+ def initialize(klass, response, options = {})
		11	+ @klass = klass
		12	+ @response = response
		13	+ @options = options
		14	+ end
		15	+
		16	+ def results
		17	+ @results \|\|= begin
		18	+ if options[:load]
		19	+ hit_ids = hits.map{\|hit\| hit["_id"] }
		20	+ records = klass
		21	+ if options[:includes]
		22	+ records = records.includes(options[:includes])
		23	+ end
		24	+ records = records.find(hit_ids)
		25	+ hit_ids = hit_ids.map(&:to_s)
		26	+ records.sort_by{\|r\| hit_ids.index(r.id.to_s) }
		27	+ else
		28	+ hits
		29	+ end
		30	+ end
		31	+ end
4		32
5	def suggestions	33	def suggestions
6	- if @response["suggest"]
7	- @response["suggest"].values.flat_map{\|v\| v.first["options"] }.sort_by{\|o\| -o["score"] }.map{\|o\| o["text"] }.uniq	34	+ if response["suggest"]
		35	+ response["suggest"].values.flat_map{\|v\| v.first["options"] }.sort_by{\|o\| -o["score"] }.map{\|o\| o["text"] }.uniq
8	else	36	else
9	raise "Pass `suggest: true` to the search method for suggestions"	37	raise "Pass `suggest: true` to the search method for suggestions"
10	end	38	end
11	end	39	end
12		40
		41	+ def each_with_hit(&block)
		42	+ results.zip(hits).each(&block)
		43	+ end
		44	+
13	def with_details	45	def with_details
14	each_with_hit.map do \|model, hit\|	46	each_with_hit.map do \|model, hit\|
15	details = {}	47	details = {}
	@@ -20,13 +52,50 @@ module Searchkick		@@ -20,13 +52,50 @@ module Searchkick
20	end	52	end
21	end	53	end
22		54
23	- # fixes deprecation warning
24	- def __find_records_by_ids(klass, ids)
25	- @options[:load] === true ? klass.find(ids) : klass.includes(@options[:load][:include]).find(ids)	55	+ def facets
		56	+ response["facets"]
26	end	57	end
27		58
28	def model_name	59	def model_name
29	- @options[:model_name]	60	+ klass.model_name
		61	+ end
		62	+
		63	+ def total_count
		64	+ response["hits"]["total"]
		65	+ end
		66	+
		67	+ def current_page
		68	+ options[:page]
		69	+ end
		70	+
		71	+ def per_page
		72	+ options[:per_page]
		73	+ end
		74	+
		75	+ def total_pages
		76	+ (total_count / per_page.to_f).ceil
		77	+ end
		78	+
		79	+ def limit_value
		80	+ per_page
		81	+ end
		82	+
		83	+ def offset_value
		84	+ current_page * per_page
		85	+ end
		86	+
		87	+ def previous_page
		88	+ current_page > 1 ? (current_page - 1) : nil
		89	+ end
		90	+
		91	+ def next_page
		92	+ current_page < total_pages ? (current_page + 1) : nil
		93	+ end
		94	+
		95	+ protected
		96	+
		97	+ def hits
		98	+ @response["hits"]["hits"]
30	end	99	end
31		100
32	end	101	end
	@@ -3,7 +3,7 @@ module Searchkick		@@ -3,7 +3,7 @@ module Searchkick
3		3
4	def similar(options = {})	4	def similar(options = {})
5	like_text = self.class.searchkick_index.retrieve(document_type, id).to_hash	5	like_text = self.class.searchkick_index.retrieve(document_type, id).to_hash
6	- .keep_if{\|k,v\| k[0] != "_" and (!options[:fields] or options[:fields].map(&:to_sym).include?(k)) }	6	+ .keep_if{\|k,v\| !options[:fields] \|\| options[:fields].map(&:to_s).include?(k) }
7	.values.compact.join(" ")	7	.values.compact.join(" ")
8		8
9	# TODO deep merge method	9	# TODO deep merge method