Added deep_paging option

Andrew Kane
1 parent 18a501ce
Showing 5 changed files with 26 additions and 9 deletions Show diff stats
CHANGELOG.md
README.md
lib/searchkick/index_options.rb
lib/searchkick/model.rb
lib/searchkick/query.rb
+## 4.1.2 (unreleased)
+
+- Added `deep_paging` option
+
 ## 4.1.1 (2019-11-19)
 - Added `chinese2` and `korean2` languages
@@ -137,8 +137,6 @@ Limit / offset
 limit: 20, offset: 40
 ```
-**Note:** By default, Elasticsearch [limits pagination](#deep-pagination) to the first 10,000 results for performance
-
 Select
 ```ruby
@@ -182,6 +180,8 @@ Get the full response from Elasticsearch
 results.response
 ```
+**Note:** By default, Elasticsearch [limits paging](#deep-paging-master) to the first 10,000 results for performance. This applies to the total count as well.
+
 ### Boosting
 Boost important fields
@@ -1518,20 +1518,20 @@ end
 products.clear_scroll
 ```
-## Deep Pagination
+## Deep Paging [master]
-By default, Elasticsearch limits pagination to the first 10,000 results [for performance](https://www.elastic.co/guide/en/elasticsearch/guide/current/pagination.html). We don’t recommend changing this, but if you need to, you can use:
+By default, Elasticsearch limits paging to the first 10,000 results. [Here’s why](https://www.elastic.co/guide/en/elasticsearch/guide/current/pagination.html). We don’t recommend changing this, but if you really need all results, you can use:
 ```ruby
 class Product < ApplicationRecord
-  searchkick settings: {index: {max_result_window: 1000000000}}
+  searchkick deep_paging: true
 end
 ```
-And search with:
+If you just need an accurate total count, you can instead use:
 ```ruby
-Product.search("pears", limit: 1000000000, body_options: {track_total_hits: true})
+Product.search("pears", body_options: {track_total_hits: true})
 ```
 ## Nested Data
@@ -456,6 +456,13 @@ module Searchkick
         mappings = mappings.symbolize_keys.deep_merge(custom_mapping.symbolize_keys)
       end
+      if options[:deep_paging]
+        if !settings.dig(:index, :max_result_window) && !settings[:"index.max_result_window"]
+          settings[:index] ||= {}
+          settings[:index][:max_result_window] = 1_000_000_000
+        end
+      end
+
       {
         settings: settings,
         mappings: mappings
@@ -3,7 +3,7 @@ module Searchkick
     def searchkick(**options)
       options = Searchkick.model_options.merge(options)
-      unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :default_fields,
+      unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :deep_paging, :default_fields,
         :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,
         :locations, :mappings, :match, :merge_mappings, :routing, :searchable, :settings, :similarity,
         :special_characters, :stem, :stem_conversions, :suggest, :synonyms, :text_end,
@@ -232,7 +232,9 @@ module Searchkick
       # pagination
       page = [options[:page].to_i, 1].max
-      per_page = (options[:limit] || options[:per_page] || 10_000).to_i
+      # maybe use index.max_result_window in the future
+      default_limit = searchkick_options[:deep_paging] ? 1_000_000_000 : 10_000
+      per_page = (options[:limit] || options[:per_page] || default_limit).to_i
       padding = [options[:padding].to_i, 0].max
       offset = options[:offset] || (page - 1) * per_page + padding
       scroll = options[:scroll]
@@ -516,6 +518,10 @@ module Searchkick
       # routing
       @routing = options[:routing] if options[:routing]
+      if searchkick_options[:deep_paging]
+        payload[:track_total_hits] = true
+      end
+
       # merge more body options
       payload = payload.deep_merge(options[:body_options]) if options[:body_options]
		1	+## 4.1.2 (unreleased)
		2	+
		3	+- Added `deep_paging` option
		4	+
1	## 4.1.1 (2019-11-19)	5	## 4.1.1 (2019-11-19)
2		6
3	- Added `chinese2` and `korean2` languages	7	- Added `chinese2` and `korean2` languages
	@@ -456,6 +456,13 @@ module Searchkick		@@ -456,6 +456,13 @@ module Searchkick
456	mappings = mappings.symbolize_keys.deep_merge(custom_mapping.symbolize_keys)	456	mappings = mappings.symbolize_keys.deep_merge(custom_mapping.symbolize_keys)
457	end	457	end
458		458
		459	+ if options[:deep_paging]
		460	+ if !settings.dig(:index, :max_result_window) && !settings[:"index.max_result_window"]
		461	+ settings[:index] \|\|= {}
		462	+ settings[:index][:max_result_window] = 1_000_000_000
		463	+ end
		464	+ end
		465	+
459	{	466	{
460	settings: settings,	467	settings: settings,
461	mappings: mappings	468	mappings: mappings
	@@ -3,7 +3,7 @@ module Searchkick		@@ -3,7 +3,7 @@ module Searchkick
3	def searchkick(**options)	3	def searchkick(**options)
4	options = Searchkick.model_options.merge(options)	4	options = Searchkick.model_options.merge(options)
5		5
6	- unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :default_fields,	6	+ unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :deep_paging, :default_fields,
7	:filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,	7	:filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,
8	:locations, :mappings, :match, :merge_mappings, :routing, :searchable, :settings, :similarity,	8	:locations, :mappings, :match, :merge_mappings, :routing, :searchable, :settings, :similarity,
9	:special_characters, :stem, :stem_conversions, :suggest, :synonyms, :text_end,	9	:special_characters, :stem, :stem_conversions, :suggest, :synonyms, :text_end,