From fe639ff0f6b11571b549cd86ba463926bfd9133a Mon Sep 17 00:00:00 2001
From: Andrew Kane <andrew@chartkick.com>
Date: Wed, 20 Nov 2019 01:18:32 -0800
Subject: [PATCH] Added deep_paging option

---
 CHANGELOG.md                    |  4 ++++
 README.md                       | 14 +++++++-------
 lib/searchkick/index_options.rb |  7 +++++++
 lib/searchkick/model.rb         |  2 +-
 lib/searchkick/query.rb         |  8 +++++++-
 5 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cc9db3d..be6349a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 4.1.2 (unreleased)
+
+- Added `deep_paging` option
+
 ## 4.1.1 (2019-11-19)
 
 - Added `chinese2` and `korean2` languages
diff --git a/README.md b/README.md
index 533ab3a..93076b6 100644
--- a/README.md
+++ b/README.md
@@ -137,8 +137,6 @@ Limit / offset
 limit: 20, offset: 40
 ```
 
-**Note:** By default, Elasticsearch [limits pagination](#deep-pagination) to the first 10,000 results for performance
-
 Select
 
 ```ruby
@@ -182,6 +180,8 @@ Get the full response from Elasticsearch
 results.response
 ```
 
+**Note:** By default, Elasticsearch [limits paging](#deep-paging-master) to the first 10,000 results for performance. This applies to the total count as well.
+
 ### Boosting
 
 Boost important fields
@@ -1518,20 +1518,20 @@ end
 products.clear_scroll
 ```
 
-## Deep Pagination
+## Deep Paging [master]
 
-By default, Elasticsearch limits pagination to the first 10,000 results [for performance](https://www.elastic.co/guide/en/elasticsearch/guide/current/pagination.html). We don’t recommend changing this, but if you need to, you can use:
+By default, Elasticsearch limits paging to the first 10,000 results. [Here’s why](https://www.elastic.co/guide/en/elasticsearch/guide/current/pagination.html). We don’t recommend changing this, but if you really need all results, you can use:
 
 ```ruby
 class Product < ApplicationRecord
-  searchkick settings: {index: {max_result_window: 1000000000}}
+  searchkick deep_paging: true
 end
 ```
 
-And search with:
+If you just need an accurate total count, you can instead use:
 
 ```ruby
-Product.search("pears", limit: 1000000000, body_options: {track_total_hits: true})
+Product.search("pears", body_options: {track_total_hits: true})
 ```
 
 ## Nested Data
diff --git a/lib/searchkick/index_options.rb b/lib/searchkick/index_options.rb
index a0c909b..8320704 100644
--- a/lib/searchkick/index_options.rb
+++ b/lib/searchkick/index_options.rb
@@ -456,6 +456,13 @@ module Searchkick
         mappings = mappings.symbolize_keys.deep_merge(custom_mapping.symbolize_keys)
       end
 
+      if options[:deep_paging]
+        if !settings.dig(:index, :max_result_window) && !settings[:"index.max_result_window"]
+          settings[:index] ||= {}
+          settings[:index][:max_result_window] = 1_000_000_000
+        end
+      end
+
       {
         settings: settings,
         mappings: mappings
diff --git a/lib/searchkick/model.rb b/lib/searchkick/model.rb
index 154a75a..8df42f7 100644
--- a/lib/searchkick/model.rb
+++ b/lib/searchkick/model.rb
@@ -3,7 +3,7 @@ module Searchkick
     def searchkick(**options)
       options = Searchkick.model_options.merge(options)
 
-      unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :default_fields,
+      unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :deep_paging, :default_fields,
         :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,
         :locations, :mappings, :match, :merge_mappings, :routing, :searchable, :settings, :similarity,
         :special_characters, :stem, :stem_conversions, :suggest, :synonyms, :text_end,
diff --git a/lib/searchkick/query.rb b/lib/searchkick/query.rb
index 014aac5..1aed600 100644
--- a/lib/searchkick/query.rb
+++ b/lib/searchkick/query.rb
@@ -232,7 +232,9 @@ module Searchkick
 
       # pagination
       page = [options[:page].to_i, 1].max
-      per_page = (options[:limit] || options[:per_page] || 10_000).to_i
+      # maybe use index.max_result_window in the future
+      default_limit = searchkick_options[:deep_paging] ? 1_000_000_000 : 10_000
+      per_page = (options[:limit] || options[:per_page] || default_limit).to_i
       padding = [options[:padding].to_i, 0].max
       offset = options[:offset] || (page - 1) * per_page + padding
       scroll = options[:scroll]
@@ -516,6 +518,10 @@ module Searchkick
       # routing
       @routing = options[:routing] if options[:routing]
 
+      if searchkick_options[:deep_paging]
+        payload[:track_total_hits] = true
+      end
+
       # merge more body options
       payload = payload.deep_merge(options[:body_options]) if options[:body_options]
 
--
libgit2 0.21.0