Added block form of scroll

Andrew Kane
1 parent 59ebb7fb
Showing 4 changed files with 52 additions and 13 deletions Show diff stats
CHANGELOG.md
README.md
lib/searchkick/results.rb
test/scroll_test.rb
+## 4.0.2 [unreleased]
+
+- Added block form of `scroll`
+
 ## 4.0.1
 - Added support for scroll API
@@ -1484,12 +1484,20 @@ To retrieve a very large number of results, use the [scroll API](https://www.ela
 ```ruby
 products = Product.search "*", scroll: "1m"
 while products.any?
-  # do something ...
+  # process batch ...
   products = products.scroll
 end
 ```
+On the master branch, you can also do:
+
+```ruby
+Product.search("*", scroll: "1m").scroll do |batch|
+  # process batch ...
+end
+```
+
 You should call `scroll` on each new set of results, not the original result.
 ## Nested Data
@@ -228,18 +228,35 @@ module Searchkick
     def scroll
       raise Searchkick::Error, "Pass `scroll` option to the search method for scrolling" unless scroll_id
-      params = {
-        scroll: options[:scroll],
-        scroll_id: scroll_id
-      }
-
-      begin
-        Searchkick::Results.new(@klass, Searchkick.client.scroll(params), @options)
-      rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
-        if e.class.to_s =~ /NotFound/ && e.message =~ /search_context_missing_exception/i
-          raise Searchkick::Error, "Scroll id has expired"
-        else
-          raise e
+      if block_given?
+        records = self
+        while records.any?
+          yield records
+          records = records.scroll
+        end
+
+        begin
+          # try to clear scroll
+          # not required as scroll will expire
+          # but there is a cost to open scrolls
+          Searchkick.client.clear_scroll(scroll_id: scroll_id)
+        rescue Elasticsearch::Transport::Transport::Error
+          # okay if it fails
+        end
+      else
+        params = {
+          scroll: options[:scroll],
+          scroll_id: scroll_id
+        }
+
+        begin
+          Searchkick::Results.new(@klass, Searchkick.client.scroll(params), @options)
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
+          if e.class.to_s =~ /NotFound/ && e.message =~ /search_context_missing_exception/i
+            raise Searchkick::Error, "Scroll id has expired"
+          else
+            raise e
+          end
         end
       end
     end
@@ -61,4 +61,14 @@ class ScrollTest &lt; Minitest::Test
     end
     assert_match /Pass .+ option/, error.message
   end
+
+  def test_scroll_block
+    store_names ["Product A", "Product B", "Product C", "Product D", "Product E", "Product F"]
+    batches_count = 0
+    Product.search("*", scroll: "1m", per_page: 2).scroll do |batch|
+      assert_equal 2, batch.size
+      batches_count += 1
+    end
+    assert_equal 3, batches_count
+  end
 end
		1	+## 4.0.2 [unreleased]
		2	+
		3	+- Added block form of `scroll`
		4	+
1	## 4.0.1	5	## 4.0.1
2		6
3	- Added support for scroll API	7	- Added support for scroll API
	@@ -228,18 +228,35 @@ module Searchkick		@@ -228,18 +228,35 @@ module Searchkick
228	def scroll	228	def scroll
229	raise Searchkick::Error, "Pass `scroll` option to the search method for scrolling" unless scroll_id	229	raise Searchkick::Error, "Pass `scroll` option to the search method for scrolling" unless scroll_id
230		230
231	- params = {
232	- scroll: options[:scroll],
233	- scroll_id: scroll_id
234	- }
235	-
236	- begin
237	- Searchkick::Results.new(@klass, Searchkick.client.scroll(params), @options)
238	- rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
239	- if e.class.to_s =~ /NotFound/ && e.message =~ /search_context_missing_exception/i
240	- raise Searchkick::Error, "Scroll id has expired"
241	- else
242	- raise e	231	+ if block_given?
		232	+ records = self
		233	+ while records.any?
		234	+ yield records
		235	+ records = records.scroll
		236	+ end
		237	+
		238	+ begin
		239	+ # try to clear scroll
		240	+ # not required as scroll will expire
		241	+ # but there is a cost to open scrolls
		242	+ Searchkick.client.clear_scroll(scroll_id: scroll_id)
		243	+ rescue Elasticsearch::Transport::Transport::Error
		244	+ # okay if it fails
		245	+ end
		246	+ else
		247	+ params = {
		248	+ scroll: options[:scroll],
		249	+ scroll_id: scroll_id
		250	+ }
		251	+
		252	+ begin
		253	+ Searchkick::Results.new(@klass, Searchkick.client.scroll(params), @options)
		254	+ rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
		255	+ if e.class.to_s =~ /NotFound/ && e.message =~ /search_context_missing_exception/i
		256	+ raise Searchkick::Error, "Scroll id has expired"
		257	+ else
		258	+ raise e
		259	+ end
243	end	260	end
244	end	261	end
245	end	262	end
	@@ -61,4 +61,14 @@ class ScrollTest < Minitest::Test		@@ -61,4 +61,14 @@ class ScrollTest < Minitest::Test
61	end	61	end
62	assert_match /Pass .+ option/, error.message	62	assert_match /Pass .+ option/, error.message
63	end	63	end
		64	+
		65	+ def test_scroll_block
		66	+ store_names ["Product A", "Product B", "Product C", "Product D", "Product E", "Product F"]
		67	+ batches_count = 0
		68	+ Product.search("*", scroll: "1m", per_page: 2).scroll do \|batch\|
		69	+ assert_equal 2, batch.size
		70	+ batches_count += 1
		71	+ end
		72	+ assert_equal 3, batches_count
		73	+ end
64	end	74	end