Added block form of scroll

Andrew Kane
1 parent 59ebb7fb
Showing 4 changed files with 52 additions and 13 deletions Show diff stats
CHANGELOG.md
README.md
lib/searchkick/results.rb
test/scroll_test.rb
+## 4.0.2 [unreleased]
+
+- Added block form of `scroll`
+
 ## 4.0.1
  
 - Added support for scroll API
@@ -1484,12 +1484,20 @@ To retrieve a very large number of results, use the [scroll API](https://www.ela
 ```ruby
 products = Product.search "*", scroll: "1m"
 while products.any?
-  # do something ...
+  # process batch ...
  
   products = products.scroll
 end
 ```
  
+On the master branch, you can also do:
+
+```ruby
+Product.search("*", scroll: "1m").scroll do |batch|
+  # process batch ...
+end
+```
+
 You should call `scroll` on each new set of results, not the original result.
  
 ## Nested Data
@@ -228,18 +228,35 @@ module Searchkick
     def scroll
       raise Searchkick::Error, "Pass `scroll` option to the search method for scrolling" unless scroll_id
  
-      params = {
-        scroll: options[:scroll],
-        scroll_id: scroll_id
-      }
-
-      begin
-        Searchkick::Results.new(@klass, Searchkick.client.scroll(params), @options)
-      rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
-        if e.class.to_s =~ /NotFound/ && e.message =~ /search_context_missing_exception/i
-          raise Searchkick::Error, "Scroll id has expired"
-        else
-          raise e
+      if block_given?
+        records = self
+        while records.any?
+          yield records
+          records = records.scroll
+        end
+
+        begin
+          # try to clear scroll
+          # not required as scroll will expire
+          # but there is a cost to open scrolls
+          Searchkick.client.clear_scroll(scroll_id: scroll_id)
+        rescue Elasticsearch::Transport::Transport::Error
+          # okay if it fails
+        end
+      else
+        params = {
+          scroll: options[:scroll],
+          scroll_id: scroll_id
+        }
+
+        begin
+          Searchkick::Results.new(@klass, Searchkick.client.scroll(params), @options)
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
+          if e.class.to_s =~ /NotFound/ && e.message =~ /search_context_missing_exception/i
+            raise Searchkick::Error, "Scroll id has expired"
+          else
+            raise e
+          end
         end
       end
     end
@@ -61,4 +61,14 @@ class ScrollTest &lt; Minitest::Test
     end
     assert_match /Pass .+ option/, error.message
   end
+
+  def test_scroll_block
+    store_names ["Product A", "Product B", "Product C", "Product D", "Product E", "Product F"]
+    batches_count = 0
+    Product.search("*", scroll: "1m", per_page: 2).scroll do |batch|
+      assert_equal 2, batch.size
+      batches_count += 1
+    end
+    assert_equal 3, batches_count
+  end
 end
	1	+## 4.0.2 [unreleased]
	2	+
	3	+- Added block form of `scroll`
	4	+
1	5	## 4.0.1
2	6
3	7	- Added support for scroll API
...	...
...	...	@@ -1484,12 +1484,20 @@ To retrieve a very large number of results, use the [scroll API](https://www.ela
1484	1484	```ruby
1485	1485	products = Product.search "*", scroll: "1m"
1486	1486	while products.any?
1487		- # do something ...
	1487	+ # process batch ...
1488	1488
1489	1489	products = products.scroll
1490	1490	end
1491	1491	```
1492	1492
	1493	+On the master branch, you can also do:
	1494	+
	1495	+```ruby
	1496	+Product.search("*", scroll: "1m").scroll do \|batch\|
	1497	+ # process batch ...
	1498	+end
	1499	+```
	1500	+
1493	1501	You should call `scroll` on each new set of results, not the original result.
1494	1502
1495	1503	## Nested Data
...	...
...	...	@@ -228,18 +228,35 @@ module Searchkick
228	228	def scroll
229	229	raise Searchkick::Error, "Pass `scroll` option to the search method for scrolling" unless scroll_id
230	230
231		- params = {
232		- scroll: options[:scroll],
233		- scroll_id: scroll_id
234		- }
235		-
236		- begin
237		- Searchkick::Results.new(@klass, Searchkick.client.scroll(params), @options)
238		- rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
239		- if e.class.to_s =~ /NotFound/ && e.message =~ /search_context_missing_exception/i
240		- raise Searchkick::Error, "Scroll id has expired"
241		- else
242		- raise e
	231	+ if block_given?
	232	+ records = self
	233	+ while records.any?
	234	+ yield records
	235	+ records = records.scroll
	236	+ end
	237	+
	238	+ begin
	239	+ # try to clear scroll
	240	+ # not required as scroll will expire
	241	+ # but there is a cost to open scrolls
	242	+ Searchkick.client.clear_scroll(scroll_id: scroll_id)
	243	+ rescue Elasticsearch::Transport::Transport::Error
	244	+ # okay if it fails
	245	+ end
	246	+ else
	247	+ params = {
	248	+ scroll: options[:scroll],
	249	+ scroll_id: scroll_id
	250	+ }
	251	+
	252	+ begin
	253	+ Searchkick::Results.new(@klass, Searchkick.client.scroll(params), @options)
	254	+ rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
	255	+ if e.class.to_s =~ /NotFound/ && e.message =~ /search_context_missing_exception/i
	256	+ raise Searchkick::Error, "Scroll id has expired"
	257	+ else
	258	+ raise e
	259	+ end
243	260	end
244	261	end
245	262	end
...	...
...	...	@@ -61,4 +61,14 @@ class ScrollTest < Minitest::Test
61	61	end
62	62	assert_match /Pass .+ option/, error.message
63	63	end
	64	+
	65	+ def test_scroll_block
	66	+ store_names ["Product A", "Product B", "Product C", "Product D", "Product E", "Product F"]
	67	+ batches_count = 0
	68	+ Product.search("*", scroll: "1m", per_page: 2).scroll do \|batch\|
	69	+ assert_equal 2, batch.size
	70	+ batches_count += 1
	71	+ end
	72	+ assert_equal 3, batches_count
	73	+ end
64	74	end
...	...