More efficient aggregations with where clauses - cc @elguapo1611

Andrew
1 parent c1b7bfdd
Showing 2 changed files with 26 additions and 9 deletions Show diff stats
CHANGELOG.md
lib/searchkick/query.rb
 ## 2.4.1 [unreleased]
  
 - Better exception when trying to access results for failed multi-search query
+- More efficient aggregations with `where` clauses
 - Added support for `faraday_middleware-aws-sigv4`
 - Added `credentials` option to `aws_credentials`
 - Added `modifier` option to `boost_by`
@@ -457,12 +457,16 @@ module Searchkick
           where[:type] = [options[:type] || klass].flatten.map { |v| searchkick_index.klass_document_type(v, true) }
         end
  
-        # filters
+        # start everything as efficient filters
+        # move to post_filters as aggs demand
         filters = where_filters(where)
-        set_filters(payload, filters) if filters.any?
+        post_filters = []
  
         # aggregations
-        set_aggregations(payload) if options[:aggs]
+        set_aggregations(payload, filters, post_filters) if options[:aggs]
+
+        # filters
+        set_filters(payload, filters, post_filters)
  
         # suggestions
         set_suggestions(payload, options[:suggest]) if options[:suggest]
@@ -655,12 +659,11 @@ module Searchkick
       @highlighted_fields = payload[:highlight][:fields].keys
     end
  
-    def set_aggregations(payload)
+    def set_aggregations(payload, filters, post_filters)
       aggs = options[:aggs]
       payload[:aggs] = {}
  
       aggs = Hash[aggs.map { |f| [f, {}] }] if aggs.is_a?(Array) # convert to more advanced syntax
-
       aggs.each do |field, agg_options|
         size = agg_options[:limit] ? agg_options[:limit] : 1_000
         shared_agg_options = agg_options.slice(:order, :min_doc_count)
@@ -705,6 +708,17 @@ module Searchkick
         where = {}
         where = (options[:where] || {}).reject { |k| k == field } unless options[:smart_aggs] == false
         agg_filters = where_filters(where.merge(agg_options[:where] || {}))
+
+        # only do one level comparison for simplicity
+        filters.select! do |filter|
+          if agg_filters.include?(filter)
+            true
+          else
+            post_filters << filter
+            false
+          end
+        end
+
         if agg_filters.any?
           payload[:aggs][field] = {
             filter: {
@@ -720,14 +734,16 @@ module Searchkick
       end
     end
  
-    def set_filters(payload, filters)
-      if options[:aggs]
+    def set_filters(payload, filters, post_filters)
+      if post_filters.any?
         payload[:post_filter] = {
           bool: {
-            filter: filters
+            filter: post_filters
           }
         }
-      else
+      end
+
+      if filters.any?
         # more efficient query if no aggs
         payload[:query] = {
           bool: {
1	1	## 2.4.1 [unreleased]
2	2
3	3	- Better exception when trying to access results for failed multi-search query
	4	+- More efficient aggregations with `where` clauses
4	5	- Added support for `faraday_middleware-aws-sigv4`
5	6	- Added `credentials` option to `aws_credentials`
6	7	- Added `modifier` option to `boost_by`
...	...
...	...	@@ -457,12 +457,16 @@ module Searchkick
457	457	where[:type] = [options[:type] \|\| klass].flatten.map { \|v\| searchkick_index.klass_document_type(v, true) }
458	458	end
459	459
460		- # filters
	460	+ # start everything as efficient filters
	461	+ # move to post_filters as aggs demand
461	462	filters = where_filters(where)
462		- set_filters(payload, filters) if filters.any?
	463	+ post_filters = []
463	464
464	465	# aggregations
465		- set_aggregations(payload) if options[:aggs]
	466	+ set_aggregations(payload, filters, post_filters) if options[:aggs]
	467	+
	468	+ # filters
	469	+ set_filters(payload, filters, post_filters)
466	470
467	471	# suggestions
468	472	set_suggestions(payload, options[:suggest]) if options[:suggest]
...	...	@@ -655,12 +659,11 @@ module Searchkick
655	659	@highlighted_fields = payload[:highlight][:fields].keys
656	660	end
657	661
658		- def set_aggregations(payload)
	662	+ def set_aggregations(payload, filters, post_filters)
659	663	aggs = options[:aggs]
660	664	payload[:aggs] = {}
661	665
662	666	aggs = Hash[aggs.map { \|f\| [f, {}] }] if aggs.is_a?(Array) # convert to more advanced syntax
663		-
664	667	aggs.each do \|field, agg_options\|
665	668	size = agg_options[:limit] ? agg_options[:limit] : 1_000
666	669	shared_agg_options = agg_options.slice(:order, :min_doc_count)
...	...	@@ -705,6 +708,17 @@ module Searchkick
705	708	where = {}
706	709	where = (options[:where] \|\| {}).reject { \|k\| k == field } unless options[:smart_aggs] == false
707	710	agg_filters = where_filters(where.merge(agg_options[:where] \|\| {}))
	711	+
	712	+ # only do one level comparison for simplicity
	713	+ filters.select! do \|filter\|
	714	+ if agg_filters.include?(filter)
	715	+ true
	716	+ else
	717	+ post_filters << filter
	718	+ false
	719	+ end
	720	+ end
	721	+
708	722	if agg_filters.any?
709	723	payload[:aggs][field] = {
710	724	filter: {
...	...	@@ -720,14 +734,16 @@ module Searchkick
720	734	end
721	735	end
722	736
723		- def set_filters(payload, filters)
724		- if options[:aggs]
	737	+ def set_filters(payload, filters, post_filters)
	738	+ if post_filters.any?
725	739	payload[:post_filter] = {
726	740	bool: {
727		- filter: filters
	741	+ filter: post_filters
728	742	}
729	743	}
730		- else
	744	+ end
	745	+
	746	+ if filters.any?
731	747	# more efficient query if no aggs
732	748	payload[:query] = {
733	749	bool: {
...	...