From f4bec8516124d65025a137eb17a66a1786544233 Mon Sep 17 00:00:00 2001 From: Bilal Date: Mon, 20 Apr 2020 19:33:29 +0200 Subject: [PATCH 1/3] change ActiveRecord joins query with custom generated SQL --- lib/pg_searchable_regex.rb | 12 +++++-- lib/text_to_sql_query.rb | 64 +++++++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/lib/pg_searchable_regex.rb b/lib/pg_searchable_regex.rb index f14e120..d1a4ab5 100644 --- a/lib/pg_searchable_regex.rb +++ b/lib/pg_searchable_regex.rb @@ -55,8 +55,16 @@ module PgSearchable def ts_search(value) return if @ts_search_fields.blank? || value.blank? - includes(@ts_joins).references(:all).where( - TextToSqlQuery.new(value, @ts_search_fields, @default_fields, @ts_search_fields_mappings).where_clause).distinct + model = ancestors.first + sql_query_object = TextToSqlQuery.new( + value, + @ts_search_fields, + @default_field, + @ts_search_fields_mappings, + @ts_joins, + model + ) + joins(sql_query_object.join_clause).where(sql_query_object.where_clause).distinct end def should_update_cache_field? diff --git a/lib/text_to_sql_query.rb b/lib/text_to_sql_query.rb index 2c5e2eb..416d182 100644 --- a/lib/text_to_sql_query.rb +++ b/lib/text_to_sql_query.rb @@ -1,7 +1,7 @@ require_relative 'parser' class TextToSqlQuery - def initialize(text, fields, default_fields, fields_mappings = {}) + def initialize(text, fields, default_fields, fields_mappings = {}, joins = [], model = nil) @text = text.to_s.strip @fields = fields.map(&:to_sym) @@ -20,6 +20,8 @@ class TextToSqlQuery fields_mappings.each do |field, value| @fields_mappings[field] = value if @fields_mappings[field] end + @joins = joins + @model = model end def where_clause @@ -28,6 +30,20 @@ class TextToSqlQuery generate_sql @parsed_tree end + def join_clause + return if @joins.empty? + + table_column_mappings + model_association_mappings + + join_clause_part = '' + @joins.each do |join| + join_sql_part = generate_join_sql_part_for join + join_clause_part += join_sql_part + end + join_clause_part + end + private def generate_sql(tree) @@ -104,4 +120,50 @@ class TextToSqlQuery result.gsub!(/%/, '\%') result end + + def table_column_mappings + @table_column_mappings = {} + @fields_mappings.each_value do |table_with_column| + split_names = table_with_column.to_s.split '.' + table_name = split_names.first + column_name = split_names.second + @table_column_mappings[table_name] = [] if @table_column_mappings[table_name].nil? + @table_column_mappings[table_name] << column_name + end + @table_column_mappings + end + + def model_association_mappings + @model_associations = {} + @model.reflect_on_all_associations.each do |association| + name = association.name + + @model_associations[name] = { + option_as: association.options[:as] || name, + type: association.type + } + end + @model_associations + end + + def generate_join_sql_part_for(join) + association_data = @model_associations[join] + join_table_name = join.to_s + raise "Join table #{join_table_name} has no association data" if association_data.nil? + + select_sql_part = '' + columns_for_table = @table_column_mappings[join_table_name] || [] + + # TODO: Can be optimized - do not include columns that are not referenced in user query + columns_for_table.each do |column_name| + select_sql_part += "string_agg(#{column_name}, '') AS #{column_name}, " + end + + option_as = association_data[:option_as] + type = association_data[:type] + model_name = @model.to_s + table_name = @model.table_name + + "LEFT JOIN (SELECT #{option_as}_id, #{select_sql_part} #{type} FROM #{join_table_name} GROUP BY #{option_as}_id, #{type}) #{join_table_name} on #{join_table_name}.#{option_as}_id = #{table_name}.id AND #{join_table_name}.#{type} = '#{model_name}'" + end end From 455845e36b7528de389c83995ddb927ce511a20e Mon Sep 17 00:00:00 2001 From: Bilal Date: Mon, 20 Apr 2020 22:21:26 +0200 Subject: [PATCH 2/3] update specs; add new specs to test for reported bug related to AND query in referenced table --- spec/lib/pg_searchable_new_spec.rb | 147 ++++++++++++++++++++++++----- spec/schema.rb | 2 + spec/support/models.rb | 16 ++++ 3 files changed, 140 insertions(+), 25 deletions(-) diff --git a/spec/lib/pg_searchable_new_spec.rb b/spec/lib/pg_searchable_new_spec.rb index 953f194..1f2c8cc 100644 --- a/spec/lib/pg_searchable_new_spec.rb +++ b/spec/lib/pg_searchable_new_spec.rb @@ -158,35 +158,132 @@ describe PgSearchable do expect(DynamicModelWithTagValues.scope_search('tag:green or value:"not"')).to contain_exactly(record1, record2) end - it 'can search in referenced column and in model columns with multiple search terms connected with logical operators' do - record1 = DynamicModelWithTagValues.create name: 'something', value: 'amazing' - record2 = DynamicModelWithTagValues.create name: 'new record', value: 'not so amazing' - record3 = DynamicModelWithTagValues.create name: 'last one', value: 'no value' - record4 = DynamicModelWithTagValues.create name: 'really last one', value: 'no value' - Tag.create(taggable: record1, value: 'red') - Tag.create(taggable: record1, value: 'green') - Tag.create(taggable: record2, value: 'black') - Tag.create(taggable: record3, value: '-12') - Tag.create(taggable: record4, value: '-') + describe 'searching in model with has_many association' do + before do + records = DynamicModelWithTagValues.create [{ name: 'something', value: 'amazing' }, + { name: 'new record', value: 'not so amazing' }, + { name: 'last one', value: 'no value' }, + { name: 'really last one', value: 'no value' }] - expect(DynamicModelWithTagValues.scope_search('tag:red or tag:black')).to contain_exactly(record1, record2) - expect(DynamicModelWithTagValues.scope_search('tag:red and tag:black')).to be_empty - expect(DynamicModelWithTagValues.scope_search('tag:red or tag:green')).to contain_exactly(record1) - expect(DynamicModelWithTagValues.scope_search('not tag:-12 and not value:amazing')).to contain_exactly(record4) + Tag.create [{ taggable: records[0], value: 'red', custom_attribute: 'rose' }, + { taggable: records[0], value: 'green', custom_attribute: 'garden' }, + { taggable: records[1], value: 'black', custom_attribute: 'sky' }, + { taggable: records[2], value: '-1/12', custom_attribute: 'gold nugget' }, + { taggable: records[3], value: '-', custom_attribute: 'unknown' }, + { taggable: records[3], value: 'red-green', custom_attribute: 'unicorn' }] + end + + it 'can search with multiple search terms connected with OR operator' do + records = DynamicModelWithTagValues.all + + expect(DynamicModelWithTagValues.scope_search('tag:red or tag:black')).to contain_exactly(records[0], records[1], records[3]) + expect(DynamicModelWithTagValues.scope_search('tag:red or tag:green')).to contain_exactly(records[0], records[3]) + end + + it 'can search with multiple search terms connected with AND operator' do + records = DynamicModelWithTagValues.all + + expect(DynamicModelWithTagValues.scope_search('tag:red and tag:black')).to be_empty + expect(DynamicModelWithTagValues.scope_search('tag:red and tag:green')).to contain_exactly(records[0], records[3]) + end + + it 'can search with multiple search terms and containing NOT' do + records = DynamicModelWithTagValues.all + + expect(DynamicModelWithTagValues.scope_search('not tag:"-1/12" and not value:amazing')).to contain_exactly(records[3]) + end + + it 'can search in referenced column and in model columns with multiple search terms connected with logical operators and with brackets' do + records = DynamicModelWithTagValues.all + + expect(DynamicModelWithTagValues.scope_search('(tag:- and not tag:12) or (value:"amazing" and not value:"not")')).to contain_exactly(records[0], records[3]) + end end - it 'can search in referenced column and in model columns with multiple search terms connected with logical operators and with brackets' do - record1 = DynamicModelWithTagValues.create name: 'something', value: 'amazing' - record2 = DynamicModelWithTagValues.create name: 'new record', value: 'not so amazing' - record3 = DynamicModelWithTagValues.create name: 'last one', value: 'no value' - record4 = DynamicModelWithTagValues.create name: 'really last one', value: 'no value' - Tag.create(taggable: record1, value: 'red') - Tag.create(taggable: record1, value: 'green') - Tag.create(taggable: record2, value: 'black') - Tag.create(taggable: record3, value: '-12') - Tag.create(taggable: record4, value: '-') + describe 'searching through multiple referenced columns in model with has_many association' do + before do + records = DynamicModelWithTagValuesAndCustomAttribute.create [{ name: 'something', value: 'amazing' }, + { name: 'new record', value: 'not so amazing' }, + { name: 'last one', value: 'no value' }, + { name: 'really last one', value: 'no value' }] - expect(DynamicModelWithTagValues.scope_search('(tag:- and not tag:12) or (value:"amazing" and not value:"not")')).to contain_exactly(record1, record4) + Tag.create [{ taggable: records[0], value: 'red', custom_attribute: 'rose' }, + { taggable: records[0], value: 'green', custom_attribute: 'garden' }, + { taggable: records[1], value: 'black', custom_attribute: 'sky' }, + { taggable: records[2], value: '-1/12', custom_attribute: 'gold nugget' }, + { taggable: records[3], value: '-', custom_attribute: 'unknown' }, + { taggable: records[3], value: 'red-green', custom_attribute: 'unicorn' }] + end + + it 'can search with multiple search terms connected with OR operator' do + records = DynamicModelWithTagValuesAndCustomAttribute.all + + expect(DynamicModelWithTagValuesAndCustomAttribute.scope_search('tag:red or custom:sky')).to contain_exactly(records[0], records[1], records[3]) + end + + it 'can search with multiple search terms connected with AND operator' do + records = DynamicModelWithTagValuesAndCustomAttribute.all + + expect(DynamicModelWithTagValuesAndCustomAttribute.scope_search('custom:unicorn and tag:black')).to be_empty + expect(DynamicModelWithTagValuesAndCustomAttribute.scope_search('tag:"-1/12" and custom:gold')).to contain_exactly(records[2]) + end + + it 'can search with multiple search terms and containing NOT' do + expect(DynamicModelWithTagValuesAndCustomAttribute.scope_search('not tag:"-1/12" and not value:amazing and not custom:unknown')).to be_empty + end + + it 'can search with multiple search terms connected with logical operators and with brackets' do + records = DynamicModelWithTagValuesAndCustomAttribute.all + + expect(DynamicModelWithTagValuesAndCustomAttribute.scope_search('(tag:- and not tag:12) or (value:"amazing" and not value:"not") or (custom:unknown or custom:rose)')).to contain_exactly(records[0], records[3]) + end + end + + describe 'searching in model with multiple has_many associations' do + before do + records = DynamicModelWithTagAndCategories.create [{ name: 'something', value: 'amazing' }, + { name: 'new record', value: 'not so amazing' }, + { name: 'last one', value: 'no value' }, + { name: 'really last one', value: 'no value' }] + + Tag.create [{ taggable: records[0], value: 'red', custom_attribute: 'rose' }, + { taggable: records[0], value: 'green', custom_attribute: 'garden' }, + { taggable: records[1], value: 'black', custom_attribute: 'sky' }, + { taggable: records[2], value: '-1/12', custom_attribute: 'gold nugget' }, + { taggable: records[3], value: '-', custom_attribute: 'unknown' }, + { taggable: records[3], value: 'red-green', custom_attribute: 'unicorn' }] + + Category.create [{ categoriable: records[0], name: 'home' }, + { categoriable: records[0], name: 'home' }, + { categoriable: records[1], name: 'world' }, + { categoriable: records[2], name: 'math' }, + { categoriable: records[3], name: 'unknown' }, + { categoriable: records[3], name: 'myth' }] + + end + + it 'can search with multiple search terms connected with OR operator' do + records = DynamicModelWithTagAndCategories.all + + expect(DynamicModelWithTagAndCategories.scope_search('tag:red or category:math')).to contain_exactly(records[0], records[2], records[3]) + end + + it 'can search with multiple search terms connected with AND operator' do + records = DynamicModelWithTagAndCategories.all + + expect(DynamicModelWithTagAndCategories.scope_search('category:home and tag:-')).to be_empty + expect(DynamicModelWithTagAndCategories.scope_search('tag:"-1/12" and category:math')).to contain_exactly(records[2]) + end + + it 'can search with multiple search terms and containing NOT' do + expect(DynamicModelWithTagAndCategories.scope_search('not tag:"-1/12" and not value:amazing and not category:unknown')).to be_empty + end + + it 'can search with multiple search terms connected with logical operators and with brackets' do + records = DynamicModelWithTagAndCategories.all + + expect(DynamicModelWithTagAndCategories.scope_search('(tag:- and not tag:12) or (value:"amazing" and not value:"not") or (category:unknown or category:math)')).to contain_exactly(records[0], records[2], records[3]) + end end end end diff --git a/spec/schema.rb b/spec/schema.rb index 0ddb2b9..ae3e2f2 100644 --- a/spec/schema.rb +++ b/spec/schema.rb @@ -27,6 +27,7 @@ ActiveRecord::Schema.define do create_table :tags, force: true do |t| t.string :value + t.string :custom_attribute t.references :category, index: true t.references :taggable, polymorphic: true, index: true t.timestamps null: false @@ -34,6 +35,7 @@ ActiveRecord::Schema.define do create_table :categories, force: true do |t| t.string :name + t.references :categoriable, polymorphic: true, index: true t.timestamps null: false end diff --git a/spec/support/models.rb b/spec/support/models.rb index 58daa0f..5dd6f36 100755 --- a/spec/support/models.rb +++ b/spec/support/models.rb @@ -82,6 +82,21 @@ class DynamicModelWithTagValues < ActiveRecord::Base has_many :tags, as: :taggable end +class DynamicModelWithTagAndCategories < ActiveRecord::Base + self.table_name = :dynamic_models + include PgSearchable + pg_search fields: %i[dynamic_models.id dynamic_models.name dynamic_models.value], fields_mappings: {tag: 'tags.value', category: 'categories.name'}, joins: [:tags, :categories] + has_many :tags, as: :taggable + has_many :categories, as: :categoriable +end + +class DynamicModelWithTagValuesAndCustomAttribute < ActiveRecord::Base + self.table_name = :dynamic_models + include PgSearchable + pg_search fields: %i[dynamic_models.id dynamic_models.name dynamic_models.value], fields_mappings: {tag: 'tags.value', custom: 'tags.custom_attribute'}, joins: [:tags] + has_many :tags, as: :taggable +end + class DynamicModelWithCategory < ActiveRecord::Base self.table_name = :dynamic_models include PgSearchable @@ -112,4 +127,5 @@ end class Category < ActiveRecord::Base has_many :tags + belongs_to :categoriable, polymorphic: true end From 0de540c5bd316ad9b67ed1b6eeeb760965c9a1d8 Mon Sep 17 00:00:00 2001 From: Bilal Date: Fri, 24 Apr 2020 11:40:53 +0200 Subject: [PATCH 3/3] fix bug after rebase --- lib/pg_searchable_regex.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pg_searchable_regex.rb b/lib/pg_searchable_regex.rb index d1a4ab5..576e9e5 100644 --- a/lib/pg_searchable_regex.rb +++ b/lib/pg_searchable_regex.rb @@ -59,7 +59,7 @@ module PgSearchable sql_query_object = TextToSqlQuery.new( value, @ts_search_fields, - @default_field, + @default_fields, @ts_search_fields_mappings, @ts_joins, model