From 838b60ed920f7ccad630cbe2bf1b766baeb7c958 Mon Sep 17 00:00:00 2001 From: Bilal Date: Thu, 16 Apr 2020 14:59:51 +0200 Subject: [PATCH 01/15] Enable multiple default columns --- lib/text_to_sql_query.rb | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/text_to_sql_query.rb b/lib/text_to_sql_query.rb index c464349..2c5e2eb 100644 --- a/lib/text_to_sql_query.rb +++ b/lib/text_to_sql_query.rb @@ -1,10 +1,17 @@ require_relative 'parser' class TextToSqlQuery - def initialize(text, fields, default_field, fields_mappings = {}) + def initialize(text, fields, default_fields, fields_mappings = {}) @text = text.to_s.strip @fields = fields.map(&:to_sym) - @default_field = default_field.to_sym + + # Keep compatibility with previous version where default_field(s) was string/symbol + @default_fields = if default_fields.is_a? Array + default_fields.map(&:to_sym) + else + [default_fields.to_sym] + end + @fields_mappings = fields_mappings.merge(@fields.reduce({}) do |mappings, field| _table_name, field_name = field.to_s.split('.') mappings[field_name.to_sym] = field @@ -29,7 +36,14 @@ class TextToSqlQuery case first_key when :DEFAULT_COLUMN escaped_node_value = handle_special_chars node_value - ["CAST(#{@default_field.to_s} AS TEXT) ILIKE ?", "%#{escaped_node_value}%"] + query_part = '(' + params_part = [] + @default_fields.each do |default_field| + query_part += "CAST(#{default_field.to_s} AS TEXT) ILIKE ? OR " + params_part << "%#{escaped_node_value}%" + end + query_part = query_part[0...-4] + ')' + [query_part, *params_part] when :OPERATOR_OR generate_expression_for_logical_operator(:OR, node_value) when :OPERATOR_AND -- 2.47.3 From b62638a647910a7bcbcc996201dfc7ea46573047 Mon Sep 17 00:00:00 2001 From: Bilal Date: Thu, 16 Apr 2020 15:00:25 +0200 Subject: [PATCH 02/15] Update and add new text-to-sql-query specs --- spec/lib/text_to_sql_query_spec.rb | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/spec/lib/text_to_sql_query_spec.rb b/spec/lib/text_to_sql_query_spec.rb index aed9c6b..74da5ff 100644 --- a/spec/lib/text_to_sql_query_spec.rb +++ b/spec/lib/text_to_sql_query_spec.rb @@ -4,7 +4,16 @@ require_relative '../../lib/text_to_sql_query' describe TextToSqlQuery do describe '.new' do # tests simple search term without column name and without quotes - it { expect(described_class.new('some-default-value', [:"players.name"], :"players.name").where_clause).to eq(['CAST(players.name AS TEXT) ILIKE ?', '%some-default-value%']) } + it { expect(described_class.new('some-default-value', [:"players.name"], :"players.name").where_clause).to eq(['(CAST(players.name AS TEXT) ILIKE ?)', '%some-default-value%']) } + + # tests simple search with two default columns without quotes + it { expect(described_class.new('search-term', [:'players.name', :'players.tags', 'players.id'], [:'players.name', :'players.tags']).where_clause).to eq(['(CAST(players.name AS TEXT) ILIKE ? OR CAST(players.tags AS TEXT) ILIKE ?)', '%search-term%', '%search-term%']) } + + # tests simple search with two default columns with quotes + it { expect(described_class.new('"search-term"', [:'players.name', :'players.tags', 'players.id'], [:'players.name', :'players.tags']).where_clause).to eq(['(CAST(players.name AS TEXT) ILIKE ? OR CAST(players.tags AS TEXT) ILIKE ?)', '%search-term%', '%search-term%']) } + + # tests simple search with three default columns without quotes + it { expect(described_class.new('search-term', [:'players.name', :'players.tags', 'players.id'], [:'players.name', :'players.tags', :'players.id']).where_clause).to eq(['(CAST(players.name AS TEXT) ILIKE ? OR CAST(players.tags AS TEXT) ILIKE ? OR CAST(players.id AS TEXT) ILIKE ?)', '%search-term%', '%search-term%', '%search-term%']) } # tests simple search term with column name and without quotes it { expect(described_class.new('name:ab', [:"players.name"], :"players.name").where_clause).to eq(['CAST(players.name AS TEXT) ILIKE ?', '%ab%']) } @@ -13,22 +22,22 @@ describe TextToSqlQuery do it { expect{described_class.new('unknown:ab', [:"players.name"], :"players.name").where_clause}.to raise_error(RuntimeError, "Unknown field 'unknown'") } # tests simple search term without column name and with quotes - it { expect(described_class.new('"ab"', [:"players.name", :"players.device_id"], :"players.device_id").where_clause).to eq(["CAST(players.device_id AS TEXT) ILIKE ?", "%ab%"]) } + it { expect(described_class.new('"ab"', [:"players.name", :"players.device_id"], :"players.device_id").where_clause).to eq(["(CAST(players.device_id AS TEXT) ILIKE ?)", "%ab%"]) } # tests simple search term with column name and with quotes it { expect(described_class.new('tags:"ab"', [:"players.name", :"players.tags"], :"players.device_id").where_clause).to eq(["CAST(players.tags AS TEXT) ILIKE ?", "%ab%"]) } # tests search without operators - it { expect(described_class.new('123 456', [:"players.name", :"players.device_id"], :"players.device_id").where_clause).to eq(["(CAST(players.device_id AS TEXT) ILIKE ? OR CAST(players.device_id AS TEXT) ILIKE ?)", "%123%", "%456%"]) } + it { expect(described_class.new('123 456', [:"players.name", :"players.device_id"], :"players.device_id").where_clause).to eq(["((CAST(players.device_id AS TEXT) ILIKE ?) OR (CAST(players.device_id AS TEXT) ILIKE ?))", "%123%", "%456%"]) } # tests search with OR operator - it { expect(described_class.new('123 or 456', [:"players.name", :"players.device_id"], :"players.device_id").where_clause).to eq(["(CAST(players.device_id AS TEXT) ILIKE ? OR CAST(players.device_id AS TEXT) ILIKE ?)", "%123%", "%456%"]) } + it { expect(described_class.new('123 or 456', [:"players.name", :"players.device_id"], :"players.device_id").where_clause).to eq(["((CAST(players.device_id AS TEXT) ILIKE ?) OR (CAST(players.device_id AS TEXT) ILIKE ?))", "%123%", "%456%"]) } # tests search with AND operator - it { expect(described_class.new('123 and 456', [:"players.name", :"players.device_id"], :"players.device_id").where_clause).to eq(["(CAST(players.device_id AS TEXT) ILIKE ? AND CAST(players.device_id AS TEXT) ILIKE ?)", "%123%", "%456%"]) } + it { expect(described_class.new('123 and 456', [:"players.name", :"players.device_id"], :"players.device_id").where_clause).to eq(["((CAST(players.device_id AS TEXT) ILIKE ?) AND (CAST(players.device_id AS TEXT) ILIKE ?))", "%123%", "%456%"]) } # tests search with NOT operator on default column - it { expect(described_class.new('not 23', [:"players.name", :"players.device_id"], :"players.device_id").where_clause).to eq(["NOT CAST(players.device_id AS TEXT) ILIKE ?", "%23%"]) } + it { expect(described_class.new('not 23', [:"players.name", :"players.device_id"], :"players.device_id").where_clause).to eq(["NOT (CAST(players.device_id AS TEXT) ILIKE ?)", "%23%"]) } # tests search with NOT operator on non-default column it { expect(described_class.new('not value:23', [:"players.name", :"players.value"], :"players.device_id").where_clause).to eq(["NOT CAST(players.value AS TEXT) ILIKE ?", "%23%"]) } @@ -55,12 +64,15 @@ describe TextToSqlQuery do it { expect(described_class.new('tags:h1-r', [:'players.name', :'players.tags', :'players.device_id'], :"players.device_id", { tags: "tags.value" }).where_clause).to eq(["CAST(tags.value AS TEXT) ILIKE ?", "%h1-r%"]) } # tests complex query - it { expect(described_class.new('(device_id:"with space" tags:mta no-quotes-id-123) or "id with quotes-5" and ( ("id with q 10" or "id with q 20") and ("id with Q 30" "id with Q 40") and not id-without-Q-50)', [:'players.name', :'players.value', :'players.device_id'], :"players.device_id", { tags: 'tags.name' }).where_clause).to eq(['((CAST(players.device_id AS TEXT) ILIKE ? OR (CAST(tags.name AS TEXT) ILIKE ? OR CAST(players.device_id AS TEXT) ILIKE ?)) OR (CAST(players.device_id AS TEXT) ILIKE ? AND (((CAST(players.device_id AS TEXT) ILIKE ? OR CAST(players.device_id AS TEXT) ILIKE ?) AND (CAST(players.device_id AS TEXT) ILIKE ? OR CAST(players.device_id AS TEXT) ILIKE ?)) AND NOT CAST(players.device_id AS TEXT) ILIKE ?)))', '%with space%', '%mta%', '%no-quotes-id-123%', '%id with quotes-5%', '%id with q 10%', '%id with q 20%', '%id with Q 30%', '%id with Q 40%', '%id-without-Q-50%']) } + it { expect(described_class.new('(device_id:"with space" tags:mta no-quotes-id-123) or "id with quotes-5" and ( ("id with q 10" or "id with q 20") and ("id with Q 30" "id with Q 40") and not id-without-Q-50)', [:'players.name', :'players.value', :'players.device_id'], :"players.device_id", { tags: 'tags.name' }).where_clause).to eq(['((CAST(players.device_id AS TEXT) ILIKE ? OR (CAST(tags.name AS TEXT) ILIKE ? OR (CAST(players.device_id AS TEXT) ILIKE ?))) OR ((CAST(players.device_id AS TEXT) ILIKE ?) AND ((((CAST(players.device_id AS TEXT) ILIKE ?) OR (CAST(players.device_id AS TEXT) ILIKE ?)) AND ((CAST(players.device_id AS TEXT) ILIKE ?) OR (CAST(players.device_id AS TEXT) ILIKE ?))) AND NOT (CAST(players.device_id AS TEXT) ILIKE ?))))', '%with space%', '%mta%', '%no-quotes-id-123%', '%id with quotes-5%', '%id with q 10%', '%id with q 20%', '%id with Q 30%', '%id with Q 40%', '%id-without-Q-50%']) } + + # tests mixed query with and without column names and with multiple default columns + it { expect(described_class.new('("green hunter" and device_id:100) or ("blue bird" and not device_id:200)', [:'players.name', :'players.value', :'players.device_id'], [:"players.name", :"players.tags"]).where_clause).to eq(['(((CAST(players.name AS TEXT) ILIKE ? OR CAST(players.tags AS TEXT) ILIKE ?) AND CAST(players.device_id AS TEXT) ILIKE ?) OR ((CAST(players.name AS TEXT) ILIKE ? OR CAST(players.tags AS TEXT) ILIKE ?) AND NOT CAST(players.device_id AS TEXT) ILIKE ?))', '%green hunter%', '%green hunter%', '%100%', '%blue bird%', '%blue bird%', '%200%']) } # tests query with multiple search terms with mixed and-or-not after dash and underscore - it { expect(described_class.new('123-and-456 -or-2 -not_not_1', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id').where_clause).to eq(['(CAST(players.device_id AS TEXT) ILIKE ? OR (CAST(players.device_id AS TEXT) ILIKE ? OR CAST(players.device_id AS TEXT) ILIKE ?))', '%123-and-456%', '%-or-2%', '%-not\_not\_1%'])} + it { expect(described_class.new('123-and-456 -or-2 -not_not_1', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id').where_clause).to eq(['((CAST(players.device_id AS TEXT) ILIKE ?) OR ((CAST(players.device_id AS TEXT) ILIKE ?) OR (CAST(players.device_id AS TEXT) ILIKE ?)))', '%123-and-456%', '%-or-2%', '%-not\_not\_1%'])} # tests query with multiple search terms with mixed and-or-not after dash and underscore - it { expect(described_class.new('andrew or ornela', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id').where_clause).to eq(['(CAST(players.device_id AS TEXT) ILIKE ? OR CAST(players.device_id AS TEXT) ILIKE ?)', '%andrew%', '%ornela%'])} + it { expect(described_class.new('andrew or ornela', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id').where_clause).to eq(['((CAST(players.device_id AS TEXT) ILIKE ?) OR (CAST(players.device_id AS TEXT) ILIKE ?))', '%andrew%', '%ornela%'])} end end -- 2.47.3 From fc5fdd3db27abe5d3dffb93c47b624381b150b57 Mon Sep 17 00:00:00 2001 From: Bilal Date: Thu, 16 Apr 2020 15:28:28 +0200 Subject: [PATCH 03/15] add model with two default fields --- spec/support/models.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spec/support/models.rb b/spec/support/models.rb index 58d632c..58daa0f 100755 --- a/spec/support/models.rb +++ b/spec/support/models.rb @@ -5,6 +5,12 @@ class VectorModel < ActiveRecord::Base pg_search fields: %i[vector_models.id vector_models.name vector_models.value], cache: :search_cache end +class VectorModelWithTwoDefaultColumns < ActiveRecord::Base + self.table_name = :vector_models + include PgSearchable + pg_search fields: %i[vector_models.id vector_models.name vector_models.value], default_fields: %i[vector_models.name vector_models.value], cache: :search_cache +end + class VectorModelWithMappings < ActiveRecord::Base self.table_name = :vector_models include PgSearchable -- 2.47.3 From a54b2586ec48e31e616e401b69ac6d513478f42e Mon Sep 17 00:00:00 2001 From: Bilal Date: Thu, 16 Apr 2020 15:30:01 +0200 Subject: [PATCH 04/15] update PgSearchable to accept array of default columns --- lib/pg_searchable_regex.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/pg_searchable_regex.rb b/lib/pg_searchable_regex.rb index 8c7374b..f14e120 100644 --- a/lib/pg_searchable_regex.rb +++ b/lib/pg_searchable_regex.rb @@ -26,7 +26,7 @@ module PgSearchable wildcard: true, external_cache_data: nil, joins: [], - default_field: "" + default_fields: [] ) @ts_search_fields = fields @ts_search_fields_mappings = fields_mappings @@ -36,7 +36,11 @@ module PgSearchable @ts_skip_cache_update = skip_callback @ts_wildcard = wildcard @ts_joins = joins - @default_field = default_field.to_s.empty? ? fields.first : default_field.to_sym + @default_fields = if default_fields.is_a? Array + default_fields.empty? ? [fields.first] : default_fields + else + default_fields.to_s.empty? ? [fields.first] : [default_fields.to_sym] + end ts_add_scope end @@ -52,7 +56,7 @@ module PgSearchable def ts_search(value) return if @ts_search_fields.blank? || value.blank? includes(@ts_joins).references(:all).where( - TextToSqlQuery.new(value, @ts_search_fields, @default_field, @ts_search_fields_mappings).where_clause).distinct + TextToSqlQuery.new(value, @ts_search_fields, @default_fields, @ts_search_fields_mappings).where_clause).distinct end def should_update_cache_field? -- 2.47.3 From 18f6875b54a1ee913134ec15c08c015b556c5793 Mon Sep 17 00:00:00 2001 From: Bilal Date: Thu, 16 Apr 2020 15:31:50 +0200 Subject: [PATCH 05/15] add specs to search multiple default columns --- spec/lib/pg_searchable_new_spec.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/spec/lib/pg_searchable_new_spec.rb b/spec/lib/pg_searchable_new_spec.rb index 2bf7fd6..953f194 100644 --- a/spec/lib/pg_searchable_new_spec.rb +++ b/spec/lib/pg_searchable_new_spec.rb @@ -47,6 +47,20 @@ describe PgSearchable do expect(VectorModel.scope_search("#{record2.id}")).to contain_exactly(record2) end + it 'can search multiple default columns if no column name is used with single search term' do + records = VectorModelWithTwoDefaultColumns.create [{ name: 'hamo', value: '5' }, { name: 'meho', value: '20 hamo' }, { name: 'munja-5', value: '300' }] + + expect(VectorModelWithTwoDefaultColumns.scope_search('name:hamo')).to contain_exactly(records[0]) + expect(VectorModelWithTwoDefaultColumns.scope_search('hamo')).to contain_exactly(records[0], records[1]) + expect(VectorModelWithTwoDefaultColumns.scope_search('5')).to contain_exactly(records[0], records[2]) + end + + it 'can search multiple default columns if no column name is used in query containing multiple search terms' do + records = VectorModelWithTwoDefaultColumns.create [{ name: 'hamo', value: '9' }, { name: 'meho', value: '5' }, { name: 'oko-9', value: '100' }] + + expect(VectorModelWithTwoDefaultColumns.scope_search('(9 and not name:hamo) or meho')).to contain_exactly(records[2], records[1]) + end + it 'searches column declared in search term' do record1 = VectorModel.create name: 'hamo', value: '-45' record2 = VectorModel.create name: 'meho', value: '120' @@ -144,8 +158,6 @@ describe PgSearchable do expect(DynamicModelWithTagValues.scope_search('tag:green or value:"not"')).to contain_exactly(record1, record2) end - - it 'can search in referenced column and in model columns with multiple search terms connected with logical operators' do record1 = DynamicModelWithTagValues.create name: 'something', value: 'amazing' record2 = DynamicModelWithTagValues.create name: 'new record', value: 'not so amazing' -- 2.47.3 From f4bec8516124d65025a137eb17a66a1786544233 Mon Sep 17 00:00:00 2001 From: Bilal Date: Mon, 20 Apr 2020 19:33:29 +0200 Subject: [PATCH 06/15] change ActiveRecord joins query with custom generated SQL --- lib/pg_searchable_regex.rb | 12 +++++-- lib/text_to_sql_query.rb | 64 +++++++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/lib/pg_searchable_regex.rb b/lib/pg_searchable_regex.rb index f14e120..d1a4ab5 100644 --- a/lib/pg_searchable_regex.rb +++ b/lib/pg_searchable_regex.rb @@ -55,8 +55,16 @@ module PgSearchable def ts_search(value) return if @ts_search_fields.blank? || value.blank? - includes(@ts_joins).references(:all).where( - TextToSqlQuery.new(value, @ts_search_fields, @default_fields, @ts_search_fields_mappings).where_clause).distinct + model = ancestors.first + sql_query_object = TextToSqlQuery.new( + value, + @ts_search_fields, + @default_field, + @ts_search_fields_mappings, + @ts_joins, + model + ) + joins(sql_query_object.join_clause).where(sql_query_object.where_clause).distinct end def should_update_cache_field? diff --git a/lib/text_to_sql_query.rb b/lib/text_to_sql_query.rb index 2c5e2eb..416d182 100644 --- a/lib/text_to_sql_query.rb +++ b/lib/text_to_sql_query.rb @@ -1,7 +1,7 @@ require_relative 'parser' class TextToSqlQuery - def initialize(text, fields, default_fields, fields_mappings = {}) + def initialize(text, fields, default_fields, fields_mappings = {}, joins = [], model = nil) @text = text.to_s.strip @fields = fields.map(&:to_sym) @@ -20,6 +20,8 @@ class TextToSqlQuery fields_mappings.each do |field, value| @fields_mappings[field] = value if @fields_mappings[field] end + @joins = joins + @model = model end def where_clause @@ -28,6 +30,20 @@ class TextToSqlQuery generate_sql @parsed_tree end + def join_clause + return if @joins.empty? + + table_column_mappings + model_association_mappings + + join_clause_part = '' + @joins.each do |join| + join_sql_part = generate_join_sql_part_for join + join_clause_part += join_sql_part + end + join_clause_part + end + private def generate_sql(tree) @@ -104,4 +120,50 @@ class TextToSqlQuery result.gsub!(/%/, '\%') result end + + def table_column_mappings + @table_column_mappings = {} + @fields_mappings.each_value do |table_with_column| + split_names = table_with_column.to_s.split '.' + table_name = split_names.first + column_name = split_names.second + @table_column_mappings[table_name] = [] if @table_column_mappings[table_name].nil? + @table_column_mappings[table_name] << column_name + end + @table_column_mappings + end + + def model_association_mappings + @model_associations = {} + @model.reflect_on_all_associations.each do |association| + name = association.name + + @model_associations[name] = { + option_as: association.options[:as] || name, + type: association.type + } + end + @model_associations + end + + def generate_join_sql_part_for(join) + association_data = @model_associations[join] + join_table_name = join.to_s + raise "Join table #{join_table_name} has no association data" if association_data.nil? + + select_sql_part = '' + columns_for_table = @table_column_mappings[join_table_name] || [] + + # TODO: Can be optimized - do not include columns that are not referenced in user query + columns_for_table.each do |column_name| + select_sql_part += "string_agg(#{column_name}, '') AS #{column_name}, " + end + + option_as = association_data[:option_as] + type = association_data[:type] + model_name = @model.to_s + table_name = @model.table_name + + "LEFT JOIN (SELECT #{option_as}_id, #{select_sql_part} #{type} FROM #{join_table_name} GROUP BY #{option_as}_id, #{type}) #{join_table_name} on #{join_table_name}.#{option_as}_id = #{table_name}.id AND #{join_table_name}.#{type} = '#{model_name}'" + end end -- 2.47.3 From 455845e36b7528de389c83995ddb927ce511a20e Mon Sep 17 00:00:00 2001 From: Bilal Date: Mon, 20 Apr 2020 22:21:26 +0200 Subject: [PATCH 07/15] update specs; add new specs to test for reported bug related to AND query in referenced table --- spec/lib/pg_searchable_new_spec.rb | 147 ++++++++++++++++++++++++----- spec/schema.rb | 2 + spec/support/models.rb | 16 ++++ 3 files changed, 140 insertions(+), 25 deletions(-) diff --git a/spec/lib/pg_searchable_new_spec.rb b/spec/lib/pg_searchable_new_spec.rb index 953f194..1f2c8cc 100644 --- a/spec/lib/pg_searchable_new_spec.rb +++ b/spec/lib/pg_searchable_new_spec.rb @@ -158,35 +158,132 @@ describe PgSearchable do expect(DynamicModelWithTagValues.scope_search('tag:green or value:"not"')).to contain_exactly(record1, record2) end - it 'can search in referenced column and in model columns with multiple search terms connected with logical operators' do - record1 = DynamicModelWithTagValues.create name: 'something', value: 'amazing' - record2 = DynamicModelWithTagValues.create name: 'new record', value: 'not so amazing' - record3 = DynamicModelWithTagValues.create name: 'last one', value: 'no value' - record4 = DynamicModelWithTagValues.create name: 'really last one', value: 'no value' - Tag.create(taggable: record1, value: 'red') - Tag.create(taggable: record1, value: 'green') - Tag.create(taggable: record2, value: 'black') - Tag.create(taggable: record3, value: '-12') - Tag.create(taggable: record4, value: '-') + describe 'searching in model with has_many association' do + before do + records = DynamicModelWithTagValues.create [{ name: 'something', value: 'amazing' }, + { name: 'new record', value: 'not so amazing' }, + { name: 'last one', value: 'no value' }, + { name: 'really last one', value: 'no value' }] - expect(DynamicModelWithTagValues.scope_search('tag:red or tag:black')).to contain_exactly(record1, record2) - expect(DynamicModelWithTagValues.scope_search('tag:red and tag:black')).to be_empty - expect(DynamicModelWithTagValues.scope_search('tag:red or tag:green')).to contain_exactly(record1) - expect(DynamicModelWithTagValues.scope_search('not tag:-12 and not value:amazing')).to contain_exactly(record4) + Tag.create [{ taggable: records[0], value: 'red', custom_attribute: 'rose' }, + { taggable: records[0], value: 'green', custom_attribute: 'garden' }, + { taggable: records[1], value: 'black', custom_attribute: 'sky' }, + { taggable: records[2], value: '-1/12', custom_attribute: 'gold nugget' }, + { taggable: records[3], value: '-', custom_attribute: 'unknown' }, + { taggable: records[3], value: 'red-green', custom_attribute: 'unicorn' }] + end + + it 'can search with multiple search terms connected with OR operator' do + records = DynamicModelWithTagValues.all + + expect(DynamicModelWithTagValues.scope_search('tag:red or tag:black')).to contain_exactly(records[0], records[1], records[3]) + expect(DynamicModelWithTagValues.scope_search('tag:red or tag:green')).to contain_exactly(records[0], records[3]) + end + + it 'can search with multiple search terms connected with AND operator' do + records = DynamicModelWithTagValues.all + + expect(DynamicModelWithTagValues.scope_search('tag:red and tag:black')).to be_empty + expect(DynamicModelWithTagValues.scope_search('tag:red and tag:green')).to contain_exactly(records[0], records[3]) + end + + it 'can search with multiple search terms and containing NOT' do + records = DynamicModelWithTagValues.all + + expect(DynamicModelWithTagValues.scope_search('not tag:"-1/12" and not value:amazing')).to contain_exactly(records[3]) + end + + it 'can search in referenced column and in model columns with multiple search terms connected with logical operators and with brackets' do + records = DynamicModelWithTagValues.all + + expect(DynamicModelWithTagValues.scope_search('(tag:- and not tag:12) or (value:"amazing" and not value:"not")')).to contain_exactly(records[0], records[3]) + end end - it 'can search in referenced column and in model columns with multiple search terms connected with logical operators and with brackets' do - record1 = DynamicModelWithTagValues.create name: 'something', value: 'amazing' - record2 = DynamicModelWithTagValues.create name: 'new record', value: 'not so amazing' - record3 = DynamicModelWithTagValues.create name: 'last one', value: 'no value' - record4 = DynamicModelWithTagValues.create name: 'really last one', value: 'no value' - Tag.create(taggable: record1, value: 'red') - Tag.create(taggable: record1, value: 'green') - Tag.create(taggable: record2, value: 'black') - Tag.create(taggable: record3, value: '-12') - Tag.create(taggable: record4, value: '-') + describe 'searching through multiple referenced columns in model with has_many association' do + before do + records = DynamicModelWithTagValuesAndCustomAttribute.create [{ name: 'something', value: 'amazing' }, + { name: 'new record', value: 'not so amazing' }, + { name: 'last one', value: 'no value' }, + { name: 'really last one', value: 'no value' }] - expect(DynamicModelWithTagValues.scope_search('(tag:- and not tag:12) or (value:"amazing" and not value:"not")')).to contain_exactly(record1, record4) + Tag.create [{ taggable: records[0], value: 'red', custom_attribute: 'rose' }, + { taggable: records[0], value: 'green', custom_attribute: 'garden' }, + { taggable: records[1], value: 'black', custom_attribute: 'sky' }, + { taggable: records[2], value: '-1/12', custom_attribute: 'gold nugget' }, + { taggable: records[3], value: '-', custom_attribute: 'unknown' }, + { taggable: records[3], value: 'red-green', custom_attribute: 'unicorn' }] + end + + it 'can search with multiple search terms connected with OR operator' do + records = DynamicModelWithTagValuesAndCustomAttribute.all + + expect(DynamicModelWithTagValuesAndCustomAttribute.scope_search('tag:red or custom:sky')).to contain_exactly(records[0], records[1], records[3]) + end + + it 'can search with multiple search terms connected with AND operator' do + records = DynamicModelWithTagValuesAndCustomAttribute.all + + expect(DynamicModelWithTagValuesAndCustomAttribute.scope_search('custom:unicorn and tag:black')).to be_empty + expect(DynamicModelWithTagValuesAndCustomAttribute.scope_search('tag:"-1/12" and custom:gold')).to contain_exactly(records[2]) + end + + it 'can search with multiple search terms and containing NOT' do + expect(DynamicModelWithTagValuesAndCustomAttribute.scope_search('not tag:"-1/12" and not value:amazing and not custom:unknown')).to be_empty + end + + it 'can search with multiple search terms connected with logical operators and with brackets' do + records = DynamicModelWithTagValuesAndCustomAttribute.all + + expect(DynamicModelWithTagValuesAndCustomAttribute.scope_search('(tag:- and not tag:12) or (value:"amazing" and not value:"not") or (custom:unknown or custom:rose)')).to contain_exactly(records[0], records[3]) + end + end + + describe 'searching in model with multiple has_many associations' do + before do + records = DynamicModelWithTagAndCategories.create [{ name: 'something', value: 'amazing' }, + { name: 'new record', value: 'not so amazing' }, + { name: 'last one', value: 'no value' }, + { name: 'really last one', value: 'no value' }] + + Tag.create [{ taggable: records[0], value: 'red', custom_attribute: 'rose' }, + { taggable: records[0], value: 'green', custom_attribute: 'garden' }, + { taggable: records[1], value: 'black', custom_attribute: 'sky' }, + { taggable: records[2], value: '-1/12', custom_attribute: 'gold nugget' }, + { taggable: records[3], value: '-', custom_attribute: 'unknown' }, + { taggable: records[3], value: 'red-green', custom_attribute: 'unicorn' }] + + Category.create [{ categoriable: records[0], name: 'home' }, + { categoriable: records[0], name: 'home' }, + { categoriable: records[1], name: 'world' }, + { categoriable: records[2], name: 'math' }, + { categoriable: records[3], name: 'unknown' }, + { categoriable: records[3], name: 'myth' }] + + end + + it 'can search with multiple search terms connected with OR operator' do + records = DynamicModelWithTagAndCategories.all + + expect(DynamicModelWithTagAndCategories.scope_search('tag:red or category:math')).to contain_exactly(records[0], records[2], records[3]) + end + + it 'can search with multiple search terms connected with AND operator' do + records = DynamicModelWithTagAndCategories.all + + expect(DynamicModelWithTagAndCategories.scope_search('category:home and tag:-')).to be_empty + expect(DynamicModelWithTagAndCategories.scope_search('tag:"-1/12" and category:math')).to contain_exactly(records[2]) + end + + it 'can search with multiple search terms and containing NOT' do + expect(DynamicModelWithTagAndCategories.scope_search('not tag:"-1/12" and not value:amazing and not category:unknown')).to be_empty + end + + it 'can search with multiple search terms connected with logical operators and with brackets' do + records = DynamicModelWithTagAndCategories.all + + expect(DynamicModelWithTagAndCategories.scope_search('(tag:- and not tag:12) or (value:"amazing" and not value:"not") or (category:unknown or category:math)')).to contain_exactly(records[0], records[2], records[3]) + end end end end diff --git a/spec/schema.rb b/spec/schema.rb index 0ddb2b9..ae3e2f2 100644 --- a/spec/schema.rb +++ b/spec/schema.rb @@ -27,6 +27,7 @@ ActiveRecord::Schema.define do create_table :tags, force: true do |t| t.string :value + t.string :custom_attribute t.references :category, index: true t.references :taggable, polymorphic: true, index: true t.timestamps null: false @@ -34,6 +35,7 @@ ActiveRecord::Schema.define do create_table :categories, force: true do |t| t.string :name + t.references :categoriable, polymorphic: true, index: true t.timestamps null: false end diff --git a/spec/support/models.rb b/spec/support/models.rb index 58daa0f..5dd6f36 100755 --- a/spec/support/models.rb +++ b/spec/support/models.rb @@ -82,6 +82,21 @@ class DynamicModelWithTagValues < ActiveRecord::Base has_many :tags, as: :taggable end +class DynamicModelWithTagAndCategories < ActiveRecord::Base + self.table_name = :dynamic_models + include PgSearchable + pg_search fields: %i[dynamic_models.id dynamic_models.name dynamic_models.value], fields_mappings: {tag: 'tags.value', category: 'categories.name'}, joins: [:tags, :categories] + has_many :tags, as: :taggable + has_many :categories, as: :categoriable +end + +class DynamicModelWithTagValuesAndCustomAttribute < ActiveRecord::Base + self.table_name = :dynamic_models + include PgSearchable + pg_search fields: %i[dynamic_models.id dynamic_models.name dynamic_models.value], fields_mappings: {tag: 'tags.value', custom: 'tags.custom_attribute'}, joins: [:tags] + has_many :tags, as: :taggable +end + class DynamicModelWithCategory < ActiveRecord::Base self.table_name = :dynamic_models include PgSearchable @@ -112,4 +127,5 @@ end class Category < ActiveRecord::Base has_many :tags + belongs_to :categoriable, polymorphic: true end -- 2.47.3 From 0de540c5bd316ad9b67ed1b6eeeb760965c9a1d8 Mon Sep 17 00:00:00 2001 From: Bilal Date: Fri, 24 Apr 2020 11:40:53 +0200 Subject: [PATCH 08/15] fix bug after rebase --- lib/pg_searchable_regex.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pg_searchable_regex.rb b/lib/pg_searchable_regex.rb index d1a4ab5..576e9e5 100644 --- a/lib/pg_searchable_regex.rb +++ b/lib/pg_searchable_regex.rb @@ -59,7 +59,7 @@ module PgSearchable sql_query_object = TextToSqlQuery.new( value, @ts_search_fields, - @default_field, + @default_fields, @ts_search_fields_mappings, @ts_joins, model -- 2.47.3 From fe96f682db4ecbe14032ca7430528fda6939226a Mon Sep 17 00:00:00 2001 From: Bilal Date: Wed, 29 Apr 2020 07:21:39 +0200 Subject: [PATCH 09/15] start using HAVING instead of manually building joins --- lib/pg_searchable_regex.rb | 6 ++-- lib/text_to_sql_query.rb | 65 +++----------------------------------- 2 files changed, 7 insertions(+), 64 deletions(-) diff --git a/lib/pg_searchable_regex.rb b/lib/pg_searchable_regex.rb index 576e9e5..1fd390b 100644 --- a/lib/pg_searchable_regex.rb +++ b/lib/pg_searchable_regex.rb @@ -55,16 +55,14 @@ module PgSearchable def ts_search(value) return if @ts_search_fields.blank? || value.blank? - model = ancestors.first sql_query_object = TextToSqlQuery.new( value, @ts_search_fields, @default_fields, @ts_search_fields_mappings, - @ts_joins, - model + @ts_joins ) - joins(sql_query_object.join_clause).where(sql_query_object.where_clause).distinct + distinct.joins(sql_query_object.join_clause).group(:id).having(sql_query_object.where_clause) end def should_update_cache_field? diff --git a/lib/text_to_sql_query.rb b/lib/text_to_sql_query.rb index 416d182..a2dc9bd 100644 --- a/lib/text_to_sql_query.rb +++ b/lib/text_to_sql_query.rb @@ -1,7 +1,7 @@ require_relative 'parser' class TextToSqlQuery - def initialize(text, fields, default_fields, fields_mappings = {}, joins = [], model = nil) + def initialize(text, fields, default_fields, fields_mappings = {}, joins = []) @text = text.to_s.strip @fields = fields.map(&:to_sym) @@ -21,7 +21,6 @@ class TextToSqlQuery @fields_mappings[field] = value if @fields_mappings[field] end @joins = joins - @model = model end def where_clause @@ -31,20 +30,12 @@ class TextToSqlQuery end def join_clause - return if @joins.empty? + return nil if @joins.empty? - table_column_mappings - model_association_mappings - - join_clause_part = '' - @joins.each do |join| - join_sql_part = generate_join_sql_part_for join - join_clause_part += join_sql_part - end - join_clause_part + return *@joins end - private +private def generate_sql(tree) first_key = tree.keys.first @@ -83,7 +74,7 @@ class TextToSqlQuery if mapping.nil? raise "Unknown field '#{first_key.to_s}'" else - ["CAST(#{mapping.to_s} AS TEXT) ILIKE ?", "%#{escaped_node_value}%"] + ["STRING_AGG(CAST(#{mapping.to_s} AS TEXT), '') ILIKE ?", "%#{escaped_node_value}%"] end end end @@ -120,50 +111,4 @@ class TextToSqlQuery result.gsub!(/%/, '\%') result end - - def table_column_mappings - @table_column_mappings = {} - @fields_mappings.each_value do |table_with_column| - split_names = table_with_column.to_s.split '.' - table_name = split_names.first - column_name = split_names.second - @table_column_mappings[table_name] = [] if @table_column_mappings[table_name].nil? - @table_column_mappings[table_name] << column_name - end - @table_column_mappings - end - - def model_association_mappings - @model_associations = {} - @model.reflect_on_all_associations.each do |association| - name = association.name - - @model_associations[name] = { - option_as: association.options[:as] || name, - type: association.type - } - end - @model_associations - end - - def generate_join_sql_part_for(join) - association_data = @model_associations[join] - join_table_name = join.to_s - raise "Join table #{join_table_name} has no association data" if association_data.nil? - - select_sql_part = '' - columns_for_table = @table_column_mappings[join_table_name] || [] - - # TODO: Can be optimized - do not include columns that are not referenced in user query - columns_for_table.each do |column_name| - select_sql_part += "string_agg(#{column_name}, '') AS #{column_name}, " - end - - option_as = association_data[:option_as] - type = association_data[:type] - model_name = @model.to_s - table_name = @model.table_name - - "LEFT JOIN (SELECT #{option_as}_id, #{select_sql_part} #{type} FROM #{join_table_name} GROUP BY #{option_as}_id, #{type}) #{join_table_name} on #{join_table_name}.#{option_as}_id = #{table_name}.id AND #{join_table_name}.#{type} = '#{model_name}'" - end end -- 2.47.3 From e0fae505849827f186e945dba93f704c278b9c3e Mon Sep 17 00:00:00 2001 From: Bilal Date: Wed, 29 Apr 2020 07:22:03 +0200 Subject: [PATCH 10/15] add new tables and models for join through association --- spec/lib/pg_searchable_new_spec.rb | 74 +++++++++++++++++++++++++++--- spec/schema.rb | 15 ++++++ spec/support/models.rb | 18 ++++++++ 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/spec/lib/pg_searchable_new_spec.rb b/spec/lib/pg_searchable_new_spec.rb index 1f2c8cc..4084fc4 100644 --- a/spec/lib/pg_searchable_new_spec.rb +++ b/spec/lib/pg_searchable_new_spec.rb @@ -161,9 +161,9 @@ describe PgSearchable do describe 'searching in model with has_many association' do before do records = DynamicModelWithTagValues.create [{ name: 'something', value: 'amazing' }, - { name: 'new record', value: 'not so amazing' }, - { name: 'last one', value: 'no value' }, - { name: 'really last one', value: 'no value' }] + { name: 'new record', value: 'not so amazing' }, + { name: 'last one', value: 'no value' }, + { name: 'really last one', value: 'no value' }] Tag.create [{ taggable: records[0], value: 'red', custom_attribute: 'rose' }, { taggable: records[0], value: 'green', custom_attribute: 'garden' }, @@ -242,9 +242,9 @@ describe PgSearchable do describe 'searching in model with multiple has_many associations' do before do records = DynamicModelWithTagAndCategories.create [{ name: 'something', value: 'amazing' }, - { name: 'new record', value: 'not so amazing' }, - { name: 'last one', value: 'no value' }, - { name: 'really last one', value: 'no value' }] + { name: 'new record', value: 'not so amazing' }, + { name: 'last one', value: 'no value' }, + { name: 'really last one', value: 'no value' }] Tag.create [{ taggable: records[0], value: 'red', custom_attribute: 'rose' }, { taggable: records[0], value: 'green', custom_attribute: 'garden' }, @@ -285,6 +285,68 @@ describe PgSearchable do expect(DynamicModelWithTagAndCategories.scope_search('(tag:- and not tag:12) or (value:"amazing" and not value:"not") or (category:unknown or category:math)')).to contain_exactly(records[0], records[2], records[3]) end end + + describe 'joins with through association' do + let(:players) { Player.all } + + before do + players = Player.create([{ name: 'first', value: 'downtown' }, + { name: 'redliner', value: 'uptown' }, + { name: 'flying', value: 'eagle' }, + { name: 'blue', value: 'marine' }]) + + + ptags = Ptag.create [{ value: 'e-ink' }, + { value: 'black&white' }, + { value: 'rich' }, + { value: 'grayscale' }, + { value: 'LED' }] + + players[0].ptags << ptags[0..1] + players[1].ptags << ptags[0..2] + players[2].ptags << ptags[2..3] + players[3].ptags << ptags[4] + end + + it 'can search in default column of model with through association' do + expect(Player.scope_search('l and not y and not blue')).to contain_exactly(players[1]) + end + + it 'can search in referenced column associated with through relation - simple search' do + expect(Player.scope_search('tag:e-ink')).to contain_exactly(players[0], players[1]) + end + + it 'can search in referenced column associated with through relaction - simple search with OR operator' do + expect(Player.scope_search('tag:e-ink OR tag:rich')).to contain_exactly(players[0], players[1], players[2]) + end + + it 'can search in referenced column associated with through relaction - simple search with AND operator' do + expect(Player.scope_search('tag:e-ink AND tag:rich')).to contain_exactly(players[1]) + end + + it 'can search in referenced column associated with through relaction - simple search with NOT operator' do + puts "===========" + puts Player.scope_search('NOT tag:e-ink').inspect + puts "===============" + expect(Player.scope_search('NOT tag:e-ink')).to contain_exactly(players[2], players[3]) + end + + it 'can search in referenced column associated with through relaction - mixed search terms with OR' do + expect(Player.scope_search('tag:e-ink OR blue')).to contain_exactly(players[0], players[1], players[3]) + end + + it 'can search in referenced column associated with through relaction - mixed search terms with AND' do + expect(Player.scope_search('tag:e-ink AND first')).to contain_exactly(players[0]) + end + + it 'can search in referenced column associated with through relaction - mixed search terms with OR NOT' do + expect(Player.scope_search('tag:led OR NOT first')).to contain_exactly(players[1], players[2], players[3]) + end + + it 'can search in referenced column associated with through relaction - mixed search terms with AND NOT' do + expect(Player.scope_search('tag:e-ink AND NOT first')).to contain_exactly(players[1]) + end + end end end end diff --git a/spec/schema.rb b/spec/schema.rb index ae3e2f2..8c20819 100644 --- a/spec/schema.rb +++ b/spec/schema.rb @@ -44,4 +44,19 @@ ActiveRecord::Schema.define do t.string :name t.timestamps null: false end + + create_table :players, force: true do |t| + t.string :name + t.string :value + t.timestamps null: false + end + + create_table :ptags, force: true do |t| + t.string :value + end + + create_table :taggings, force: true do |t| + t.belongs_to :player + t.belongs_to :ptag + end end diff --git a/spec/support/models.rb b/spec/support/models.rb index 5dd6f36..721b80f 100755 --- a/spec/support/models.rb +++ b/spec/support/models.rb @@ -129,3 +129,21 @@ class Category < ActiveRecord::Base has_many :tags belongs_to :categoriable, polymorphic: true end + +class Player < ActiveRecord::Base + include PgSearchable + pg_search fields: %i[players.id players.name players.value], default_fields: [:name], fields_mappings: { tag: 'ptags.value' }, joins: [:ptags] + has_many :taggings + has_many :ptags, through: :taggings +end + +class Ptag < ActiveRecord::Base + self.table_name = :ptags + has_many :taggings + has_many :players, through: :taggings +end + +class Tagging < ActiveRecord::Base + belongs_to :player + belongs_to :ptag +end \ No newline at end of file -- 2.47.3 From f8352dcaa14e6fe0db5068539437bd888ffc210e Mon Sep 17 00:00:00 2001 From: Bilal Date: Wed, 29 Apr 2020 16:52:54 +0200 Subject: [PATCH 11/15] transform INNER JOIN to LEFT OUTER --- lib/pg_searchable_regex.rb | 6 ++++-- lib/text_to_sql_query.rb | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/pg_searchable_regex.rb b/lib/pg_searchable_regex.rb index 1fd390b..ecb8888 100644 --- a/lib/pg_searchable_regex.rb +++ b/lib/pg_searchable_regex.rb @@ -47,7 +47,7 @@ module PgSearchable def ts_add_scope class_eval do scope ts_scope_method, ->(value) do - resulting_ids = ts_search(value).map(&:id) + resulting_ids = ts_search(value).rows.map { |row| row[0] } where(id: resulting_ids) end end @@ -62,7 +62,9 @@ module PgSearchable @ts_search_fields_mappings, @ts_joins ) - distinct.joins(sql_query_object.join_clause).group(:id).having(sql_query_object.where_clause) + sql_query = select(:id).distinct.joins(sql_query_object.join_clause).group(:id).having(sql_query_object.where_clause) + modified_sql_query = sql_query.to_sql.gsub('INNER', 'LEFT OUTER') # TODO: Search terms should not be replaced! + ActiveRecord::Base.connection.exec_query(modified_sql_query) end def should_update_cache_field? diff --git a/lib/text_to_sql_query.rb b/lib/text_to_sql_query.rb index a2dc9bd..cd63baf 100644 --- a/lib/text_to_sql_query.rb +++ b/lib/text_to_sql_query.rb @@ -31,7 +31,6 @@ class TextToSqlQuery def join_clause return nil if @joins.empty? - return *@joins end @@ -109,6 +108,7 @@ private result.gsub!(/\_/, '\_') result.tr!('\\', '\\') result.gsub!(/%/, '\%') + result.downcase! result end end -- 2.47.3 From 15fc34263306c918bee685b255cd3060d847f4d4 Mon Sep 17 00:00:00 2001 From: Bilal Date: Wed, 29 Apr 2020 16:53:18 +0200 Subject: [PATCH 12/15] remove comment --- lib/pg_searchable_regex.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pg_searchable_regex.rb b/lib/pg_searchable_regex.rb index ecb8888..f6bce45 100644 --- a/lib/pg_searchable_regex.rb +++ b/lib/pg_searchable_regex.rb @@ -63,7 +63,7 @@ module PgSearchable @ts_joins ) sql_query = select(:id).distinct.joins(sql_query_object.join_clause).group(:id).having(sql_query_object.where_clause) - modified_sql_query = sql_query.to_sql.gsub('INNER', 'LEFT OUTER') # TODO: Search terms should not be replaced! + modified_sql_query = sql_query.to_sql.gsub('INNER', 'LEFT OUTER') ActiveRecord::Base.connection.exec_query(modified_sql_query) end -- 2.47.3 From 6c9878ecea1d88ccd124c0e588872becae0b760d Mon Sep 17 00:00:00 2001 From: Bilal Date: Wed, 29 Apr 2020 16:53:41 +0200 Subject: [PATCH 13/15] add spec to test searching for INNER search term --- spec/lib/pg_searchable_new_spec.rb | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/spec/lib/pg_searchable_new_spec.rb b/spec/lib/pg_searchable_new_spec.rb index 4084fc4..51824d3 100644 --- a/spec/lib/pg_searchable_new_spec.rb +++ b/spec/lib/pg_searchable_new_spec.rb @@ -158,6 +158,19 @@ describe PgSearchable do expect(DynamicModelWithTagValues.scope_search('tag:green or value:"not"')).to contain_exactly(record1, record2) end + it 'can search for search term containing INNER word' do + records = DynamicModelWithTagValues.create [{ name: 'inner', value: 'amazing' }, + { name: 'new record', value: 'INNER' }] + + Tag.create [{ taggable: records[0], value: 'red', custom_attribute: 'rose' }, + { taggable: records[1], value: 'INNER', custom_attribute: 'garden' }] + + expect(DynamicModelWithTagValues.scope_search('name:inner')).to contain_exactly(records[0]) + expect(DynamicModelWithTagValues.scope_search('value:INNER')).to contain_exactly(records[1]) + expect(DynamicModelWithTagValues.scope_search('tag:inner')).to contain_exactly(records[1]) + expect(DynamicModelWithTagValues.scope_search('tag:INNER')).to contain_exactly(records[1]) + end + describe 'searching in model with has_many association' do before do records = DynamicModelWithTagValues.create [{ name: 'something', value: 'amazing' }, @@ -325,9 +338,6 @@ describe PgSearchable do end it 'can search in referenced column associated with through relaction - simple search with NOT operator' do - puts "===========" - puts Player.scope_search('NOT tag:e-ink').inspect - puts "===============" expect(Player.scope_search('NOT tag:e-ink')).to contain_exactly(players[2], players[3]) end -- 2.47.3 From c7b173cb79bf40fa914d46c6e44b7379323e2103 Mon Sep 17 00:00:00 2001 From: Bilal Date: Wed, 29 Apr 2020 17:10:54 +0200 Subject: [PATCH 14/15] refactor text_to_sql spec to include STRING_AGG wrap --- spec/lib/text_to_sql_query_spec.rb | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/spec/lib/text_to_sql_query_spec.rb b/spec/lib/text_to_sql_query_spec.rb index 74da5ff..5a1016d 100644 --- a/spec/lib/text_to_sql_query_spec.rb +++ b/spec/lib/text_to_sql_query_spec.rb @@ -16,7 +16,7 @@ describe TextToSqlQuery do it { expect(described_class.new('search-term', [:'players.name', :'players.tags', 'players.id'], [:'players.name', :'players.tags', :'players.id']).where_clause).to eq(['(CAST(players.name AS TEXT) ILIKE ? OR CAST(players.tags AS TEXT) ILIKE ? OR CAST(players.id AS TEXT) ILIKE ?)', '%search-term%', '%search-term%', '%search-term%']) } # tests simple search term with column name and without quotes - it { expect(described_class.new('name:ab', [:"players.name"], :"players.name").where_clause).to eq(['CAST(players.name AS TEXT) ILIKE ?', '%ab%']) } + it { expect(described_class.new('name:ab', [:"players.name"], :"players.name").where_clause).to eq(["STRING_AGG(CAST(players.name AS TEXT), '') ILIKE ?", '%ab%']) } # tests simple search term with unknown column name and without quotes it { expect{described_class.new('unknown:ab', [:"players.name"], :"players.name").where_clause}.to raise_error(RuntimeError, "Unknown field 'unknown'") } @@ -25,7 +25,7 @@ describe TextToSqlQuery do it { expect(described_class.new('"ab"', [:"players.name", :"players.device_id"], :"players.device_id").where_clause).to eq(["(CAST(players.device_id AS TEXT) ILIKE ?)", "%ab%"]) } # tests simple search term with column name and with quotes - it { expect(described_class.new('tags:"ab"', [:"players.name", :"players.tags"], :"players.device_id").where_clause).to eq(["CAST(players.tags AS TEXT) ILIKE ?", "%ab%"]) } + it { expect(described_class.new('tags:"ab"', [:"players.name", :"players.tags"], :"players.device_id").where_clause).to eq(["STRING_AGG(CAST(players.tags AS TEXT), '') ILIKE ?", "%ab%"]) } # tests search without operators it { expect(described_class.new('123 456', [:"players.name", :"players.device_id"], :"players.device_id").where_clause).to eq(["((CAST(players.device_id AS TEXT) ILIKE ?) OR (CAST(players.device_id AS TEXT) ILIKE ?))", "%123%", "%456%"]) } @@ -40,34 +40,34 @@ describe TextToSqlQuery do it { expect(described_class.new('not 23', [:"players.name", :"players.device_id"], :"players.device_id").where_clause).to eq(["NOT (CAST(players.device_id AS TEXT) ILIKE ?)", "%23%"]) } # tests search with NOT operator on non-default column - it { expect(described_class.new('not value:23', [:"players.name", :"players.value"], :"players.device_id").where_clause).to eq(["NOT CAST(players.value AS TEXT) ILIKE ?", "%23%"]) } + it { expect(described_class.new('not value:23', [:"players.name", :"players.value"], :"players.device_id").where_clause).to eq(["NOT STRING_AGG(CAST(players.value AS TEXT), '') ILIKE ?", "%23%"]) } # tests search with mixed logical operators - it { expect(described_class.new('name:ab and not value:hf-1', [:"players.name", :"players.value"], :"players.device_id").where_clause).to eq(['(CAST(players.name AS TEXT) ILIKE ? AND NOT CAST(players.value AS TEXT) ILIKE ?)', '%ab%', '%hf-1%']) } + it { expect(described_class.new('name:ab and not value:hf-1', [:"players.name", :"players.value"], :"players.device_id").where_clause).to eq(["(STRING_AGG(CAST(players.name AS TEXT), '') ILIKE ? AND NOT STRING_AGG(CAST(players.value AS TEXT), '') ILIKE ?)", '%ab%', '%hf-1%']) } # tests search with mixed logical operators without NOT' - it { expect(described_class.new('name:a and name:b or name:c', [:"players.name", :"players.value"], :"players.device_id").where_clause).to eq(['((CAST(players.name AS TEXT) ILIKE ? AND CAST(players.name AS TEXT) ILIKE ?) OR CAST(players.name AS TEXT) ILIKE ?)', '%a%', '%b%', '%c%']) } + it { expect(described_class.new('name:a and name:b or name:c', [:"players.name", :"players.value"], :"players.device_id").where_clause).to eq(["((STRING_AGG(CAST(players.name AS TEXT), '') ILIKE ? AND STRING_AGG(CAST(players.name AS TEXT), '') ILIKE ?) OR STRING_AGG(CAST(players.name AS TEXT), '') ILIKE ?)", '%a%', '%b%', '%c%']) } # tests search with brackets in expression - it { expect(described_class.new('name:a and (name:b or name:c)', [:"players.name", :"players.value"], :"players.device_id").where_clause).to eq(['(CAST(players.name AS TEXT) ILIKE ? AND (CAST(players.name AS TEXT) ILIKE ? OR CAST(players.name AS TEXT) ILIKE ?))', '%a%', '%b%', '%c%']) } + it { expect(described_class.new('name:a and (name:b or name:c)', [:"players.name", :"players.value"], :"players.device_id").where_clause).to eq(["(STRING_AGG(CAST(players.name AS TEXT), '') ILIKE ? AND (STRING_AGG(CAST(players.name AS TEXT), '') ILIKE ? OR STRING_AGG(CAST(players.name AS TEXT), '') ILIKE ?))", '%a%', '%b%', '%c%']) } # tests search with brackets in expression and with NOT operator - it { expect(described_class.new('name:a and not (name:b or name:c)', [:"players.name", :"players.value"], :"players.device_id").where_clause).to eq(['(CAST(players.name AS TEXT) ILIKE ? AND NOT (CAST(players.name AS TEXT) ILIKE ? OR CAST(players.name AS TEXT) ILIKE ?))', '%a%', '%b%', '%c%']) } + it { expect(described_class.new('name:a and not (name:b or name:c)', [:"players.name", :"players.value"], :"players.device_id").where_clause).to eq(["(STRING_AGG(CAST(players.name AS TEXT), '') ILIKE ? AND NOT (STRING_AGG(CAST(players.name AS TEXT), '') ILIKE ? OR STRING_AGG(CAST(players.name AS TEXT), '') ILIKE ?))", '%a%', '%b%', '%c%']) } # tests search with special characters in search term - it { expect(described_class.new('name:"%a_\"', [:"players.name", :"players.value"], :"players.device_id").where_clause).to eq(['CAST(players.name AS TEXT) ILIKE ?', '%\%a\_\\%']) } + it { expect(described_class.new('name:"%a_\"', [:"players.name", :"players.value"], :"players.device_id").where_clause).to eq(["STRING_AGG(CAST(players.name AS TEXT), '') ILIKE ?", '%\%a\_\\%']) } # tests search with field mappings - it { expect(described_class.new('tags:h1-r', [:'players.name', :'players.value', :'players.device_id'], :"players.device_id", { tags: "tags.value" }).where_clause).to eq(['CAST(tags.value AS TEXT) ILIKE ?', '%h1-r%']) } + it { expect(described_class.new('tags:h1-r', [:'players.name', :'players.value', :'players.device_id'], :"players.device_id", { tags: "tags.value" }).where_clause).to eq(["STRING_AGG(CAST(tags.value AS TEXT), '') ILIKE ?", '%h1-r%']) } # tests search with field mappings when fields array has same mapping - it { expect(described_class.new('tags:h1-r', [:'players.name', :'players.tags', :'players.device_id'], :"players.device_id", { tags: "tags.value" }).where_clause).to eq(["CAST(tags.value AS TEXT) ILIKE ?", "%h1-r%"]) } + it { expect(described_class.new('tags:h1-r', [:'players.name', :'players.tags', :'players.device_id'], :"players.device_id", { tags: "tags.value" }).where_clause).to eq(["STRING_AGG(CAST(tags.value AS TEXT), '') ILIKE ?", "%h1-r%"]) } # tests complex query - it { expect(described_class.new('(device_id:"with space" tags:mta no-quotes-id-123) or "id with quotes-5" and ( ("id with q 10" or "id with q 20") and ("id with Q 30" "id with Q 40") and not id-without-Q-50)', [:'players.name', :'players.value', :'players.device_id'], :"players.device_id", { tags: 'tags.name' }).where_clause).to eq(['((CAST(players.device_id AS TEXT) ILIKE ? OR (CAST(tags.name AS TEXT) ILIKE ? OR (CAST(players.device_id AS TEXT) ILIKE ?))) OR ((CAST(players.device_id AS TEXT) ILIKE ?) AND ((((CAST(players.device_id AS TEXT) ILIKE ?) OR (CAST(players.device_id AS TEXT) ILIKE ?)) AND ((CAST(players.device_id AS TEXT) ILIKE ?) OR (CAST(players.device_id AS TEXT) ILIKE ?))) AND NOT (CAST(players.device_id AS TEXT) ILIKE ?))))', '%with space%', '%mta%', '%no-quotes-id-123%', '%id with quotes-5%', '%id with q 10%', '%id with q 20%', '%id with Q 30%', '%id with Q 40%', '%id-without-Q-50%']) } + it { expect(described_class.new('(device_id:"with space" tags:mta no-quotes-id-123) or "id with quotes-5" and ( ("id with q 10" or "id with q 20") and ("id with Q 30" "id with Q 40") and not id-without-Q-50)', [:'players.name', :'players.value', :'players.device_id'], :"players.device_id", { tags: 'tags.name' }).where_clause).to eq(["((STRING_AGG(CAST(players.device_id AS TEXT), '') ILIKE ? OR (STRING_AGG(CAST(tags.name AS TEXT), '') ILIKE ? OR (CAST(players.device_id AS TEXT) ILIKE ?))) OR ((CAST(players.device_id AS TEXT) ILIKE ?) AND ((((CAST(players.device_id AS TEXT) ILIKE ?) OR (CAST(players.device_id AS TEXT) ILIKE ?)) AND ((CAST(players.device_id AS TEXT) ILIKE ?) OR (CAST(players.device_id AS TEXT) ILIKE ?))) AND NOT (CAST(players.device_id AS TEXT) ILIKE ?))))", '%with space%', '%mta%', '%no-quotes-id-123%', '%id with quotes-5%', '%id with q 10%', '%id with q 20%', '%id with q 30%', '%id with q 40%', '%id-without-q-50%']) } # tests mixed query with and without column names and with multiple default columns - it { expect(described_class.new('("green hunter" and device_id:100) or ("blue bird" and not device_id:200)', [:'players.name', :'players.value', :'players.device_id'], [:"players.name", :"players.tags"]).where_clause).to eq(['(((CAST(players.name AS TEXT) ILIKE ? OR CAST(players.tags AS TEXT) ILIKE ?) AND CAST(players.device_id AS TEXT) ILIKE ?) OR ((CAST(players.name AS TEXT) ILIKE ? OR CAST(players.tags AS TEXT) ILIKE ?) AND NOT CAST(players.device_id AS TEXT) ILIKE ?))', '%green hunter%', '%green hunter%', '%100%', '%blue bird%', '%blue bird%', '%200%']) } + it { expect(described_class.new('("green hunter" and device_id:100) or ("blue bird" and not device_id:200)', [:'players.name', :'players.value', :'players.device_id'], [:"players.name", :"players.tags"]).where_clause).to eq(["(((CAST(players.name AS TEXT) ILIKE ? OR CAST(players.tags AS TEXT) ILIKE ?) AND STRING_AGG(CAST(players.device_id AS TEXT), '') ILIKE ?) OR ((CAST(players.name AS TEXT) ILIKE ? OR CAST(players.tags AS TEXT) ILIKE ?) AND NOT STRING_AGG(CAST(players.device_id AS TEXT), '') ILIKE ?))", '%green hunter%', '%green hunter%', '%100%', '%blue bird%', '%blue bird%', '%200%']) } # tests query with multiple search terms with mixed and-or-not after dash and underscore it { expect(described_class.new('123-and-456 -or-2 -not_not_1', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id').where_clause).to eq(['((CAST(players.device_id AS TEXT) ILIKE ?) OR ((CAST(players.device_id AS TEXT) ILIKE ?) OR (CAST(players.device_id AS TEXT) ILIKE ?)))', '%123-and-456%', '%-or-2%', '%-not\_not\_1%'])} -- 2.47.3 From 3c4f3f3b584916ef4843fed6c2bddfa1f20e2ad4 Mon Sep 17 00:00:00 2001 From: Bilal Date: Wed, 29 Apr 2020 18:00:58 +0200 Subject: [PATCH 15/15] add more search tests --- spec/lib/pg_searchable_new_spec.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/spec/lib/pg_searchable_new_spec.rb b/spec/lib/pg_searchable_new_spec.rb index 51824d3..01f9204 100644 --- a/spec/lib/pg_searchable_new_spec.rb +++ b/spec/lib/pg_searchable_new_spec.rb @@ -356,6 +356,13 @@ describe PgSearchable do it 'can search in referenced column associated with through relaction - mixed search terms with AND NOT' do expect(Player.scope_search('tag:e-ink AND NOT first')).to contain_exactly(players[1]) end + + it 'can search in referenced column and in model columns - complex search' do + expect(Player.scope_search('(tag:e-ink AND tag:rich) OR (blue or value:eagle)')).to contain_exactly(players[1], players[2], players[3]) + expect(Player.scope_search('tag:e-ink AND tag:rich AND NOT tag:black')).to be_empty + expect(Player.scope_search('name:l AND (value:uptown or value:marine) AND (tag:e-ink and tag:rich)')).to contain_exactly(players[1]) + expect(Player.scope_search('name:l AND (value:uptown or value:marine) AND (tag:e-ink and tag:led)')).to be_empty + end end end end -- 2.47.3