From 84a48b4acfde0a7dd3d33c46d9c0c4570a316ae8 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Fri, 31 Jan 2020 15:10:48 +0100 Subject: [PATCH 1/5] change LIKE to ILIKE --- spec/query_generator_spec.rb | 30 +++++++++++++++--------------- text_to_sql_query.rb | 6 +++--- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/spec/query_generator_spec.rb b/spec/query_generator_spec.rb index 4226f20..1896e50 100644 --- a/spec/query_generator_spec.rb +++ b/spec/query_generator_spec.rb @@ -5,91 +5,91 @@ class SqlGeneratorTester it 'tests simple search term without column name and without quotes' do @query = TextToSqlQuery.new('-123', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['players.device_id LIKE ?', '%-123%'] + expect(@query.where_clause).to eq ['players.device_id ILIKE ?', '%-123%'] end it 'tests simple search term with column name and without quotes' do @query = TextToSqlQuery.new('title:ab', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['players.title LIKE ?', '%ab%'] + expect(@query.where_clause).to eq ['players.title ILIKE ?', '%ab%'] end it 'tests simple search term with unknown column name and without quotes' do @query = TextToSqlQuery.new('unknown:ab', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['players.device_id LIKE ?', '%ab%'] + expect(@query.where_clause).to eq ['players.device_id ILIKE ?', '%ab%'] end it 'tests simple search term without column name and with quotes' do @query = TextToSqlQuery.new('"ab"', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['players.device_id LIKE ?', '%"ab"%'] + expect(@query.where_clause).to eq ['players.device_id ILIKE ?', '%"ab"%'] end it 'tests simple search term with column name and with quotes' do @query = TextToSqlQuery.new('tag:"ab"', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['players.tag LIKE ?', '%"ab"%'] + expect(@query.where_clause).to eq ['players.tag ILIKE ?', '%"ab"%'] end it 'tests search without operators' do @query = TextToSqlQuery.new('123 456', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['(players.device_id LIKE ? OR players.device_id LIKE ?)', '%123%', '%456%'] + expect(@query.where_clause).to eq ['(players.device_id ILIKE ? OR players.device_id ILIKE ?)', '%123%', '%456%'] end it 'tests search with OR operator' do @query = TextToSqlQuery.new('123 or 456', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['(players.device_id LIKE ? OR players.device_id LIKE ?)', '%123%', '%456%'] + expect(@query.where_clause).to eq ['(players.device_id ILIKE ? OR players.device_id ILIKE ?)', '%123%', '%456%'] end it 'tests search with AND operator' do @query = TextToSqlQuery.new('123 and 456', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['(players.device_id LIKE ? AND players.device_id LIKE ?)', '%123%', '%456%'] + expect(@query.where_clause).to eq ['(players.device_id ILIKE ? AND players.device_id ILIKE ?)', '%123%', '%456%'] end it 'tests search with NOT operator on default column' do @query = TextToSqlQuery.new('not 23', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['NOT players.device_id LIKE ?', '%23%'] + expect(@query.where_clause).to eq ['NOT players.device_id ILIKE ?', '%23%'] end it 'tests search with NOT operator on non-default column' do @query = TextToSqlQuery.new('not tag:23', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['NOT players.tag LIKE ?', '%23%'] + expect(@query.where_clause).to eq ['NOT players.tag ILIKE ?', '%23%'] end it 'tests search with mixed logical operators' do @query = TextToSqlQuery.new('title:ab and not tag:hf-1', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['(players.title LIKE ? AND NOT players.tag LIKE ?)', '%ab%', '%hf-1%'] + expect(@query.where_clause).to eq ['(players.title ILIKE ? AND NOT players.tag ILIKE ?)', '%ab%', '%hf-1%'] end it 'tests search with mixed logical operators without NOT' do @query = TextToSqlQuery.new('title:a and title:b or title:c', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['((players.title LIKE ? AND players.title LIKE ?) OR players.title LIKE ?)', '%a%', '%b%', '%c%'] + expect(@query.where_clause).to eq ['((players.title ILIKE ? AND players.title ILIKE ?) OR players.title ILIKE ?)', '%a%', '%b%', '%c%'] end it 'tests search with brackets in expression' do @query = TextToSqlQuery.new('title:a and (title:b or title:c)', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['(players.title LIKE ? AND (players.title LIKE ? OR players.title LIKE ?))', '%a%', '%b%', '%c%'] + expect(@query.where_clause).to eq ['(players.title ILIKE ? AND (players.title ILIKE ? OR players.title ILIKE ?))', '%a%', '%b%', '%c%'] end it 'tests search with brackets in expression and with NOT operator' do @query = TextToSqlQuery.new('title:a and not (title:b or title:c)', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['(players.title LIKE ? AND NOT (players.title LIKE ? OR players.title LIKE ?))', '%a%', '%b%', '%c%'] + expect(@query.where_clause).to eq ['(players.title ILIKE ? AND NOT (players.title ILIKE ? OR players.title ILIKE ?))', '%a%', '%b%', '%c%'] end it 'tests search with special characters in search term' do @query = TextToSqlQuery.new('title:"%a_\"', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['players.title LIKE ?', '%"\%a\_\\"%'] + expect(@query.where_clause).to eq ['players.title ILIKE ?', '%"\%a\_\\"%'] end end end \ No newline at end of file diff --git a/text_to_sql_query.rb b/text_to_sql_query.rb index 241149a..12f5412 100644 --- a/text_to_sql_query.rb +++ b/text_to_sql_query.rb @@ -26,7 +26,7 @@ class TextToSqlQuery case first_key when :DEFAULT_COLUMN escaped_node_value = escape_special_chars node_value - ["#{@default_field.to_s} LIKE ?", "%#{escaped_node_value}%"] + ["#{@default_field.to_s} ILIKE ?", "%#{escaped_node_value}%"] when :OPERATOR_OR generate_expression_for_logical_operator(:OR, node_value) when :OPERATOR_AND @@ -48,9 +48,9 @@ class TextToSqlQuery escaped_node_value = escape_special_chars node_value mapping = @fields_mappings[first_key.to_sym] if mapping.nil? - ["#{@default_field.to_s} LIKE ?", "%#{escaped_node_value}%"] + ["#{@default_field.to_s} ILIKE ?", "%#{escaped_node_value}%"] else - ["#{mapping.to_s} LIKE ?", "%#{escaped_node_value}%"] + ["#{mapping.to_s} ILIKE ?", "%#{escaped_node_value}%"] end end end From 87712860ddf4954f66d7f6a5445e51b25446e42e Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Fri, 31 Jan 2020 16:11:12 +0100 Subject: [PATCH 2/5] remove double quotes from search term --- spec/query_generator_spec.rb | 14 ++++++++++---- text_to_sql_query.rb | 8 ++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/spec/query_generator_spec.rb b/spec/query_generator_spec.rb index 1896e50..0ee9ba4 100644 --- a/spec/query_generator_spec.rb +++ b/spec/query_generator_spec.rb @@ -21,15 +21,15 @@ class SqlGeneratorTester end it 'tests simple search term without column name and with quotes' do - @query = TextToSqlQuery.new('"ab"', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') + @query = TextToSqlQuery.new('ab', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['players.device_id ILIKE ?', '%"ab"%'] + expect(@query.where_clause).to eq ['players.device_id ILIKE ?', '%ab%'] end it 'tests simple search term with column name and with quotes' do @query = TextToSqlQuery.new('tag:"ab"', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['players.tag ILIKE ?', '%"ab"%'] + expect(@query.where_clause).to eq ['players.tag ILIKE ?', '%ab%'] end it 'tests search without operators' do @@ -89,7 +89,13 @@ class SqlGeneratorTester it 'tests search with special characters in search term' do @query = TextToSqlQuery.new('title:"%a_\"', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['players.title ILIKE ?', '%"\%a\_\\"%'] + expect(@query.where_clause).to eq ['players.title ILIKE ?', '%\%a\_\\%'] + end + + it 'tests search with field mappings' do + @query = TextToSqlQuery.new('tags:h1-r', [:'players.title', :'players.name', :'players.device_id'], :'players.device_id', { tags: "tags.name" }) + + expect(@query.where_clause).to eq ['tags.name ILIKE ?', '%h1-r%'] end end end \ No newline at end of file diff --git a/text_to_sql_query.rb b/text_to_sql_query.rb index 12f5412..fb4bd6b 100644 --- a/text_to_sql_query.rb +++ b/text_to_sql_query.rb @@ -25,7 +25,7 @@ class TextToSqlQuery node_value = tree[first_key] case first_key when :DEFAULT_COLUMN - escaped_node_value = escape_special_chars node_value + escaped_node_value = handle_special_chars node_value ["#{@default_field.to_s} ILIKE ?", "%#{escaped_node_value}%"] when :OPERATOR_OR generate_expression_for_logical_operator(:OR, node_value) @@ -45,7 +45,7 @@ class TextToSqlQuery else # key is column name - escaped_node_value = escape_special_chars node_value + escaped_node_value = handle_special_chars node_value mapping = @fields_mappings[first_key.to_sym] if mapping.nil? ["#{@default_field.to_s} ILIKE ?", "%#{escaped_node_value}%"] @@ -80,8 +80,8 @@ class TextToSqlQuery ["(#{first_operand_expression} #{operator.to_s} #{second_operand_expression})"] + first_operand_params + second_operand_params end - def escape_special_chars(text) - result = text + def handle_special_chars(text) + result = text.gsub(/\"/, '') result.gsub!(/\_/, '\_') result.tr!('\\', '\\') result.gsub!(/%/, '\%') From e521a4de095a1c63d2f0336bab91ec15b493564e Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Fri, 31 Jan 2020 17:44:12 +0100 Subject: [PATCH 3/5] raise error on unknown column; add complex query test --- spec/query_generator_spec.rb | 27 ++++++++++++++++++++++++++- text_to_sql_query.rb | 4 +++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/spec/query_generator_spec.rb b/spec/query_generator_spec.rb index 0ee9ba4..bd617af 100644 --- a/spec/query_generator_spec.rb +++ b/spec/query_generator_spec.rb @@ -17,7 +17,7 @@ class SqlGeneratorTester it 'tests simple search term with unknown column name and without quotes' do @query = TextToSqlQuery.new('unknown:ab', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id') - expect(@query.where_clause).to eq ['players.device_id ILIKE ?', '%ab%'] + expect { @query.where_clause }.to raise_error(RuntimeError, "Unknown field 'unknown'") end it 'tests simple search term without column name and with quotes' do @@ -97,5 +97,30 @@ class SqlGeneratorTester expect(@query.where_clause).to eq ['tags.name ILIKE ?', '%h1-r%'] end + + it 'tests complex query' do + text = '(device_id:"with space" tags:mta no-quotes-id-123)'\ + 'or "id with quotes-5" and ( ("id with q 10" or "id with q 20")'\ + 'and ("id with Q 30" "id with Q 40") and not id-without-Q-50)' + + fields = [:'players.title', :'players.name', :'players.device_id'] + default_field = :'players.device_id' + fields_mappings = { tags: 'tags.name' } + + @query = TextToSqlQuery.new(text, fields, default_field, fields_mappings) + + result_expression = '((players.device_id ILIKE ? OR (tags.name ILIKE ? OR players.device_id ILIKE ?)) OR (players.device_id ILIKE ? AND (((players.device_id ILIKE ? OR players.device_id ILIKE ?) AND (players.device_id ILIKE ? OR players.device_id ILIKE ?)) AND NOT players.device_id ILIKE ?)))' + result_params = ['%with space%', + '%mta%', + '%no-quotes-id-123%', + '%id with quotes-5%', + '%id with q 10%', + '%id with q 20%', + '%id with Q 30%', + '%id with Q 40%', + '%id-without-Q-50%'] + + expect(@query.where_clause).to eq [result_expression, result_params].flatten + end end end \ No newline at end of file diff --git a/text_to_sql_query.rb b/text_to_sql_query.rb index fb4bd6b..4fbefa8 100644 --- a/text_to_sql_query.rb +++ b/text_to_sql_query.rb @@ -48,7 +48,9 @@ class TextToSqlQuery escaped_node_value = handle_special_chars node_value mapping = @fields_mappings[first_key.to_sym] if mapping.nil? - ["#{@default_field.to_s} ILIKE ?", "%#{escaped_node_value}%"] + # ["#{@default_field.to_s} ILIKE ?", "%#{escaped_node_value}%"] + #['', ''] + raise "Unknown field '#{first_key.to_s}'" # "Unknown field '#{first_key.to_s}' in query " else ["#{mapping.to_s} ILIKE ?", "%#{escaped_node_value}%"] end From 3a828c6fe937c63e6df13d8cc3974599e8f6cfb4 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Fri, 31 Jan 2020 17:51:30 +0100 Subject: [PATCH 4/5] remove obsolete comments --- text_to_sql_query.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/text_to_sql_query.rb b/text_to_sql_query.rb index 4fbefa8..a4d1a13 100644 --- a/text_to_sql_query.rb +++ b/text_to_sql_query.rb @@ -48,9 +48,7 @@ class TextToSqlQuery escaped_node_value = handle_special_chars node_value mapping = @fields_mappings[first_key.to_sym] if mapping.nil? - # ["#{@default_field.to_s} ILIKE ?", "%#{escaped_node_value}%"] - #['', ''] - raise "Unknown field '#{first_key.to_s}'" # "Unknown field '#{first_key.to_s}' in query " + raise "Unknown field '#{first_key.to_s}'" else ["#{mapping.to_s} ILIKE ?", "%#{escaped_node_value}%"] end From 91f699906a9ea2356daa86974958c5a2c75be361 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Fri, 31 Jan 2020 19:23:10 +0100 Subject: [PATCH 5/5] keep mappings over field array values --- spec/query_generator_spec.rb | 6 ++++++ text_to_sql_query.rb | 3 +++ 2 files changed, 9 insertions(+) diff --git a/spec/query_generator_spec.rb b/spec/query_generator_spec.rb index bd617af..af5338c 100644 --- a/spec/query_generator_spec.rb +++ b/spec/query_generator_spec.rb @@ -98,6 +98,12 @@ class SqlGeneratorTester expect(@query.where_clause).to eq ['tags.name ILIKE ?', '%h1-r%'] end + it 'tests search with field mappings when fields array has same mapping' do + @query = TextToSqlQuery.new('tags:hs1-r', [:'players.title', :'players.tags', :'players.device_id'], :'players.device_id', { tags: "tags.name" }) + + expect(@query.where_clause).to eq ['tags.name ILIKE ?', '%hs1-r%'] + end + it 'tests complex query' do text = '(device_id:"with space" tags:mta no-quotes-id-123)'\ 'or "id with quotes-5" and ( ("id with q 10" or "id with q 20")'\ diff --git a/text_to_sql_query.rb b/text_to_sql_query.rb index a4d1a13..f0bbce2 100644 --- a/text_to_sql_query.rb +++ b/text_to_sql_query.rb @@ -10,6 +10,9 @@ class TextToSqlQuery mappings[field_name.to_sym] = field mappings end) + fields_mappings.each do |field, value| + @fields_mappings[field] = value if @fields_mappings[field] + end end def where_clause