+
@@ -588,7 +588,7 @@
- # frozen_string_literal: true
+ require './parser'
@@ -600,79 +600,79 @@
- # transforms "english like" text queries into a where clause with regex
+ class TextToSqlQuery
- # https://www.postgresql.org/docs/9.5/textsearch-controls.html#TEXTSEARCH-PARSING-QUERIES
+ def initialize(text, fields, default_field, fields_mappings = {})
- class TextToRegexQuery
-
-
-
- 1
-
- def initialize(text, fields, default_field, fields_mappings = {})
-
-
-
- 35
-
@text = text.to_s.strip
-
- 35
+
+
@fields = fields.map(&:to_sym)
-
- 35
+
+
@default_field = default_field.to_sym
-
- 35
+
+
@fields_mappings = fields_mappings.merge(@fields.reduce({}) do |mappings, field|
-
- 37
+
- table_name, field_name = field.to_s.split(".")
+
+ _table_name, field_name = field.to_s.split('.')
-
- 37
+
+
mappings[field_name.to_sym] = field
-
- 7
+
- mappings
+
+ mappings
+
+
+
+
+
+ end)
+
+
+
+
+
+ fields_mappings.each do |field, value|
+
+
+
+
+
+ @fields_mappings[field] = value if @fields_mappings[field]
- end)
+ end
@@ -687,297 +687,459 @@
def where_clause(query)
+
+ def where_clause
-
- 5
+
- @cleared_text = @text.dup
+
+ @parser = Query.new
-
- 5
+
- @column_chunks = []
+
+ @parsed_tree = @parser.parse(@text)
-
- 5
+
- remove_duplicated_spaces
+
+ generate_sql @parsed_tree
-
- 5
+
- extract_columns
+
+ end
-
- 5
+
- escape_special_characters
+
+ generate_where_clause(query)
+
+ private
- end
+ def generate_sql(tree)
-
- 1
+
- private
+
+ first_key = tree.keys.first
- node_value = tree[first_key]
-
- 1
+
- def remove_duplicated_spaces
+
+ case first_key
-
- 5
+
- @cleared_text.gsub!(/\s+/, ' ')
+
+ when :DEFAULT_COLUMN
- end
+ escaped_node_value = handle_special_chars node_value
- ["#{@default_field.to_s} ILIKE ?", "%#{escaped_node_value}%"]
-
- 1
+
- def escape_special_characters
+
+ when :OPERATOR_OR
-
- 5
+
- @cleared_text.gsub!(/\_/, '\_')
+
+ generate_expression_for_logical_operator(:OR, node_value)
-
- 5
+
- @cleared_text.tr!('\\', '\\')
+
+ when :OPERATOR_AND
-
- 5
+
- @cleared_text.gsub!(/%/, '\%')
+
+ generate_expression_for_logical_operator(:AND, node_value)
- end
+ when :OPERATOR_NOT
+ not_array = generate_sql node_value
+
+
+
+
+
def extract_columns
-
-
-
- 5
- column_search_term_pairs = @cleared_text.scan(/([A-Za-z0-9_]+:[\w\_-]+)/)
+ if not_array.length < 2
+ raise "There should be more than 1 element for expression following NOT operator"
+
+
+
+
+
+ end
+
+
+
+
+
@column_chunks = (column_search_term_pairs.flatten.map do |pair|
+
+ not_expression = not_array.first
-
- 7
+
- column, term = pair.split(':')
+
+ not_params = not_array[1..]
-
- 7
+
- next unless @fields_mappings.include?(column.to_sym)
-
-
-
- 5
- @cleared_text.gsub!(pair, '')
-
-
-
- 5
-
- { @fields_mappings[column.to_sym] => term }
+ end).compact
+ ["NOT #{not_expression}"] + not_params
-
- 5
+
- unless @cleared_text.strip.empty?
+
+ @column_chunks = [{ @default_field.to_s => @cleared_text.strip }] + @column_chunks
+
+ else
- end
+ # key is column name
-
- 5
+
- @column_chunks
+
+ escaped_node_value = handle_special_chars node_value
- end
+ mapping = @fields_mappings[first_key.to_sym]
- if mapping.nil?
-
- 1
+
- def generate_where_clause(query)
+
+ raise "Unknown field '#{first_key.to_s}'"
-
- 5
+
- where_clause = ''
+
+ else
-
- 13
+
- columns = @column_chunks.map { |c| c.keys.first }
+
+ ["#{mapping.to_s} ILIKE ?", "%#{escaped_node_value}%"]
-
- 13
+
- values = @column_chunks.map { |c| c.values.first }
+
+ end
- columns.each do |column|
-
-
-
- 8
-
- where_clause += "#{column} LIKE ? OR "
-
-
-
-
-
end
-
- 5
-
- where_clause += " 1<>1 "
-
-
-
-
-
- query.where([where_clause] + values)
-
-
-
+
end
+
+
+
+ def generate_expression_for_logical_operator(operator, operator_array)
+
+
+
+
+
+ if operator_array.length != 2
+
+
+
+
+
+ raise "There should be two array elements for #{operator.to_s} operator"
+
+
+
+
+
+ end
+
+
+
+
+
+ first_operand = generate_sql operator_array.first
+
+
+
+
+
+ second_operand = generate_sql operator_array.last
+
+
+
+
+
+ if first_operand.length < 2
+
+
+
+
+
+ raise 'There should be more than 1 element in first operand array'
+
+
+
+
+
+ end
+
+
+
+
+
+ if second_operand.length < 2
+
+
+
+
+
+ raise 'There should be more than 1 element in second operand array'
+
+
+
+
+
+ end
+
+
+
+
+
+ first_operand_expression = first_operand.first
+
+
+
+
+
+ first_operand_params = first_operand[1..]
+
+
+
+
+
+ second_operand_expression = second_operand.first
+
+
+
+
+
+ second_operand_params = second_operand[1..]
+
+
+
+
+
+ ["(#{first_operand_expression} #{operator.to_s} #{second_operand_expression})"] + first_operand_params + second_operand_params
+
+
+
+
+
+ end
+
+
+
+
+
+ def handle_special_chars(text)
+
+
+
+
+
+ result = text.gsub(/\"/, '')
+
+
+
+
+
+ result.gsub!(/\_/, '\_')
+
+
+
+
+
+ result.tr!('\\', '\\')
+
+
+
+
+
+ result.gsub!(/%/, '\%')
+
+
+
+
+
+ result
+
+
+
+
+
+ end
+
+
+
+
+
end
diff --git a/lib/lexer.rb b/lib/lexer.rb
new file mode 100644
index 0000000..80cb51f
--- /dev/null
+++ b/lib/lexer.rb
@@ -0,0 +1,107 @@
+#--
+# DO NOT MODIFY!!!!
+# This file is automatically generated by rex 1.0.7
+# from lexical definition file "./lib/specification.rex".
+#++
+
+require 'racc/parser'
+class Query < Racc::Parser
+ require 'strscan'
+
+ class ScanError < StandardError ; end
+
+ attr_reader :lineno
+ attr_reader :filename
+ attr_accessor :state
+
+ def scan_setup(str)
+ @ss = StringScanner.new(str)
+ @lineno = 1
+ @state = nil
+ end
+
+ def action
+ yield
+ end
+
+ def scan_str(str)
+ scan_setup(str)
+ do_parse
+ end
+ alias :scan :scan_str
+
+ def load_file( filename )
+ @filename = filename
+ File.open(filename, "r") do |f|
+ scan_setup(f.read)
+ end
+ end
+
+ def scan_file( filename )
+ load_file(filename)
+ do_parse
+ end
+
+
+ def next_token
+ return if @ss.eos?
+
+ # skips empty actions
+ until token = _next_token or @ss.eos?; end
+ token
+ end
+
+ def _next_token
+ text = @ss.peek(1)
+ @lineno += 1 if text == "\n"
+ token = case @state
+ when nil
+ case
+ when (text = @ss.scan(/ +/))
+ ;
+
+ when (text = @ss.scan(/\(/))
+ action { return [:L_BRACKET, text] }
+
+ when (text = @ss.scan(/\)/))
+ action { return [:R_BRACKET, text] }
+
+ when (text = @ss.scan(/(?i)or/))
+ action { return [:OPERATOR_OR, text] }
+
+ when (text = @ss.scan(/(?i)and/))
+ action { return [:OPERATOR_AND, text] }
+
+ when (text = @ss.scan(/(?i)not/))
+ action { return [:OPERATOR_NOT, text] }
+
+ when (text = @ss.scan(/"([^"]*)"/))
+ action { return [:TERM_WITH_QUOTES, text] }
+
+ when (text = @ss.scan(/[a-zA-Z0-9\-_]+/))
+ action { return [:TERM_WITHOUT_QUOTES, text] }
+
+ when (text = @ss.scan(/\:/))
+ action { return [:COLON, text] }
+
+
+ else
+ text = @ss.string[@ss.pos .. -1]
+ raise ScanError, "can not match: '" + text + "'"
+ end # if
+
+ else
+ raise ScanError, "undefined state: '" + state.to_s + "'"
+ end # case state
+ token
+ end # def _next_token
+
+ def tokenize(code)
+ scan_setup(code)
+ tokens = []
+ while token = next_token
+ tokens << token
+ end
+ tokens
+ end
+end # class
diff --git a/lib/parser-parser-part/.gitignore b/lib/parser-parser-part/.gitignore
deleted file mode 100644
index c8d090a..0000000
--- a/lib/parser-parser-part/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-.idea
-
-lexer.rb
-parser.rb
\ No newline at end of file
diff --git a/lib/parser-parser-part/README.md b/lib/parser-parser-part/README.md
deleted file mode 100644
index 2eae07a..0000000
--- a/lib/parser-parser-part/README.md
+++ /dev/null
@@ -1,19 +0,0 @@
-# parser
-
-### Prerequisites
-
-* Rexical (rex)
-* Racc
-
-### Available commands
-
-* `rake lexer` - generates `lexer.rb` file based on `specification.rex` file
-* `rake parser` - generates `parser.rb` file based on `grammar.y` file
-* `rake generate` - generates `lexer.rb` and `parser.rb` files
-
-### Testing
-
-To run only `lexer` tests, execute : `rspec spec/query_lexer_spec.rb`
-To run only `parser` tests, execute : `rspec spec/query_parser_spec.rb`
-
-To run all tests, execute : `rake spec`
\ No newline at end of file
diff --git a/lib/parser-parser-part/Rakefile b/lib/parser-parser-part/Rakefile
deleted file mode 100644
index 92064f7..0000000
--- a/lib/parser-parser-part/Rakefile
+++ /dev/null
@@ -1,20 +0,0 @@
-require 'rspec/core/rake_task'
-
-RSpec::Core::RakeTask.new do |c|
- options = ['--color']
- options += %w[--format documentation]
- c.rspec_opts = options
-end
-
-desc 'Generate Lexer'
-task :lexer do
- `rex specification.rex -o lexer.rb`
-end
-
-desc 'Generate Parser'
-task :parser do
- `racc grammar.y -o parser.rb`
-end
-
-desc 'Generate Lexer and Parser'
-task generate: %i[lexer parser]
\ No newline at end of file
diff --git a/lib/parser-parser-part/grammar.y b/lib/parser-parser-part/grammar.y
deleted file mode 100644
index 5484994..0000000
--- a/lib/parser-parser-part/grammar.y
+++ /dev/null
@@ -1,26 +0,0 @@
-class Query
- prechigh
- left OPERATOR_NOT
- left OPERATOR_AND
- left OPERATOR_OR
- preclow
- rule
- target: expression
- | /* none */ { result = 0 }
-
- expression: TERM_WITHOUT_QUOTES { result = {:DEFAULT_COLUMN => val[0]} }
- | TERM_WITH_QUOTES { result = {:DEFAULT_COLUMN => val[0]} }
- | TERM_WITHOUT_QUOTES COLON TERM_WITHOUT_QUOTES { result = {val[0] => val[2]} }
- | TERM_WITHOUT_QUOTES COLON TERM_WITH_QUOTES { result = {val[0] => val[2]} }
- | expression OPERATOR_OR expression { result = {:OPERATOR_OR => [val[0], val[2]]} }
- | expression OPERATOR_AND expression { result = {:OPERATOR_AND => [val[0], val[2]]} }
- | L_BRACKET expression R_BRACKET { result = val[1] }
-end
-
----- header
- require_relative 'lexer'
-
----- inner
- def parse(input)
- scan_str(input)
- end
\ No newline at end of file
diff --git a/lib/parser-parser-part/spec/query_lexer_spec.rb b/lib/parser-parser-part/spec/query_lexer_spec.rb
deleted file mode 100644
index b97a65a..0000000
--- a/lib/parser-parser-part/spec/query_lexer_spec.rb
+++ /dev/null
@@ -1,259 +0,0 @@
-require './lexer'
-
-class QueryLexerTester
- describe 'Testing the Lexer' do
- before do
- @evaluator = Query.new
- end
-
- it 'tests bracket expression' do
- @result = @evaluator.tokenize('()')
- expect(@result.length).to eq 2
-
- expect(@result[0][0]).to eq :L_BRACKET
- expect(@result[1][0]).to eq :R_BRACKET
- end
-
- it 'tests bracket expression with spaces' do
- @result = @evaluator.tokenize(' ( ) ')
- expect(@result.length).to eq 2
-
- expect(@result[0][0]).to eq :L_BRACKET
- expect(@result[1][0]).to eq :R_BRACKET
- end
-
- it 'tests expression with OR operator' do
- @result = @evaluator.tokenize('() or () OR ()')
- expect(@result.length).to eq 8
-
- expect(@result[0][0]).to eq :L_BRACKET
- expect(@result[1][0]).to eq :R_BRACKET
- expect(@result[2][0]).to eq :OPERATOR_OR
- expect(@result[3][0]).to eq :L_BRACKET
- expect(@result[4][0]).to eq :R_BRACKET
- expect(@result[5][0]).to eq :OPERATOR_OR
- expect(@result[6][0]).to eq :L_BRACKET
- expect(@result[7][0]).to eq :R_BRACKET
- end
-
- it 'tests expression with AND operator' do
- @result = @evaluator.tokenize('() AND () and ()')
- expect(@result.length).to eq 8
-
- expect(@result[0][0]).to eq :L_BRACKET
- expect(@result[1][0]).to eq :R_BRACKET
- expect(@result[2][0]).to eq :OPERATOR_AND
- expect(@result[3][0]).to eq :L_BRACKET
- expect(@result[4][0]).to eq :R_BRACKET
- expect(@result[5][0]).to eq :OPERATOR_AND
- expect(@result[6][0]).to eq :L_BRACKET
- expect(@result[7][0]).to eq :R_BRACKET
- end
-
- it 'tests expression with NOT OR and NOT AND operator' do
- @result = @evaluator.tokenize('() NOT or () not AND ()')
- expect(@result.length).to eq 10
-
- expect(@result[0][0]).to eq :L_BRACKET
- expect(@result[1][0]).to eq :R_BRACKET
- expect(@result[2][0]).to eq :OPERATOR_NOT
- expect(@result[3][0]).to eq :OPERATOR_OR
- expect(@result[4][0]).to eq :L_BRACKET
- expect(@result[5][0]).to eq :R_BRACKET
- expect(@result[6][0]).to eq :OPERATOR_NOT
- expect(@result[7][0]).to eq :OPERATOR_AND
- expect(@result[8][0]).to eq :L_BRACKET
- expect(@result[9][0]).to eq :R_BRACKET
- end
-
- it 'tests search term under quotes' do
- @result = @evaluator.tokenize('"123-456"')
- expect(@result.length).to eq 1
-
- expect(@result[0][0]).to eq :TERM_WITH_QUOTES
- expect(@result[0][1]).to eq '"123-456"'
- end
-
- it 'tests term without quotes' do
- @result = @evaluator.tokenize('device_id')
- expect(@result.length).to eq 1
-
- expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[0][1]).to eq 'device_id'
- end
-
- it 'tests integer term without quotes' do
- @result = @evaluator.tokenize('123')
- expect(@result.length).to eq 1
-
- expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[0][1]).to eq '123'
- end
-
- it 'tests multiple terms without quotes' do
- @result = @evaluator.tokenize('device_id tag 123-456 name123')
-
- expect(@result.length).to eq 4
-
- expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[0][1]).to eq 'device_id'
- expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[1][1]).to eq 'tag'
- expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[2][1]).to eq '123-456'
- expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[3][1]).to eq 'name123'
- end
-
- it 'tests simple query with column name and search term without quotes' do
- @result = @evaluator.tokenize('name:JF')
-
- expect(@result.length).to eq 3
-
- expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[0][1]).to eq 'name'
- expect(@result[1][0]).to eq :COLON
- expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[2][1]).to eq 'JF'
- end
-
- it 'tests simple query with two columns with name and search terms without quotes' do
- @result = @evaluator.tokenize('name:JF tag:mta')
-
- expect(@result.length).to eq 6
-
- expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[0][1]).to eq 'name'
- expect(@result[1][0]).to eq :COLON
- expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[2][1]).to eq 'JF'
- expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[3][1]).to eq 'tag'
- expect(@result[4][0]).to eq :COLON
- expect(@result[5][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[5][1]).to eq 'mta'
-
- end
-
- it 'tests simple query with column name and search term with quotes' do
- @result = @evaluator.tokenize('name:"name with space"')
-
- expect(@result.length).to eq 3
-
- expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[0][1]).to eq 'name'
- expect(@result[1][0]).to eq :COLON
- expect(@result[2][0]).to eq :TERM_WITH_QUOTES
- expect(@result[2][1]).to eq '"name with space"'
- end
-
- it 'tests search term with quotes containing non alphanumerical characters' do
- @result = @evaluator.tokenize('"|*|/\()#-!=<>&$"')
-
- expect(@result.length).to eq 1
-
- expect(@result[0][0]).to eq :TERM_WITH_QUOTES
- expect(@result[0][1]).to eq '"|*|/\()#-!=<>&$"'
- end
-
- it 'tests simple query in brackets' do
- @result = @evaluator.tokenize('(name:"name with space")')
-
- expect(@result.length).to eq 5
-
- expect(@result[0][0]).to eq :L_BRACKET
- expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[1][1]).to eq 'name'
- expect(@result[2][0]).to eq :COLON
- expect(@result[3][0]).to eq :TERM_WITH_QUOTES
- expect(@result[3][1]).to eq '"name with space"'
- expect(@result[4][0]).to eq :R_BRACKET
- end
-
- it 'tests multiple query wtih brackets' do
- @result = @evaluator.tokenize('(name:"name with space") or (tag:mta)')
-
- expect(@result.length).to eq 11
-
- expect(@result[0][0]).to eq :L_BRACKET
- expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[1][1]).to eq 'name'
- expect(@result[2][0]).to eq :COLON
- expect(@result[3][0]).to eq :TERM_WITH_QUOTES
- expect(@result[3][1]).to eq '"name with space"'
- expect(@result[4][0]).to eq :R_BRACKET
- expect(@result[5][0]).to eq :OPERATOR_OR
- expect(@result[6][0]).to eq :L_BRACKET
- expect(@result[7][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[7][1]).to eq 'tag'
- expect(@result[8][0]).to eq :COLON
- expect(@result[9][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[9][1]).to eq 'mta'
- expect(@result[10][0]).to eq :R_BRACKET
-
- end
-
- it 'tests complex query' do
- @result = @evaluator.tokenize('(device-id:"with space" tag:mta no-quotes-id-123)'\
- 'or "id with quotes-5" and ( ("id with q 10" or "id with q 20")'\
- 'and ("id with Q 30" "id with Q 40") and not id-without-Q-50)')
-
- expect(@result.length).to eq 27
-
- expect(@result[0][0]).to eq :L_BRACKET
- expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[1][1]).to eq 'device-id'
- expect(@result[2][0]).to eq :COLON
- expect(@result[3][0]).to eq :TERM_WITH_QUOTES
- expect(@result[3][1]).to eq '"with space"'
- expect(@result[4][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[4][1]).to eq 'tag'
- expect(@result[5][0]).to eq :COLON
- expect(@result[6][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[6][1]).to eq 'mta'
- expect(@result[7][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[7][1]).to eq 'no-quotes-id-123'
- expect(@result[8][0]).to eq :R_BRACKET
-
- expect(@result[9][0]).to eq :OPERATOR_OR
- expect(@result[10][0]).to eq :TERM_WITH_QUOTES
- expect(@result[10][1]).to eq '"id with quotes-5"'
- expect(@result[11][0]).to eq :OPERATOR_AND
-
- expect(@result[12][0]).to eq :L_BRACKET
- expect(@result[13][0]).to eq :L_BRACKET
- expect(@result[14][0]).to eq :TERM_WITH_QUOTES
- expect(@result[14][1]).to eq '"id with q 10"'
- expect(@result[15][0]).to eq :OPERATOR_OR
- expect(@result[16][0]).to eq :TERM_WITH_QUOTES
- expect(@result[16][1]).to eq '"id with q 20"'
- expect(@result[17][0]).to eq :R_BRACKET
-
- expect(@result[18][0]).to eq :OPERATOR_AND
- expect(@result[19][0]).to eq :L_BRACKET
- expect(@result[20][0]).to eq :TERM_WITH_QUOTES
- expect(@result[20][1]).to eq '"id with Q 30"'
- expect(@result[21][0]).to eq :TERM_WITH_QUOTES
- expect(@result[21][1]).to eq '"id with Q 40"'
- expect(@result[22][0]).to eq :R_BRACKET
-
- expect(@result[23][0]).to eq :OPERATOR_AND
- expect(@result[24][0]).to eq :OPERATOR_NOT
- expect(@result[25][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[25][1]).to eq 'id-without-Q-50'
- expect(@result[26][0]).to eq :R_BRACKET
- end
-
- it 'tests query with -or-, -and- and -not- words inside quoted expression' do
- @result = @evaluator.tokenize('tag:"tag with or and not inside"')
-
- expect(@result.length).to eq 3
-
- expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES
- expect(@result[0][1]).to eq 'tag'
- expect(@result[1][0]).to eq :COLON
- expect(@result[2][0]).to eq :TERM_WITH_QUOTES
- expect(@result[2][1]).to eq '"tag with or and not inside"'
- end
- end
-end
diff --git a/lib/parser-parser-part/spec/query_parser_spec.rb b/lib/parser-parser-part/spec/query_parser_spec.rb
deleted file mode 100644
index f26c4a3..0000000
--- a/lib/parser-parser-part/spec/query_parser_spec.rb
+++ /dev/null
@@ -1,164 +0,0 @@
-require './parser'
-
-class QueryParserTester
- describe 'Testing the Parser' do
- before do
- @evaluator = Query.new
- end
-
- it 'tests query with only one search term without quotes and without column name' do
- @result = @evaluator.parse('-123')
-
- expect(@result[:DEFAULT_COLUMN]).to eq '-123'
- end
-
- it 'tests query with only one search term with quotes and without column name' do
- @result = @evaluator.parse('"OR 128"')
-
- expect(@result[:DEFAULT_COLUMN]).to eq '"OR 128"'
- end
-
- it 'tests query with one column and search term without quotes' do
- @result = @evaluator.parse('tag:mta')
-
- expect(@result['tag']).to eq 'mta'
- end
-
- it 'tests query with one column and search term with quotes' do
- @result = @evaluator.parse('tag:"tag 120"')
-
- expect(@result['tag']).to eq '"tag 120"'
- end
-
- it 'tests query with two columns connected with OR and search terms without quotes' do
- @result = @evaluator.parse('tag:mta OR tag:12')
-
- @expected_array = [
- { 'tag' => 'mta' },
- { 'tag' => '12' }
- ]
-
- expect(@result.count).to eq 1
- expect(@result[:OPERATOR_OR]).to eq @expected_array
- end
-
- it 'tests query with two columns connected with OR and search terms with quotes' do
- @result = @evaluator.parse('tag:mta OR tag:"tag 12"')
-
- @expected_array = [
- { 'tag' => 'mta' },
- { 'tag' => '"tag 12"' }
- ]
-
- expect(@result.count).to eq 1
- expect(@result[:OPERATOR_OR]).to eq @expected_array
- end
-
- it 'tests query with two columns connected with AND and search terms without quotes' do
- @result = @evaluator.parse('tag:mta AND tag:12')
-
- @expected_array = [
- { 'tag' => 'mta' },
- { 'tag' => '12' }
- ]
-
- expect(@result.count).to eq 1
- expect(@result[:OPERATOR_AND]).to eq @expected_array
- end
-
- it 'tests query with two columns connected with AND and search terms with quotes' do
- @result = @evaluator.parse('tag:mta and tag:"tag 12"')
-
- @expected_array = [
- { 'tag' => 'mta' },
- { 'tag' => '"tag 12"' }
- ]
-
- expect(@result.count).to eq 1
- expect(@result[:OPERATOR_AND]).to eq @expected_array
- end
-
- it 'tests simple query with brackets' do
- @result = @evaluator.parse('(123)')
-
- expect(@result.count).to eq 1
- expect(@result[:DEFAULT_COLUMN]).to eq '123'
- end
-
- it 'tests simple query with brackets and with a column name' do
- @result = @evaluator.parse('(name:JF)')
-
- expect(@result.count).to eq 1
- expect(@result['name']).to eq 'JF'
- end
-
- it 'tests query with OR operator in brackets' do
- @result = @evaluator.parse('(name:JF or tag:mta)')
-
- @expected_array = [
- { 'name' => 'JF' },
- { 'tag' => 'mta' }
- ]
-
- expect(@result.count).to eq 1
- expect(@result[:OPERATOR_OR]).to eq @expected_array
- end
-
- it 'tests query with two simple brackets expressions' do
- @result = @evaluator.parse('(name:JF) and (-456)')
-
- @expected_array = [
- { 'name' => 'JF' },
- { :DEFAULT_COLUMN => '-456' }
- ]
-
- expect(@result.count).to eq 1
- expect(@result[:OPERATOR_AND]).to eq @expected_array
- end
-
- it 'tests query with two brackets expressions' do
- @result = @evaluator.parse('(name:JF or tag:"tag 0") and (-456)')
-
- @expected_array_part_1 = [
- { 'name' => 'JF' },
- { 'tag' => '"tag 0"' }
- ]
-
- @expected_array_total = [
- {:OPERATOR_OR => @expected_array_part_1},
- {:DEFAULT_COLUMN => '-456'}
- ]
-
- expect(@result.count).to eq 1
- expect(@result[:OPERATOR_AND]).to eq @expected_array_total
- end
-
- it 'tests operator precedence' do
- @result1 = @evaluator.parse('tag:mta or name:JF and 12_4')
- @result2 = @evaluator.parse('tag:mta or (name:JF and 12_4)')
-
- expect(@result1).to eq @result2
-
- expect(@result1.length).to eq 1
-
- @expected_array_part_2 = [
- {'name' => 'JF'},
- {:DEFAULT_COLUMN => '12_4'}
- ]
-
- @expected_array_total = [
- {'tag' => 'mta'},
- {:OPERATOR_AND => @expected_array_part_2}
- ]
-
- expect(@result1[:OPERATOR_OR]).to eq @expected_array_total
-
- end
-
- # Tests to write :
- # * query with multiple column names and search terms without logical operators
- # * AND NOT, OR NOT tests
-
-
- end
-end
diff --git a/lib/parser-parser-part/specification.rex b/lib/parser-parser-part/specification.rex
deleted file mode 100644
index 4ac4649..0000000
--- a/lib/parser-parser-part/specification.rex
+++ /dev/null
@@ -1,35 +0,0 @@
-class Query
-macro
- L_BRACKET \(
- R_BRACKET \)
- SPACE \ + # Space char
- OPERATOR_OR (?i)or
- OPERATOR_AND (?i)and
- OPERATOR_NOT (?i)not
- TERM_WITH_QUOTES "([^"]*)"
- TERM_WITHOUT_QUOTES [a-zA-Z0-9-_]+
- COLON \:
-
-
-
-rule
- {SPACE} # No action
- {L_BRACKET} { return [:L_BRACKET, text] }
- {R_BRACKET} { return [:R_BRACKET, text] }
- {OPERATOR_OR} { return [:OPERATOR_OR, text] }
- {OPERATOR_AND} { return [:OPERATOR_AND, text] }
- {OPERATOR_NOT} { return [:OPERATOR_NOT, text] }
- {TERM_WITH_QUOTES} { return [:TERM_WITH_QUOTES, text] }
- {TERM_WITHOUT_QUOTES} { return [:TERM_WITHOUT_QUOTES, text] }
- {COLON} { return [:COLON, text] }
-
-inner
- def tokenize(code)
- scan_setup(code)
- tokens = []
- while token = next_token
- tokens << token
- end
- tokens
- end
-end
\ No newline at end of file
diff --git a/lib/parser.rb b/lib/parser.rb
new file mode 100644
index 0000000..58fc3c7
--- /dev/null
+++ b/lib/parser.rb
@@ -0,0 +1,181 @@
+#
+# DO NOT MODIFY!!!!
+# This file is automatically generated by Racc 1.4.16
+# from Racc grammar file "".
+#
+
+require 'racc/parser.rb'
+
+ require_relative 'lexer'
+
+class Query < Racc::Parser
+
+module_eval(<<'...end grammar.y/module_eval...', 'grammar.y', 24)
+ def parse(input)
+ scan_str(input)
+ end
+...end grammar.y/module_eval...
+##### State transition tables begin ###
+
+racc_action_table = [
+ 8, 7, 3, 4, 11, 5, 16, 3, 4, 9,
+ 5, 3, 4, 6, 5, 3, 4, 8, 5, 8,
+ 7, 14, 15 ]
+
+racc_action_check = [
+ 10, 10, 8, 8, 6, 8, 10, 0, 0, 3,
+ 0, 5, 5, 1, 5, 7, 7, 12, 7, 2,
+ 2, 9, 9 ]
+
+racc_action_pointer = [
+ 2, 13, 16, 2, nil, 6, 4, 10, -3, 16,
+ -3, nil, 14, nil, nil, nil, nil ]
+
+racc_action_default = [
+ -2, -10, -1, -3, -4, -10, -10, -10, -10, -10,
+ -10, 17, -7, -8, -5, -6, -9 ]
+
+racc_goto_table = [
+ 2, 1, nil, nil, nil, 10, nil, 12, 13 ]
+
+racc_goto_check = [
+ 2, 1, nil, nil, nil, 2, nil, 2, 2 ]
+
+racc_goto_pointer = [
+ nil, 1, 0 ]
+
+racc_goto_default = [
+ nil, nil, nil ]
+
+racc_reduce_table = [
+ 0, 0, :racc_error,
+ 1, 11, :_reduce_none,
+ 0, 11, :_reduce_2,
+ 1, 12, :_reduce_3,
+ 1, 12, :_reduce_4,
+ 3, 12, :_reduce_5,
+ 3, 12, :_reduce_6,
+ 3, 12, :_reduce_7,
+ 3, 12, :_reduce_8,
+ 3, 12, :_reduce_9 ]
+
+racc_reduce_n = 10
+
+racc_shift_n = 17
+
+racc_token_table = {
+ false => 0,
+ :error => 1,
+ :OPERATOR_NOT => 2,
+ :OPERATOR_AND => 3,
+ :OPERATOR_OR => 4,
+ :TERM_WITHOUT_QUOTES => 5,
+ :TERM_WITH_QUOTES => 6,
+ :COLON => 7,
+ :L_BRACKET => 8,
+ :R_BRACKET => 9 }
+
+racc_nt_base = 10
+
+racc_use_result_var = true
+
+Racc_arg = [
+ racc_action_table,
+ racc_action_check,
+ racc_action_default,
+ racc_action_pointer,
+ racc_goto_table,
+ racc_goto_check,
+ racc_goto_default,
+ racc_goto_pointer,
+ racc_nt_base,
+ racc_reduce_table,
+ racc_token_table,
+ racc_shift_n,
+ racc_reduce_n,
+ racc_use_result_var ]
+
+Racc_token_to_s_table = [
+ "$end",
+ "error",
+ "OPERATOR_NOT",
+ "OPERATOR_AND",
+ "OPERATOR_OR",
+ "TERM_WITHOUT_QUOTES",
+ "TERM_WITH_QUOTES",
+ "COLON",
+ "L_BRACKET",
+ "R_BRACKET",
+ "$start",
+ "target",
+ "expression" ]
+
+Racc_debug_parser = false
+
+##### State transition tables end #####
+
+# reduce 0 omitted
+
+# reduce 1 omitted
+
+module_eval(<<'.,.,', 'grammar.y', 8)
+ def _reduce_2(val, _values, result)
+ result = 0
+ result
+ end
+.,.,
+
+module_eval(<<'.,.,', 'grammar.y', 10)
+ def _reduce_3(val, _values, result)
+ result = {:DEFAULT_COLUMN => val[0]}
+ result
+ end
+.,.,
+
+module_eval(<<'.,.,', 'grammar.y', 11)
+ def _reduce_4(val, _values, result)
+ result = {:DEFAULT_COLUMN => val[0]}
+ result
+ end
+.,.,
+
+module_eval(<<'.,.,', 'grammar.y', 12)
+ def _reduce_5(val, _values, result)
+ result = {val[0] => val[2]}
+ result
+ end
+.,.,
+
+module_eval(<<'.,.,', 'grammar.y', 13)
+ def _reduce_6(val, _values, result)
+ result = {val[0] => val[2]}
+ result
+ end
+.,.,
+
+module_eval(<<'.,.,', 'grammar.y', 14)
+ def _reduce_7(val, _values, result)
+ result = {:OPERATOR_OR => [val[0], val[2]]}
+ result
+ end
+.,.,
+
+module_eval(<<'.,.,', 'grammar.y', 15)
+ def _reduce_8(val, _values, result)
+ result = {:OPERATOR_AND => [val[0], val[2]]}
+ result
+ end
+.,.,
+
+module_eval(<<'.,.,', 'grammar.y', 16)
+ def _reduce_9(val, _values, result)
+ result = val[1]
+ result
+ end
+.,.,
+
+def _reduce_none(val, _values, result)
+ val[0]
+end
+
+end # class Query
diff --git a/lib/pg_searchable_regex.rb b/lib/pg_searchable_regex.rb
index 1dc287c..4387373 100644
--- a/lib/pg_searchable_regex.rb
+++ b/lib/pg_searchable_regex.rb
@@ -3,7 +3,7 @@
require 'active_support'
require 'squeel'
require_relative './text_to_tsquery'
-require_relative './text_to_regex_query'
+require_relative './text_to_sql_query'
module PgSearchable
extend ActiveSupport::Concern
@@ -48,7 +48,8 @@ module PgSearchable
def ts_search(value)
return if @ts_search_fields.blank? || value.blank?
- TextToRegexQuery.new(value, @ts_search_fields, @default_field, @ts_search_fields_mappings).where_clause( includes(@ts_joins).references(:all))
+ includes(@ts_joins).references(:all).where(
+ TextToSqlQuery.new(value, @ts_search_fields, @default_field, @ts_search_fields_mappings).where_clause)
end
def should_update_cache_field?
diff --git a/lib/specification.rex b/lib/specification.rex
index 4ac4649..c92a37e 100644
--- a/lib/specification.rex
+++ b/lib/specification.rex
@@ -7,7 +7,7 @@ macro
OPERATOR_AND (?i)and
OPERATOR_NOT (?i)not
TERM_WITH_QUOTES "([^"]*)"
- TERM_WITHOUT_QUOTES [a-zA-Z0-9-_]+
+ TERM_WITHOUT_QUOTES [a-zA-Z0-9\-_]+
COLON \:
@@ -32,4 +32,4 @@ inner
end
tokens
end
-end
\ No newline at end of file
+end
diff --git a/lib/text_to_sql_query.rb b/lib/text_to_sql_query.rb
new file mode 100644
index 0000000..c7cc6f8
--- /dev/null
+++ b/lib/text_to_sql_query.rb
@@ -0,0 +1,92 @@
+require_relative 'parser'
+
+class TextToSqlQuery
+ def initialize(text, fields, default_field, fields_mappings = {})
+ @text = text.to_s.strip
+ @fields = fields.map(&:to_sym)
+ @default_field = default_field.to_sym
+ @fields_mappings = fields_mappings.merge(@fields.reduce({}) do |mappings, field|
+ _table_name, field_name = field.to_s.split('.')
+ mappings[field_name.to_sym] = field
+ mappings
+ end)
+ fields_mappings.each do |field, value|
+ @fields_mappings[field] = value if @fields_mappings[field]
+ end
+ end
+
+ def where_clause
+ @parser = Query.new
+ @parsed_tree = @parser.parse(@text)
+ generate_sql @parsed_tree
+ end
+
+ private
+
+ def generate_sql(tree)
+ first_key = tree.keys.first
+ node_value = tree[first_key]
+ case first_key
+ when :DEFAULT_COLUMN
+ escaped_node_value = handle_special_chars node_value
+ ["#{@default_field.to_s} ILIKE ?", "%#{escaped_node_value}%"]
+ when :OPERATOR_OR
+ generate_expression_for_logical_operator(:OR, node_value)
+ when :OPERATOR_AND
+ generate_expression_for_logical_operator(:AND, node_value)
+ when :OPERATOR_NOT
+ not_array = generate_sql node_value
+
+ if not_array.length < 2
+ raise "There should be more than 1 element for expression following NOT operator"
+ end
+
+ not_expression = not_array.shift
+
+ ["NOT #{not_expression}"] + not_params
+
+ else
+ # key is column name
+ escaped_node_value = handle_special_chars node_value
+ mapping = @fields_mappings[first_key.to_sym]
+ if mapping.nil?
+ raise "Unknown field '#{first_key.to_s}'"
+ else
+ ["#{mapping.to_s} ILIKE ?", "%#{escaped_node_value}%"]
+ end
+ end
+ end
+
+ def generate_expression_for_logical_operator(operator, operator_array)
+ if operator_array.length != 2
+ raise "There should be two array elements for #{operator.to_s} operator"
+ end
+
+ first_operand = generate_sql operator_array.first
+ second_operand = generate_sql operator_array.last
+
+ if first_operand.length < 2
+ raise 'There should be more than 1 element in first operand array'
+ end
+
+ if second_operand.length < 2
+ raise 'There should be more than 1 element in second operand array'
+ end
+
+ first_operand_expression = first_operand.shift
+ first_operand_params = first_operand
+
+ second_operand_expression = second_operand.shift
+ second_operand_params = second_operand
+
+ ["(#{first_operand_expression} #{operator.to_s} #{second_operand_expression})"] + first_operand_params + second_operand_params
+ end
+
+ def handle_special_chars(text)
+ result = text.gsub(/\"/, '')
+ result.gsub!(/\_/, '\_')
+ result.tr!('\\', '\\')
+ result.gsub!(/%/, '\%')
+ result
+ end
+end
diff --git a/pkg/pg_searchable_regex-1.0.13.gem b/pkg/pg_searchable_regex-1.0.13.gem
deleted file mode 100644
index 118da0b..0000000
Binary files a/pkg/pg_searchable_regex-1.0.13.gem and /dev/null differ
diff --git a/pkg/pg_searchable_regex-1.0.21.gem b/pkg/pg_searchable_regex-1.0.21.gem
new file mode 100644
index 0000000..1d3941a
Binary files /dev/null and b/pkg/pg_searchable_regex-1.0.21.gem differ
diff --git a/spec/lib/text_to_sql_query_spec.rb b/spec/lib/text_to_sql_query_spec.rb
new file mode 100644
index 0000000..fd76b8f
--- /dev/null
+++ b/spec/lib/text_to_sql_query_spec.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+describe TextToRegexQuery do
+ include_examples 'pg_search', SimpleVectorModel
+ describe '.new' do
+ # tests simple search term without column name and without quotes
+ it { expect(described_class.new('some-default-value', [:"players.name"], :"players.name").where_clause(SimpleVectorModel)).to eq(['players.name ILIKE ?', '%some-default-value%']) }
+
+ # tests simple search term with column name and without quotes
+ it { expect(described_class.new('title:ab', [:"players.title"], :"players.title").where_clause(SimpleVectorModel)).to eq(['players.title ILIKE ?', '%ab%']) }
+
+ # tests simple search term with unknown column name and without quotes
+ it { expect(described_class.new('unknown:ab', [:"players.name"], :"players.name").where_clause(SimpleVectorModel)).to raise_error(RuntimeError, "Unknown field 'unknown'") }
+
+ # tests simple search term without column name and with quotes
+ it { expect(described_class.new('"ab"', [:"players.name", :"players.device_id"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(["players.device_id ILIKE ?", "%ab"]) }
+
+ # tests simple search term with column name and with quotes
+ it { expect(described_class.new('tag:"ab"', [:"players.name", :"players.tag"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(["players.tag ILIKE ?", "%ab%"]) }
+
+ # tests search without operators
+ it { expect(described_class.new('123 456', [:"players.name", :"players.device_id"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(["(players.device_id ILIKE ? OR players.device_id ILIKE ?)", "%123%", "%456%"]) }
+
+ # tests search with OR operator
+ it { expect(described_class.new('123 or 456', [:"players.name", :"players.device_id"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(["(players.device_id ILIKE ? OR players.device_id ILIKE ?)", "%123%", "%456%"]) }
+
+ # tests search with AND operator
+ it { expect(described_class.new('123 and 456', [:"players.name", :"players.device_id"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(["(players.device_id ILIKE ? AND players.device_id ILIKE ?)", "%123%", "%456%"]) }
+
+ # tests search with NOT operator on default column
+ it { expect(described_class.new('not 23', [:"players.name", :"players.device_id"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(["NOT players.device_id ILIKE ?", "%23%"]) }
+
+ # tests search with NOT operator on non-default column
+ it { expect(described_class.new('not tag:23', [:"players.name", :"players.tag"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(["NOT players.tag ILIKE ?", "%23%"]) }
+
+ # tests search with mixed logical operators
+ it { expect(described_class.new('title:ab and not tag:hf-1', [:"players.title", :"players.tag"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(['(players.title ILIKE ? AND NOT players.tag ILIKE ?)', '%ab%', '%hf-1%']) }
+
+ # tests search with mixed logical operators without NOT'
+ it { expect(described_class.new('title:a and title:b or title:c', [:"players.title", :"players.tag"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(['((players.title ILIKE ? AND players.title ILIKE ?) OR players.title ILIKE ?)', '%a%', '%b%', '%c%']) }
+
+ # tests search with brackets in expression
+ it { expect(described_class.new('title:a and (title:b or title:c)', [:"players.title", :"players.tag"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(['(players.title ILIKE ? AND (players.title ILIKE ? OR players.title ILIKE ?))', '%a%', '%b%', '%c%']) }
+
+ # tests search with brackets in expression and with NOT operator
+ it { expect(described_class.new('title:a and not (title:b or title:c)', [:"players.title", :"players.tag"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(['(players.title ILIKE ? AND NOT (players.title ILIKE ? OR players.title ILIKE ?))', '%a%', '%b%', '%c%']) }
+
+ # tests search with special characters in search term
+ it { expect(described_class.new('title:"%a_\"', [:"players.title", :"players.tag"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(['players.title ILIKE ?', '%\%a\_\\%']) }
+
+ # tests search with field mappings
+ it { expect(described_class.new('tags:h1-r', [:'players.title', :'players.name', :'players.device_id'], :"players.device_id", { tags: "tags.name" }).where_clause(SimpleVectorModel)).to eq(['tags.name ILIKE ?', '%h1-r%']) }
+
+ # tests search with field mappings when fields array has same mapping
+ it { expect(described_class.new('tags:hs1-r', [:'players.title', :'players.tags', :'players.device_id'], :"players.device_id", { tags: "tags.name" }).where_clause(SimpleVectorModel)).to eq(["players.tag ILIKE ?", "%ab%"]) }
+
+ # tests complex query
+ it { expect(described_class.new('(device_id:"with space" tags:mta no-quotes-id-123) or "id with quotes-5" and ( ("id with q 10" or "id with q 20") and ("id with Q 30" "id with Q 40") and not id-without-Q-50)', [:'players.title', :'players.name', :'players.device_id'], :"players.device_id", { tags: 'tags.name' }).where_clause(SimpleVectorModel)).to eq(['((players.device_id ILIKE ? OR (tags.name ILIKE ? OR players.device_id ILIKE ?)) OR (players.device_id ILIKE ? AND (((players.device_id ILIKE ? OR players.device_id ILIKE ?) AND (players.device_id ILIKE ? OR players.device_id ILIKE ?)) AND NOT players.device_id ILIKE ?)))', '%with space%', '%mta%', '%no-quotes-id-123%', '%id with quotes-5%', '%id with q 10%', '%id with q 20%', '%id with Q 30%', '%id with Q 40%', '%id-without-Q-50%']) }
+
+ end
+end
diff --git a/version.rb b/version.rb
index cbff164..fc5959a 100644
--- a/version.rb
+++ b/version.rb
@@ -3,7 +3,7 @@
module Version
MAJOR = 1
MINOR = 0
- PATCH = 13
+ PATCH = 21
def self.to_s
[MAJOR, MINOR, PATCH].compact.join('.')
