senaduka/old-gem
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Upstream sync

Browse Source
This commit is contained in:
Senad Uka
2020-02-05 13:40:37 +01:00
parent 2749c53aac
commit 530951ff45
22 changed files with 1016 additions and 885 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Gemfile
View File

@@ -23,4 +23,6 @@ group :development do
gem 'rubocop', '~> 0.54', require: false
gem 'rubocop-rspec', '~> 1.23', require: false
gem 'simplecov', '~> 0.16', require: false
gem "rexical"
gem "racc"
end

4
Gemfile.lock
View File

@@ -76,11 +76,13 @@ GEM
coderay (~> 1.1.0)
method_source (~> 0.9.0)
public_suffix (3.1.1)
racc (1.4.16)
rack (1.6.11)
rainbow (3.0.0)
rake (12.3.2)
rdoc (3.12.2)
json (~> 1.4)
rexical (1.0.7)
rspec (3.8.0)
rspec-core (~> 3.8.0)
rspec-expectations (~> 3.8.0)
@@ -132,7 +134,9 @@ DEPENDENCIES
juwelier (~> 2.1)
pg (~> 0.15)
pry (~> 0.12)
racc
rdoc (~> 3.12)
rexical
rspec (~> 3.8)
rubocop (~> 0.54)
rubocop-rspec (~> 1.23)

8
README.md
View File

@@ -119,3 +119,11 @@ To run the test suite create `.env.test.local` file containing the same entries
## CONTRIBUTING
Make sure the test coverage remains at 100%, there are no rubocop complaints (`bundle exec rubocop`) and make a Pull Request.
## Modifying parser and lexer
* `rake lexer` - generates `lexer.rb` file based on `specification.rex` file
* `rake parser` - generates `parser.rb` file based on `grammar.y` file
* `rake generate` - generates both `lexer.rb` and `parser.rb` files

13
Rakefile
View File

@@ -44,3 +44,16 @@ Rake::RDocTask.new do |rdoc|
rdoc.rdoc_files.include('README*')
rdoc.rdoc_files.include('lib/**/*.rb')
end
desc 'Generate Lexer'
task :lexer do
`rex ./lib/specification.rex -o ./lib/lexer.rb`
end
desc 'Generate Parser'
task :parser do
`racc ./lib/grammar.y -o ./lib/parser.rb`
end
desc 'Generate Lexer and Parser'
task generate: %i[lexer parser]

223
coverage/.resultset.json
View File

@@ -9,69 +9,69 @@
1,
1,
null,
1,
1,
0,
0,
null,
1,
12,
null,
null,
null,
null,
null,
1,
1,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
12,
12,
12,
12,
12,
12,
12,
12,
12,
12,
null,
null,
1,
12,
43,
null,
null,
null,
1,
31,
30,
null,
null,
1,
0,
0,
null,
null,
1,
null,
null,
null,
0,
0,
null,
null,
1,
12,
null,
null,
1,
null,
null,
null,
null,
null,
null,
null,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
null,
null,
1,
0,
0,
0,
null,
null,
null,
0,
0,
0,
null,
null,
0,
0,
null,
null,
0,
0,
null,
null,
0,
0,
null,
null,
0,
0,
null,
null,
0,
0,
0,
0,
@@ -180,75 +180,102 @@
null,
null
],
"/home/hamo/projects/toptal/outfrontmedia/pg_searchable/lib/text_to_regex_query.rb": [
"/home/hamo/projects/toptal/outfrontmedia/pg_searchable/lib/text_to_sql_query.rb": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
1,
1,
35,
35,
35,
35,
37,
37,
7,
null,
null,
null,
1,
5,
5,
5,
5,
5,
5,
null,
null,
1,
null,
1,
5,
null,
null,
1,
5,
5,
5,
null,
null,
1,
5,
null,
5,
7,
7,
5,
5,
null,
5,
3,
null,
5,
null,
null,
1,
5,
13,
13,
null,
5,
8,
null,
5,
null,
5,
null,
null
]
},
"timestamp": 1579269179
"timestamp": 1580827025
}
}

672
coverage/index.html
View File

File diff suppressed because it is too large Load Diff

107
lib/lexer.rb Normal file
View File

@@ -0,0 +1,107 @@
#--
# DO NOT MODIFY!!!!
# This file is automatically generated by rex 1.0.7
# from lexical definition file "./lib/specification.rex".
#++
require 'racc/parser'
class Query < Racc::Parser
require 'strscan'
class ScanError < StandardError ; end
attr_reader :lineno
attr_reader :filename
attr_accessor :state
def scan_setup(str)
@ss = StringScanner.new(str)
@lineno = 1
@state = nil
end
def action
yield
end
def scan_str(str)
scan_setup(str)
do_parse
end
alias :scan :scan_str
def load_file( filename )
@filename = filename
File.open(filename, "r") do |f|
scan_setup(f.read)
end
end
def scan_file( filename )
load_file(filename)
do_parse
end
def next_token
return if @ss.eos?
# skips empty actions
until token = _next_token or @ss.eos?; end
token
end
def _next_token
text = @ss.peek(1)
@lineno += 1 if text == "\n"
token = case @state
when nil
case
when (text = @ss.scan(/ +/))
;
when (text = @ss.scan(/\(/))
action { return [:L_BRACKET, text] }
when (text = @ss.scan(/\)/))
action { return [:R_BRACKET, text] }
when (text = @ss.scan(/(?i)or/))
action { return [:OPERATOR_OR, text] }
when (text = @ss.scan(/(?i)and/))
action { return [:OPERATOR_AND, text] }
when (text = @ss.scan(/(?i)not/))
action { return [:OPERATOR_NOT, text] }
when (text = @ss.scan(/"([^"]*)"/))
action { return [:TERM_WITH_QUOTES, text] }
when (text = @ss.scan(/[a-zA-Z0-9\-_]+/))
action { return [:TERM_WITHOUT_QUOTES, text] }
when (text = @ss.scan(/\:/))
action { return [:COLON, text] }
else
text = @ss.string[@ss.pos .. -1]
raise ScanError, "can not match: '" + text + "'"
end # if
else
raise ScanError, "undefined state: '" + state.to_s + "'"
end # case state
token
end # def _next_token
def tokenize(code)
scan_setup(code)
tokens = []
while token = next_token
tokens << token
end
tokens
end
end # class

4
lib/parser-parser-part/.gitignore vendored
View File

@@ -1,4 +0,0 @@
.idea
lexer.rb
parser.rb

19
lib/parser-parser-part/README.md
View File

@@ -1,19 +0,0 @@
# parser
### Prerequisites
* Rexical (rex)
* Racc
### Available commands
* `rake lexer` - generates `lexer.rb` file based on `specification.rex` file
* `rake parser` - generates `parser.rb` file based on `grammar.y` file
* `rake generate` - generates `lexer.rb` and `parser.rb` files
### Testing
To run only `lexer` tests, execute : `rspec spec/query_lexer_spec.rb`
To run only `parser` tests, execute : `rspec spec/query_parser_spec.rb`
To run all tests, execute : `rake spec`

20
lib/parser-parser-part/Rakefile
View File

@@ -1,20 +0,0 @@
require 'rspec/core/rake_task'
RSpec::Core::RakeTask.new do |c|
options = ['--color']
options += %w[--format documentation]
c.rspec_opts = options
end
desc 'Generate Lexer'
task :lexer do
`rex specification.rex -o lexer.rb`
end
desc 'Generate Parser'
task :parser do
`racc grammar.y -o parser.rb`
end
desc 'Generate Lexer and Parser'
task generate: %i[lexer parser]

26
lib/parser-parser-part/grammar.y
View File

@@ -1,26 +0,0 @@
class Query
prechigh
left OPERATOR_NOT
left OPERATOR_AND
left OPERATOR_OR
preclow
rule
target: expression
| /* none */ { result = 0 }
expression: TERM_WITHOUT_QUOTES { result = {:DEFAULT_COLUMN => val[0]} }
| TERM_WITH_QUOTES { result = {:DEFAULT_COLUMN => val[0]} }
| TERM_WITHOUT_QUOTES COLON TERM_WITHOUT_QUOTES { result = {val[0] => val[2]} }
| TERM_WITHOUT_QUOTES COLON TERM_WITH_QUOTES { result = {val[0] => val[2]} }
| expression OPERATOR_OR expression { result = {:OPERATOR_OR => [val[0], val[2]]} }
| expression OPERATOR_AND expression { result = {:OPERATOR_AND => [val[0], val[2]]} }
| L_BRACKET expression R_BRACKET { result = val[1] }
end
---- header
require_relative 'lexer'
---- inner
def parse(input)
scan_str(input)
end

259
lib/parser-parser-part/spec/query_lexer_spec.rb
View File

@@ -1,259 +0,0 @@
require './lexer'
class QueryLexerTester
describe 'Testing the Lexer' do
before do
@evaluator = Query.new
end
it 'tests bracket expression' do
@result = @evaluator.tokenize('()')
expect(@result.length).to eq 2
expect(@result[0][0]).to eq :L_BRACKET
expect(@result[1][0]).to eq :R_BRACKET
end
it 'tests bracket expression with spaces' do
@result = @evaluator.tokenize(' ( ) ')
expect(@result.length).to eq 2
expect(@result[0][0]).to eq :L_BRACKET
expect(@result[1][0]).to eq :R_BRACKET
end
it 'tests expression with OR operator' do
@result = @evaluator.tokenize('() or () OR ()')
expect(@result.length).to eq 8
expect(@result[0][0]).to eq :L_BRACKET
expect(@result[1][0]).to eq :R_BRACKET
expect(@result[2][0]).to eq :OPERATOR_OR
expect(@result[3][0]).to eq :L_BRACKET
expect(@result[4][0]).to eq :R_BRACKET
expect(@result[5][0]).to eq :OPERATOR_OR
expect(@result[6][0]).to eq :L_BRACKET
expect(@result[7][0]).to eq :R_BRACKET
end
it 'tests expression with AND operator' do
@result = @evaluator.tokenize('() AND () and ()')
expect(@result.length).to eq 8
expect(@result[0][0]).to eq :L_BRACKET
expect(@result[1][0]).to eq :R_BRACKET
expect(@result[2][0]).to eq :OPERATOR_AND
expect(@result[3][0]).to eq :L_BRACKET
expect(@result[4][0]).to eq :R_BRACKET
expect(@result[5][0]).to eq :OPERATOR_AND
expect(@result[6][0]).to eq :L_BRACKET
expect(@result[7][0]).to eq :R_BRACKET
end
it 'tests expression with NOT OR and NOT AND operator' do
@result = @evaluator.tokenize('() NOT or () not AND ()')
expect(@result.length).to eq 10
expect(@result[0][0]).to eq :L_BRACKET
expect(@result[1][0]).to eq :R_BRACKET
expect(@result[2][0]).to eq :OPERATOR_NOT
expect(@result[3][0]).to eq :OPERATOR_OR
expect(@result[4][0]).to eq :L_BRACKET
expect(@result[5][0]).to eq :R_BRACKET
expect(@result[6][0]).to eq :OPERATOR_NOT
expect(@result[7][0]).to eq :OPERATOR_AND
expect(@result[8][0]).to eq :L_BRACKET
expect(@result[9][0]).to eq :R_BRACKET
end
it 'tests search term under quotes' do
@result = @evaluator.tokenize('"123-456"')
expect(@result.length).to eq 1
expect(@result[0][0]).to eq :TERM_WITH_QUOTES
expect(@result[0][1]).to eq '"123-456"'
end
it 'tests term without quotes' do
@result = @evaluator.tokenize('device_id')
expect(@result.length).to eq 1
expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[0][1]).to eq 'device_id'
end
it 'tests integer term without quotes' do
@result = @evaluator.tokenize('123')
expect(@result.length).to eq 1
expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[0][1]).to eq '123'
end
it 'tests multiple terms without quotes' do
@result = @evaluator.tokenize('device_id tag 123-456 name123')
expect(@result.length).to eq 4
expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[0][1]).to eq 'device_id'
expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[1][1]).to eq 'tag'
expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[2][1]).to eq '123-456'
expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[3][1]).to eq 'name123'
end
it 'tests simple query with column name and search term without quotes' do
@result = @evaluator.tokenize('name:JF')
expect(@result.length).to eq 3
expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[0][1]).to eq 'name'
expect(@result[1][0]).to eq :COLON
expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[2][1]).to eq 'JF'
end
it 'tests simple query with two columns with name and search terms without quotes' do
@result = @evaluator.tokenize('name:JF tag:mta')
expect(@result.length).to eq 6
expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[0][1]).to eq 'name'
expect(@result[1][0]).to eq :COLON
expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[2][1]).to eq 'JF'
expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[3][1]).to eq 'tag'
expect(@result[4][0]).to eq :COLON
expect(@result[5][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[5][1]).to eq 'mta'
end
it 'tests simple query with column name and search term with quotes' do
@result = @evaluator.tokenize('name:"name with space"')
expect(@result.length).to eq 3
expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[0][1]).to eq 'name'
expect(@result[1][0]).to eq :COLON
expect(@result[2][0]).to eq :TERM_WITH_QUOTES
expect(@result[2][1]).to eq '"name with space"'
end
it 'tests search term with quotes containing non alphanumerical characters' do
@result = @evaluator.tokenize('"|*|/\()#-!=<>&$"')
expect(@result.length).to eq 1
expect(@result[0][0]).to eq :TERM_WITH_QUOTES
expect(@result[0][1]).to eq '"|*|/\()#-!=<>&$"'
end
it 'tests simple query in brackets' do
@result = @evaluator.tokenize('(name:"name with space")')
expect(@result.length).to eq 5
expect(@result[0][0]).to eq :L_BRACKET
expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[1][1]).to eq 'name'
expect(@result[2][0]).to eq :COLON
expect(@result[3][0]).to eq :TERM_WITH_QUOTES
expect(@result[3][1]).to eq '"name with space"'
expect(@result[4][0]).to eq :R_BRACKET
end
it 'tests multiple query wtih brackets' do
@result = @evaluator.tokenize('(name:"name with space") or (tag:mta)')
expect(@result.length).to eq 11
expect(@result[0][0]).to eq :L_BRACKET
expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[1][1]).to eq 'name'
expect(@result[2][0]).to eq :COLON
expect(@result[3][0]).to eq :TERM_WITH_QUOTES
expect(@result[3][1]).to eq '"name with space"'
expect(@result[4][0]).to eq :R_BRACKET
expect(@result[5][0]).to eq :OPERATOR_OR
expect(@result[6][0]).to eq :L_BRACKET
expect(@result[7][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[7][1]).to eq 'tag'
expect(@result[8][0]).to eq :COLON
expect(@result[9][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[9][1]).to eq 'mta'
expect(@result[10][0]).to eq :R_BRACKET
end
it 'tests complex query' do
@result = @evaluator.tokenize('(device-id:"with space" tag:mta no-quotes-id-123)'\
'or "id with quotes-5" and ( ("id with q 10" or "id with q 20")'\
'and ("id with Q 30" "id with Q 40") and not id-without-Q-50)')
expect(@result.length).to eq 27
expect(@result[0][0]).to eq :L_BRACKET
expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[1][1]).to eq 'device-id'
expect(@result[2][0]).to eq :COLON
expect(@result[3][0]).to eq :TERM_WITH_QUOTES
expect(@result[3][1]).to eq '"with space"'
expect(@result[4][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[4][1]).to eq 'tag'
expect(@result[5][0]).to eq :COLON
expect(@result[6][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[6][1]).to eq 'mta'
expect(@result[7][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[7][1]).to eq 'no-quotes-id-123'
expect(@result[8][0]).to eq :R_BRACKET
expect(@result[9][0]).to eq :OPERATOR_OR
expect(@result[10][0]).to eq :TERM_WITH_QUOTES
expect(@result[10][1]).to eq '"id with quotes-5"'
expect(@result[11][0]).to eq :OPERATOR_AND
expect(@result[12][0]).to eq :L_BRACKET
expect(@result[13][0]).to eq :L_BRACKET
expect(@result[14][0]).to eq :TERM_WITH_QUOTES
expect(@result[14][1]).to eq '"id with q 10"'
expect(@result[15][0]).to eq :OPERATOR_OR
expect(@result[16][0]).to eq :TERM_WITH_QUOTES
expect(@result[16][1]).to eq '"id with q 20"'
expect(@result[17][0]).to eq :R_BRACKET
expect(@result[18][0]).to eq :OPERATOR_AND
expect(@result[19][0]).to eq :L_BRACKET
expect(@result[20][0]).to eq :TERM_WITH_QUOTES
expect(@result[20][1]).to eq '"id with Q 30"'
expect(@result[21][0]).to eq :TERM_WITH_QUOTES
expect(@result[21][1]).to eq '"id with Q 40"'
expect(@result[22][0]).to eq :R_BRACKET
expect(@result[23][0]).to eq :OPERATOR_AND
expect(@result[24][0]).to eq :OPERATOR_NOT
expect(@result[25][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[25][1]).to eq 'id-without-Q-50'
expect(@result[26][0]).to eq :R_BRACKET
end
it 'tests query with -or-, -and- and -not- words inside quoted expression' do
@result = @evaluator.tokenize('tag:"tag with or and not inside"')
expect(@result.length).to eq 3
expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[0][1]).to eq 'tag'
expect(@result[1][0]).to eq :COLON
expect(@result[2][0]).to eq :TERM_WITH_QUOTES
expect(@result[2][1]).to eq '"tag with or and not inside"'
end
end
end

164
lib/parser-parser-part/spec/query_parser_spec.rb
View File

@@ -1,164 +0,0 @@
require './parser'
class QueryParserTester
describe 'Testing the Parser' do
before do
@evaluator = Query.new
end
it 'tests query with only one search term without quotes and without column name' do
@result = @evaluator.parse('-123')
expect(@result[:DEFAULT_COLUMN]).to eq '-123'
end
it 'tests query with only one search term with quotes and without column name' do
@result = @evaluator.parse('"OR 128"')
expect(@result[:DEFAULT_COLUMN]).to eq '"OR 128"'
end
it 'tests query with one column and search term without quotes' do
@result = @evaluator.parse('tag:mta')
expect(@result['tag']).to eq 'mta'
end
it 'tests query with one column and search term with quotes' do
@result = @evaluator.parse('tag:"tag 120"')
expect(@result['tag']).to eq '"tag 120"'
end
it 'tests query with two columns connected with OR and search terms without quotes' do
@result = @evaluator.parse('tag:mta OR tag:12')
@expected_array = [
{ 'tag' => 'mta' },
{ 'tag' => '12' }
]
expect(@result.count).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_array
end
it 'tests query with two columns connected with OR and search terms with quotes' do
@result = @evaluator.parse('tag:mta OR tag:"tag 12"')
@expected_array = [
{ 'tag' => 'mta' },
{ 'tag' => '"tag 12"' }
]
expect(@result.count).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_array
end
it 'tests query with two columns connected with AND and search terms without quotes' do
@result = @evaluator.parse('tag:mta AND tag:12')
@expected_array = [
{ 'tag' => 'mta' },
{ 'tag' => '12' }
]
expect(@result.count).to eq 1
expect(@result[:OPERATOR_AND]).to eq @expected_array
end
it 'tests query with two columns connected with AND and search terms with quotes' do
@result = @evaluator.parse('tag:mta and tag:"tag 12"')
@expected_array = [
{ 'tag' => 'mta' },
{ 'tag' => '"tag 12"' }
]
expect(@result.count).to eq 1
expect(@result[:OPERATOR_AND]).to eq @expected_array
end
it 'tests simple query with brackets' do
@result = @evaluator.parse('(123)')
expect(@result.count).to eq 1
expect(@result[:DEFAULT_COLUMN]).to eq '123'
end
it 'tests simple query with brackets and with a column name' do
@result = @evaluator.parse('(name:JF)')
expect(@result.count).to eq 1
expect(@result['name']).to eq 'JF'
end
it 'tests query with OR operator in brackets' do
@result = @evaluator.parse('(name:JF or tag:mta)')
@expected_array = [
{ 'name' => 'JF' },
{ 'tag' => 'mta' }
]
expect(@result.count).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_array
end
it 'tests query with two simple brackets expressions' do
@result = @evaluator.parse('(name:JF) and (-456)')
@expected_array = [
{ 'name' => 'JF' },
{ :DEFAULT_COLUMN => '-456' }
]
expect(@result.count).to eq 1
expect(@result[:OPERATOR_AND]).to eq @expected_array
end
it 'tests query with two brackets expressions' do
@result = @evaluator.parse('(name:JF or tag:"tag 0") and (-456)')
@expected_array_part_1 = [
{ 'name' => 'JF' },
{ 'tag' => '"tag 0"' }
]
@expected_array_total = [
{:OPERATOR_OR => @expected_array_part_1},
{:DEFAULT_COLUMN => '-456'}
]
expect(@result.count).to eq 1
expect(@result[:OPERATOR_AND]).to eq @expected_array_total
end
it 'tests operator precedence' do
@result1 = @evaluator.parse('tag:mta or name:JF and 12_4')
@result2 = @evaluator.parse('tag:mta or (name:JF and 12_4)')
expect(@result1).to eq @result2
expect(@result1.length).to eq 1
@expected_array_part_2 = [
{'name' => 'JF'},
{:DEFAULT_COLUMN => '12_4'}
]
@expected_array_total = [
{'tag' => 'mta'},
{:OPERATOR_AND => @expected_array_part_2}
]
expect(@result1[:OPERATOR_OR]).to eq @expected_array_total
end
# Tests to write :
# * query with multiple column names and search terms without logical operators
# * AND NOT, OR NOT tests
end
end

35
lib/parser-parser-part/specification.rex
View File

@@ -1,35 +0,0 @@
class Query
macro
L_BRACKET \(
R_BRACKET \)
SPACE \ + # Space char
OPERATOR_OR (?i)or
OPERATOR_AND (?i)and
OPERATOR_NOT (?i)not
TERM_WITH_QUOTES "([^"]*)"
TERM_WITHOUT_QUOTES [a-zA-Z0-9-_]+
COLON \:
rule
{SPACE} # No action
{L_BRACKET} { return [:L_BRACKET, text] }
{R_BRACKET} { return [:R_BRACKET, text] }
{OPERATOR_OR} { return [:OPERATOR_OR, text] }
{OPERATOR_AND} { return [:OPERATOR_AND, text] }
{OPERATOR_NOT} { return [:OPERATOR_NOT, text] }
{TERM_WITH_QUOTES} { return [:TERM_WITH_QUOTES, text] }
{TERM_WITHOUT_QUOTES} { return [:TERM_WITHOUT_QUOTES, text] }
{COLON} { return [:COLON, text] }
inner
def tokenize(code)
scan_setup(code)
tokens = []
while token = next_token
tokens << token
end
tokens
end
end

181
lib/parser.rb Normal file
View File

@@ -0,0 +1,181 @@
#
# DO NOT MODIFY!!!!
# This file is automatically generated by Racc 1.4.16
# from Racc grammar file "".
#
require 'racc/parser.rb'
require_relative 'lexer'
class Query < Racc::Parser
module_eval(<<'...end grammar.y/module_eval...', 'grammar.y', 24)
def parse(input)
scan_str(input)
end
...end grammar.y/module_eval...
##### State transition tables begin ###
racc_action_table = [
8, 7, 3, 4, 11, 5, 16, 3, 4, 9,
5, 3, 4, 6, 5, 3, 4, 8, 5, 8,
7, 14, 15 ]
racc_action_check = [
10, 10, 8, 8, 6, 8, 10, 0, 0, 3,
0, 5, 5, 1, 5, 7, 7, 12, 7, 2,
2, 9, 9 ]
racc_action_pointer = [
2, 13, 16, 2, nil, 6, 4, 10, -3, 16,
-3, nil, 14, nil, nil, nil, nil ]
racc_action_default = [
-2, -10, -1, -3, -4, -10, -10, -10, -10, -10,
-10, 17, -7, -8, -5, -6, -9 ]
racc_goto_table = [
2, 1, nil, nil, nil, 10, nil, 12, 13 ]
racc_goto_check = [
2, 1, nil, nil, nil, 2, nil, 2, 2 ]
racc_goto_pointer = [
nil, 1, 0 ]
racc_goto_default = [
nil, nil, nil ]
racc_reduce_table = [
0, 0, :racc_error,
1, 11, :_reduce_none,
0, 11, :_reduce_2,
1, 12, :_reduce_3,
1, 12, :_reduce_4,
3, 12, :_reduce_5,
3, 12, :_reduce_6,
3, 12, :_reduce_7,
3, 12, :_reduce_8,
3, 12, :_reduce_9 ]
racc_reduce_n = 10
racc_shift_n = 17
racc_token_table = {
false => 0,
:error => 1,
:OPERATOR_NOT => 2,
:OPERATOR_AND => 3,
:OPERATOR_OR => 4,
:TERM_WITHOUT_QUOTES => 5,
:TERM_WITH_QUOTES => 6,
:COLON => 7,
:L_BRACKET => 8,
:R_BRACKET => 9 }
racc_nt_base = 10
racc_use_result_var = true
Racc_arg = [
racc_action_table,
racc_action_check,
racc_action_default,
racc_action_pointer,
racc_goto_table,
racc_goto_check,
racc_goto_default,
racc_goto_pointer,
racc_nt_base,
racc_reduce_table,
racc_token_table,
racc_shift_n,
racc_reduce_n,
racc_use_result_var ]
Racc_token_to_s_table = [
"$end",
"error",
"OPERATOR_NOT",
"OPERATOR_AND",
"OPERATOR_OR",
"TERM_WITHOUT_QUOTES",
"TERM_WITH_QUOTES",
"COLON",
"L_BRACKET",
"R_BRACKET",
"$start",
"target",
"expression" ]
Racc_debug_parser = false
##### State transition tables end #####
# reduce 0 omitted
# reduce 1 omitted
module_eval(<<'.,.,', 'grammar.y', 8)
def _reduce_2(val, _values, result)
result = 0
result
end
.,.,
module_eval(<<'.,.,', 'grammar.y', 10)
def _reduce_3(val, _values, result)
result = {:DEFAULT_COLUMN => val[0]}
result
end
.,.,
module_eval(<<'.,.,', 'grammar.y', 11)
def _reduce_4(val, _values, result)
result = {:DEFAULT_COLUMN => val[0]}
result
end
.,.,
module_eval(<<'.,.,', 'grammar.y', 12)
def _reduce_5(val, _values, result)
result = {val[0] => val[2]}
result
end
.,.,
module_eval(<<'.,.,', 'grammar.y', 13)
def _reduce_6(val, _values, result)
result = {val[0] => val[2]}
result
end
.,.,
module_eval(<<'.,.,', 'grammar.y', 14)
def _reduce_7(val, _values, result)
result = {:OPERATOR_OR => [val[0], val[2]]}
result
end
.,.,
module_eval(<<'.,.,', 'grammar.y', 15)
def _reduce_8(val, _values, result)
result = {:OPERATOR_AND => [val[0], val[2]]}
result
end
.,.,
module_eval(<<'.,.,', 'grammar.y', 16)
def _reduce_9(val, _values, result)
result = val[1]
result
end
.,.,
def _reduce_none(val, _values, result)
val[0]
end
end # class Query

5
lib/pg_searchable_regex.rb
View File

@@ -3,7 +3,7 @@
require 'active_support'
require 'squeel'
require_relative './text_to_tsquery'
require_relative './text_to_regex_query'
require_relative './text_to_sql_query'
module PgSearchable
extend ActiveSupport::Concern
@@ -48,7 +48,8 @@ module PgSearchable
def ts_search(value)
return if @ts_search_fields.blank? || value.blank?
TextToRegexQuery.new(value, @ts_search_fields, @default_field, @ts_search_fields_mappings).where_clause( includes(@ts_joins).references(:all))
includes(@ts_joins).references(:all).where(
TextToSqlQuery.new(value, @ts_search_fields, @default_field, @ts_search_fields_mappings).where_clause)
end
def should_update_cache_field?

4
lib/specification.rex
View File

@@ -7,7 +7,7 @@ macro
OPERATOR_AND (?i)and
OPERATOR_NOT (?i)not
TERM_WITH_QUOTES "([^"]*)"
TERM_WITHOUT_QUOTES [a-zA-Z0-9-_]+
TERM_WITHOUT_QUOTES [a-zA-Z0-9\-_]+
COLON \:
@@ -32,4 +32,4 @@ inner
end
tokens
end
end
end

92
lib/text_to_sql_query.rb Normal file
View File

@@ -0,0 +1,92 @@
require_relative 'parser'
class TextToSqlQuery
def initialize(text, fields, default_field, fields_mappings = {})
@text = text.to_s.strip
@fields = fields.map(&:to_sym)
@default_field = default_field.to_sym
@fields_mappings = fields_mappings.merge(@fields.reduce({}) do |mappings, field|
_table_name, field_name = field.to_s.split('.')
mappings[field_name.to_sym] = field
mappings
end)
fields_mappings.each do |field, value|
@fields_mappings[field] = value if @fields_mappings[field]
end
end
def where_clause
@parser = Query.new
@parsed_tree = @parser.parse(@text)
generate_sql @parsed_tree
end
private
def generate_sql(tree)
first_key = tree.keys.first
node_value = tree[first_key]
case first_key
when :DEFAULT_COLUMN
escaped_node_value = handle_special_chars node_value
["#{@default_field.to_s} ILIKE ?", "%#{escaped_node_value}%"]
when :OPERATOR_OR
generate_expression_for_logical_operator(:OR, node_value)
when :OPERATOR_AND
generate_expression_for_logical_operator(:AND, node_value)
when :OPERATOR_NOT
not_array = generate_sql node_value
if not_array.length < 2
raise "There should be more than 1 element for expression following NOT operator"
end
not_expression = not_array.shift
["NOT #{not_expression}"] + not_params
else
# key is column name
escaped_node_value = handle_special_chars node_value
mapping = @fields_mappings[first_key.to_sym]
if mapping.nil?
raise "Unknown field '#{first_key.to_s}'"
else
["#{mapping.to_s} ILIKE ?", "%#{escaped_node_value}%"]
end
end
end
def generate_expression_for_logical_operator(operator, operator_array)
if operator_array.length != 2
raise "There should be two array elements for #{operator.to_s} operator"
end
first_operand = generate_sql operator_array.first
second_operand = generate_sql operator_array.last
if first_operand.length < 2
raise 'There should be more than 1 element in first operand array'
end
if second_operand.length < 2
raise 'There should be more than 1 element in second operand array'
end
first_operand_expression = first_operand.shift
first_operand_params = first_operand
second_operand_expression = second_operand.shift
second_operand_params = second_operand
["(#{first_operand_expression} #{operator.to_s} #{second_operand_expression})"] + first_operand_params + second_operand_params
end
def handle_special_chars(text)
result = text.gsub(/\"/, '')
result.gsub!(/\_/, '\_')
result.tr!('\\', '\\')
result.gsub!(/%/, '\%')
result
end
end

BIN
pkg/pg_searchable_regex-1.0.13.gem
View File

Binary file not shown.

BIN
pkg/pg_searchable_regex-1.0.21.gem Normal file
View File

Binary file not shown.

61
spec/lib/text_to_sql_query_spec.rb Normal file
View File

@@ -0,0 +1,61 @@
# frozen_string_literal: true
describe TextToRegexQuery do
include_examples 'pg_search', SimpleVectorModel
describe '.new' do
# tests simple search term without column name and without quotes
it { expect(described_class.new('some-default-value', [:"players.name"], :"players.name").where_clause(SimpleVectorModel)).to eq(['players.name ILIKE ?', '%some-default-value%']) }
# tests simple search term with column name and without quotes
it { expect(described_class.new('title:ab', [:"players.title"], :"players.title").where_clause(SimpleVectorModel)).to eq(['players.title ILIKE ?', '%ab%']) }
# tests simple search term with unknown column name and without quotes
it { expect(described_class.new('unknown:ab', [:"players.name"], :"players.name").where_clause(SimpleVectorModel)).to raise_error(RuntimeError, "Unknown field 'unknown'") }
# tests simple search term without column name and with quotes
it { expect(described_class.new('"ab"', [:"players.name", :"players.device_id"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(["players.device_id ILIKE ?", "%ab"]) }
# tests simple search term with column name and with quotes
it { expect(described_class.new('tag:"ab"', [:"players.name", :"players.tag"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(["players.tag ILIKE ?", "%ab%"]) }
# tests search without operators
it { expect(described_class.new('123 456', [:"players.name", :"players.device_id"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(["(players.device_id ILIKE ? OR players.device_id ILIKE ?)", "%123%", "%456%"]) }
# tests search with OR operator
it { expect(described_class.new('123 or 456', [:"players.name", :"players.device_id"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(["(players.device_id ILIKE ? OR players.device_id ILIKE ?)", "%123%", "%456%"]) }
# tests search with AND operator
it { expect(described_class.new('123 and 456', [:"players.name", :"players.device_id"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(["(players.device_id ILIKE ? AND players.device_id ILIKE ?)", "%123%", "%456%"]) }
# tests search with NOT operator on default column
it { expect(described_class.new('not 23', [:"players.name", :"players.device_id"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(["NOT players.device_id ILIKE ?", "%23%"]) }
# tests search with NOT operator on non-default column
it { expect(described_class.new('not tag:23', [:"players.name", :"players.tag"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(["NOT players.tag ILIKE ?", "%23%"]) }
# tests search with mixed logical operators
it { expect(described_class.new('title:ab and not tag:hf-1', [:"players.title", :"players.tag"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(['(players.title ILIKE ? AND NOT players.tag ILIKE ?)', '%ab%', '%hf-1%']) }
# tests search with mixed logical operators without NOT'
it { expect(described_class.new('title:a and title:b or title:c', [:"players.title", :"players.tag"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(['((players.title ILIKE ? AND players.title ILIKE ?) OR players.title ILIKE ?)', '%a%', '%b%', '%c%']) }
# tests search with brackets in expression
it { expect(described_class.new('title:a and (title:b or title:c)', [:"players.title", :"players.tag"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(['(players.title ILIKE ? AND (players.title ILIKE ? OR players.title ILIKE ?))', '%a%', '%b%', '%c%']) }
# tests search with brackets in expression and with NOT operator
it { expect(described_class.new('title:a and not (title:b or title:c)', [:"players.title", :"players.tag"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(['(players.title ILIKE ? AND NOT (players.title ILIKE ? OR players.title ILIKE ?))', '%a%', '%b%', '%c%']) }
# tests search with special characters in search term
it { expect(described_class.new('title:"%a_\"', [:"players.title", :"players.tag"], :"players.device_id").where_clause(SimpleVectorModel)).to eq(['players.title ILIKE ?', '%\%a\_\\%']) }
# tests search with field mappings
it { expect(described_class.new('tags:h1-r', [:'players.title', :'players.name', :'players.device_id'], :"players.device_id", { tags: "tags.name" }).where_clause(SimpleVectorModel)).to eq(['tags.name ILIKE ?', '%h1-r%']) }
# tests search with field mappings when fields array has same mapping
it { expect(described_class.new('tags:hs1-r', [:'players.title', :'players.tags', :'players.device_id'], :"players.device_id", { tags: "tags.name" }).where_clause(SimpleVectorModel)).to eq(["players.tag ILIKE ?", "%ab%"]) }
# tests complex query
it { expect(described_class.new('(device_id:"with space" tags:mta no-quotes-id-123) or "id with quotes-5" and ( ("id with q 10" or "id with q 20") and ("id with Q 30" "id with Q 40") and not id-without-Q-50)', [:'players.title', :'players.name', :'players.device_id'], :"players.device_id", { tags: 'tags.name' }).where_clause(SimpleVectorModel)).to eq(['((players.device_id ILIKE ? OR (tags.name ILIKE ? OR players.device_id ILIKE ?)) OR (players.device_id ILIKE ? AND (((players.device_id ILIKE ? OR players.device_id ILIKE ?) AND (players.device_id ILIKE ? OR players.device_id ILIKE ?)) AND NOT players.device_id ILIKE ?)))', '%with space%', '%mta%', '%no-quotes-id-123%', '%id with quotes-5%', '%id with q 10%', '%id with q 20%', '%id with Q 30%', '%id with Q 40%', '%id-without-Q-50%']) }
end
end

2
version.rb
View File

@@ -3,7 +3,7 @@
module Version
MAJOR = 1
MINOR = 0
PATCH = 13
PATCH = 21
def self.to_s
[MAJOR, MINOR, PATCH].compact.join('.')