senaduka/old-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'parser-part' into 'master'

Browse Source 
Parser part

See merge request saburly/reklamice/parser!2
This commit was merged in pull request #2.
This commit is contained in:
Bilal Catic
2020-01-27 07:18:02 +00:00
parent 323e815d5f 6459d04cec
commit cf26304e9e
6 changed files with 436 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -1,3 +1,4 @@
.idea
lexer.rb
lexer.rb
parser.rb

10
README.md
View File

@@ -1,15 +1,19 @@
# parser
###Prerequisites
### Prerequisites
* Rexical (rex)
* Racc
###Available commands
### Available commands
* `rake lexer` - generates `lexer.rb` file based on `specification.rex` file
* `rake parser` - generates `parser.rb` file based on `grammar.y` file
* `rake generate` - generates `lexer.rb` and `parser.rb` files
###Testing
### Testing
To run only `lexer` tests, execute : `rspec spec/query_lexer_spec.rb`
To run only `parser` tests, execute : `rspec spec/query_parser_spec.rb`
To run all tests, execute : `rake spec`

14
Rakefile
View File

@@ -2,11 +2,19 @@ require 'rspec/core/rake_task'
RSpec::Core::RakeTask.new do |c|
options = ['--color']
options += ["--format", "documentation"]
options += %w[--format documentation]
c.rspec_opts = options
end
desc "Generate Lexer"
desc 'Generate Lexer'
task :lexer do
`rex specification.rex -o lexer.rb`
end
end
desc 'Generate Parser'
task :parser do
`racc grammar.y -o parser.rb`
end
desc 'Generate Lexer and Parser'
task generate: %i[lexer parser]

28
grammar.y Normal file
View File

@@ -0,0 +1,28 @@
class Query
prechigh
left OPERATOR_NOT
left OPERATOR_AND
left OPERATOR_OR
preclow
rule
target: expression
| /* none */ { result = 0 }
expression: TERM_WITHOUT_QUOTES { result = {:DEFAULT_COLUMN => val[0]} }
| TERM_WITH_QUOTES { result = {:DEFAULT_COLUMN => val[0]} }
| TERM_WITHOUT_QUOTES COLON TERM_WITHOUT_QUOTES { result = {val[0] => val[2]} }
| TERM_WITHOUT_QUOTES COLON TERM_WITH_QUOTES { result = {val[0] => val[2]} }
| expression OPERATOR_OR expression { result = {:OPERATOR_OR => [val[0], val[2]]} }
| expression OPERATOR_AND expression { result = {:OPERATOR_AND => [val[0], val[2]]} }
| OPERATOR_NOT expression { result = {:OPERATOR_NOT => val[1]} }
| L_BRACKET expression R_BRACKET { result = val[1] }
| expression expression { result = {:OPERATOR_OR => [val[0], val[1]]} }
end
---- header
require_relative 'lexer'
---- inner
def parse(input)
scan_str(input)
end

18
spec/query_lexer_spec.rb
View File

@@ -117,6 +117,24 @@ class QueryLexerTester
expect(@result[2][1]).to eq 'JF'
end
it 'tests simple query with two columns with name and search terms without quotes' do
@result = @evaluator.tokenize('name:JF tag:mta')
expect(@result.length).to eq 6
expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[0][1]).to eq 'name'
expect(@result[1][0]).to eq :COLON
expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[2][1]).to eq 'JF'
expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[3][1]).to eq 'tag'
expect(@result[4][0]).to eq :COLON
expect(@result[5][0]).to eq :TERM_WITHOUT_QUOTES
expect(@result[5][1]).to eq 'mta'
end
it 'tests simple query with column name and search term with quotes' do
@result = @evaluator.tokenize('name:"name with space"')

370
spec/query_parser_spec.rb Normal file
View File

@@ -0,0 +1,370 @@
require './parser'
class QueryParserTester
describe 'Testing the Parser' do
before do
@evaluator = Query.new
end
it 'tests query with only one search term without quotes and without column name' do
@result = @evaluator.parse('-123')
expect(@result[:DEFAULT_COLUMN]).to eq '-123'
end
it 'tests query with only one search term with quotes and without column name' do
@result = @evaluator.parse('"OR 128"')
expect(@result[:DEFAULT_COLUMN]).to eq '"OR 128"'
end
it 'tests query with one column and search term without quotes' do
@result = @evaluator.parse('tag:mta')
expect(@result['tag']).to eq 'mta'
end
it 'tests query with one column and search term with quotes' do
@result = @evaluator.parse('tag:"tag 120"')
expect(@result['tag']).to eq '"tag 120"'
end
it 'tests query with two columns connected with OR and search terms without quotes' do
@result = @evaluator.parse('tag:mta OR tag:12')
@expected_array = [
{ 'tag' => 'mta' },
{ 'tag' => '12' }
]
expect(@result.count).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_array
end
it 'tests query with two columns connected with OR and search terms with quotes' do
@result = @evaluator.parse('tag:mta OR tag:"tag 12"')
@expected_array = [
{ 'tag' => 'mta' },
{ 'tag' => '"tag 12"' }
]
expect(@result.count).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_array
end
it 'tests query with two columns connected with AND and search terms without quotes' do
@result = @evaluator.parse('tag:mta AND tag:12')
@expected_array = [
{ 'tag' => 'mta' },
{ 'tag' => '12' }
]
expect(@result.count).to eq 1
expect(@result[:OPERATOR_AND]).to eq @expected_array
end
it 'tests query with two columns connected with AND and search terms with quotes' do
@result = @evaluator.parse('tag:mta and tag:"tag 12"')
@expected_array = [
{ 'tag' => 'mta' },
{ 'tag' => '"tag 12"' }
]
expect(@result.count).to eq 1
expect(@result[:OPERATOR_AND]).to eq @expected_array
end
it 'tests simple query with brackets' do
@result = @evaluator.parse('(123)')
expect(@result.count).to eq 1
expect(@result[:DEFAULT_COLUMN]).to eq '123'
end
it 'tests simple query with brackets and with a column name' do
@result = @evaluator.parse('(name:JF)')
expect(@result.count).to eq 1
expect(@result['name']).to eq 'JF'
end
it 'tests query with OR operator in brackets' do
@result = @evaluator.parse('(name:JF or tag:mta)')
@expected_array = [
{ 'name' => 'JF' },
{ 'tag' => 'mta' }
]
expect(@result.count).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_array
end
it 'tests query with two simple brackets expressions' do
@result = @evaluator.parse('(name:JF) and (-456)')
@expected_array = [
{ 'name' => 'JF' },
{ :DEFAULT_COLUMN => '-456' }
]
expect(@result.count).to eq 1
expect(@result[:OPERATOR_AND]).to eq @expected_array
end
it 'tests query with two brackets expressions' do
@result = @evaluator.parse('(name:JF or tag:"tag 0") and (-456)')
@expected_array_part_1 = [
{ 'name' => 'JF' },
{ 'tag' => '"tag 0"' }
]
@expected_array_total = [
{:OPERATOR_OR => @expected_array_part_1},
{:DEFAULT_COLUMN => '-456'}
]
expect(@result.count).to eq 1
expect(@result[:OPERATOR_AND]).to eq @expected_array_total
end
it 'tests operator precedence, or - and' do
@result1 = @evaluator.parse('tag:mta or name:JF and 12_4')
@result2 = @evaluator.parse('tag:mta or (name:JF and 12_4)')
expect(@result1).to eq @result2
expect(@result1.length).to eq 1
@expected_array_part_2 = [
{'name' => 'JF'},
{:DEFAULT_COLUMN => '12_4'}
]
@expected_array_total = [
{'tag' => 'mta'},
{:OPERATOR_AND => @expected_array_part_2}
]
expect(@result1[:OPERATOR_OR]).to eq @expected_array_total
end
it 'tests operator precedence, and - or' do
@result1 = @evaluator.parse('tag:mta and name:JF or 12_4')
@result2 = @evaluator.parse('(tag:mta and name:JF) or 12_4')
expect(@result1).to eq @result2
expect(@result1.length).to eq 1
@expected_array_part_2 = [
{'tag' => 'mta'},
{'name' => 'JF'}
]
@expected_array_total = [
{:OPERATOR_AND => @expected_array_part_2},
{:DEFAULT_COLUMN => '12_4'}
]
expect(@result1[:OPERATOR_OR]).to eq @expected_array_total
end
it 'tests simple query with not operator' do
@result = @evaluator.parse('not -54')
expect(@result.length).to be 1
expect(@result[:OPERATOR_NOT][:DEFAULT_COLUMN]).to eq '-54'
end
it 'tests query with mixed NOT and AND logic operator' do
@result = @evaluator.parse('name:"some wild name" and not -123-456')
@expected_and_array = [
{'name' => '"some wild name"'},
{:OPERATOR_NOT => {:DEFAULT_COLUMN => '-123-456'}}
]
expect(@result.length).to eq 1
expect(@result[:OPERATOR_AND]).to eq @expected_and_array
end
it 'tests query with mixed NOT, AND and OR logic operators' do
@result = @evaluator.parse('name:"some wild name" or not (tag:mta and -123-456)')
@expected_and_array_operands = [
{'tag' => 'mta'},
{:DEFAULT_COLUMN => '-123-456'}
]
@expected_or_array_operands = [
{'name' => '"some wild name"'},
{:OPERATOR_NOT => {:OPERATOR_AND => @expected_and_array_operands}}
]
expect(@result.length).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_or_array_operands
end
it 'tests query with two default column search terms without quotes and without logical operators' do
@result = @evaluator.parse('id-5 -456')
@expected_array = [
{:DEFAULT_COLUMN => 'id-5'},
{:DEFAULT_COLUMN => '-456'}
]
expect(@result.length).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_array
end
it 'tests query with two default column search terms with quotes and without logical operators' do
@result = @evaluator.parse('"id-5 no Q" -456')
@expected_array = [
{:DEFAULT_COLUMN => '"id-5 no Q"'},
{:DEFAULT_COLUMN => '-456'}
]
expect(@result.length).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_array
end
it 'tests query with two default column search terms with quotes on both terms and without logical operators' do
@result = @evaluator.parse('"id-5 no Q" "wild id -456"')
@expected_array = [
{:DEFAULT_COLUMN => '"id-5 no Q"'},
{:DEFAULT_COLUMN => '"wild id -456"'}
]
expect(@result.length).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_array
end
it 'tests query with one default column and one non defualt column, search terms without quotes and without logical operators' do
@result = @evaluator.parse('tag:mta -456')
@expected_array = [
{'tag' => 'mta'},
{:DEFAULT_COLUMN => '-456'}
]
expect(@result.length).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_array
end
it 'tests query with one default column and one non defualt column, search terms with quotes and without logical operators' do
@result = @evaluator.parse('-1-23 name:"WILD name"')
@expected_array = [
{:DEFAULT_COLUMN => '-1-23'},
{'name' => '"WILD name"'}
]
expect(@result.length).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_array
end
it 'tests query with two columns, search terms with quotes and without logical operators' do
@result = @evaluator.parse('tag:-1-23 name:"WILD name"')
@expected_array = [
{'tag' => '-1-23'},
{'name' => '"WILD name"'}
]
expect(@result.length).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_array
end
it 'tests query with two columns, search terms with quotes on both and without logical operators' do
@result = @evaluator.parse('tag:"1 2 3" name:"WILD name"')
@expected_array = [
{'tag' => '"1 2 3"'},
{'name' => '"WILD name"'}
]
expect(@result.length).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_array
end
it 'tests query with three columns, search terms with quotes on both and without logical operators' do
@result = @evaluator.parse('tag:"1 2 3" name:"WILD name" name:"ANOTHER wild name"')
@expected_array = [
{'tag' => '"1 2 3"'},
{:OPERATOR_OR => [
{'name' => '"WILD name"'},
{'name' => '"ANOTHER wild name"'}
]}
]
expect(@result.length).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_array
end
it 'tests complex query' do
@result = @evaluator.parse('(device-id:"with space" tag:mta no-quotes-id-123)'\
'or "id with quotes-5" and ( ("id with q 10" or "id with q 20")'\
'and ("id with Q 30" "id with Q 40") and not id-without-Q-50)')
# (device-id:"with space" tag:mta no-quotes-id-123) or "id with quotes-5" and ( ("id with q 10" or "id with q 20") and ("id with Q 30" "id with Q 40") and not id-without-Q-50)
# _____________________A___________________________ or _______B___________ and __________________________________________________C______________________________________________
# (____A1________________ ___A2__ ______A3________) or _______B___________ and ( ____________C1____________________ and ______________C2_______________ and ______C3___________)
@A = {:OPERATOR_OR => [
{'device-id' => '"with space"'},
{:OPERATOR_OR => [
{'tag' => 'mta'},
{:DEFAULT_COLUMN => 'no-quotes-id-123'}]
}
]}
@B = {:DEFAULT_COLUMN => '"id with quotes-5"'}
@C1 = {:OPERATOR_OR => [
{:DEFAULT_COLUMN => '"id with q 10"'},
{:DEFAULT_COLUMN => '"id with q 20"'}
]}
@C2 = {:OPERATOR_OR => [
{:DEFAULT_COLUMN => '"id with Q 30"'},
{:DEFAULT_COLUMN => '"id with Q 40"'}
]}
@C3 = {:OPERATOR_NOT => {:DEFAULT_COLUMN => 'id-without-Q-50'}}
@C = {:OPERATOR_AND => [
{:OPERATOR_AND => [
@C1,
@C2
]},
@C3
]}
@expected_final_array_result = [
@A,
{:OPERATOR_AND => [
@B,
@C
]}
]
expect(@result.length).to eq 1
expect(@result[:OPERATOR_OR]).to eq @expected_final_array_result
end
end
end