From cb93670a58c7e76f6d28782dbcd444d59236c2fa Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Thu, 23 Jan 2020 19:16:16 +0100 Subject: [PATCH 1/4] setup files for parser part --- .gitignore | 3 ++- README.md | 10 +++++++--- Rakefile | 14 +++++++++++--- grammar.y | 9 +++++++++ spec/query_parser_spec.rb | 9 +++++++++ 5 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 grammar.y create mode 100644 spec/query_parser_spec.rb diff --git a/.gitignore b/.gitignore index 837c184..c8d090a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea -lexer.rb \ No newline at end of file +lexer.rb +parser.rb \ No newline at end of file diff --git a/README.md b/README.md index 3123cb2..2eae07a 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,19 @@ # parser -###Prerequisites +### Prerequisites * Rexical (rex) +* Racc -###Available commands +### Available commands * `rake lexer` - generates `lexer.rb` file based on `specification.rex` file +* `rake parser` - generates `parser.rb` file based on `grammar.y` file +* `rake generate` - generates `lexer.rb` and `parser.rb` files -###Testing +### Testing To run only `lexer` tests, execute : `rspec spec/query_lexer_spec.rb` +To run only `parser` tests, execute : `rspec spec/query_parser_spec.rb` To run all tests, execute : `rake spec` \ No newline at end of file diff --git a/Rakefile b/Rakefile index b63d98d..92064f7 100644 --- a/Rakefile +++ b/Rakefile @@ -2,11 +2,19 @@ require 'rspec/core/rake_task' RSpec::Core::RakeTask.new do |c| options = ['--color'] - options += ["--format", "documentation"] + options += %w[--format documentation] c.rspec_opts = options end -desc "Generate Lexer" +desc 'Generate Lexer' task :lexer do `rex specification.rex -o lexer.rb` -end \ No newline at end of file +end + +desc 'Generate Parser' +task :parser do + `racc grammar.y -o parser.rb` +end + +desc 'Generate Lexer and Parser' +task generate: %i[lexer parser] \ No newline at end of file diff --git a/grammar.y b/grammar.y new file mode 100644 index 0000000..72d6f25 --- /dev/null +++ b/grammar.y @@ -0,0 +1,9 @@ +class Query + rule + expression: +end + +---- inner + def parse(input) + scan_str(input) + end \ No newline at end of file diff --git a/spec/query_parser_spec.rb b/spec/query_parser_spec.rb new file mode 100644 index 0000000..a314de2 --- /dev/null +++ b/spec/query_parser_spec.rb @@ -0,0 +1,9 @@ +require './parser' + +class QueryParserTester + describe 'Testing the Parser' do + before do + @evaluator = Query.new + end + end +end -- 2.47.3 From 2bf8f95896a53f4b9fbec0b4d52140b7996ba6b8 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Thu, 23 Jan 2020 23:00:16 +0100 Subject: [PATCH 2/4] add simple grammar rules and tests for simple cases --- grammar.y | 11 ++++++++++- spec/query_parser_spec.rb | 26 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/grammar.y b/grammar.y index 72d6f25..7c8657c 100644 --- a/grammar.y +++ b/grammar.y @@ -1,8 +1,17 @@ class Query rule - expression: + target: expression + | /* none */ { result = 0 } + + expression: TERM_WITHOUT_QUOTES { result = {:DEFAULT_COLUMN => val[0]} } + | TERM_WITH_QUOTES { result = {:DEFAULT_COLUMN => val[0]} } + | TERM_WITHOUT_QUOTES COLON TERM_WITHOUT_QUOTES { result = {val[0] => val[2]} } + | TERM_WITHOUT_QUOTES COLON TERM_WITH_QUOTES { result = {val[0] => val[2]} } end +---- header + require_relative 'lexer' + ---- inner def parse(input) scan_str(input) diff --git a/spec/query_parser_spec.rb b/spec/query_parser_spec.rb index a314de2..8e2de5b 100644 --- a/spec/query_parser_spec.rb +++ b/spec/query_parser_spec.rb @@ -5,5 +5,31 @@ class QueryParserTester before do @evaluator = Query.new end + + it 'tests query with only one search term without quotes and without column name' do + @result = @evaluator.parse('-123') + + expect(@result[:DEFAULT_COLUMN]).to eq '-123' + end + + it 'tests query with only one search term with quotes and without column name' do + @result = @evaluator.parse('"OR 128"') + + expect(@result[:DEFAULT_COLUMN]).to eq '"OR 128"' + end + + it 'tests query with one column and search term without quotes' do + @result = @evaluator.parse('tag:mta') + + expect(@result['tag']).to eq 'mta' + end + + it 'tests query with one column and search term with quotes' do + @result = @evaluator.parse('tag:"tag 120"') + + expect(@result['tag']).to eq '"tag 120"' + end + + end end -- 2.47.3 From 7d2e9b289fae25e9350bc43f8082cf2cc9689ba4 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Fri, 24 Jan 2020 14:43:46 +0100 Subject: [PATCH 3/4] add more rules tu parser grammar and write more tests for new rules --- grammar.y | 8 +++ spec/query_lexer_spec.rb | 18 ++++++ spec/query_parser_spec.rb | 129 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+) diff --git a/grammar.y b/grammar.y index 7c8657c..5484994 100644 --- a/grammar.y +++ b/grammar.y @@ -1,4 +1,9 @@ class Query + prechigh + left OPERATOR_NOT + left OPERATOR_AND + left OPERATOR_OR + preclow rule target: expression | /* none */ { result = 0 } @@ -7,6 +12,9 @@ class Query | TERM_WITH_QUOTES { result = {:DEFAULT_COLUMN => val[0]} } | TERM_WITHOUT_QUOTES COLON TERM_WITHOUT_QUOTES { result = {val[0] => val[2]} } | TERM_WITHOUT_QUOTES COLON TERM_WITH_QUOTES { result = {val[0] => val[2]} } + | expression OPERATOR_OR expression { result = {:OPERATOR_OR => [val[0], val[2]]} } + | expression OPERATOR_AND expression { result = {:OPERATOR_AND => [val[0], val[2]]} } + | L_BRACKET expression R_BRACKET { result = val[1] } end ---- header diff --git a/spec/query_lexer_spec.rb b/spec/query_lexer_spec.rb index 9fd4e77..b97a65a 100644 --- a/spec/query_lexer_spec.rb +++ b/spec/query_lexer_spec.rb @@ -117,6 +117,24 @@ class QueryLexerTester expect(@result[2][1]).to eq 'JF' end + it 'tests simple query with two columns with name and search terms without quotes' do + @result = @evaluator.tokenize('name:JF tag:mta') + + expect(@result.length).to eq 6 + + expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[0][1]).to eq 'name' + expect(@result[1][0]).to eq :COLON + expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[2][1]).to eq 'JF' + expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[3][1]).to eq 'tag' + expect(@result[4][0]).to eq :COLON + expect(@result[5][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[5][1]).to eq 'mta' + + end + it 'tests simple query with column name and search term with quotes' do @result = @evaluator.tokenize('name:"name with space"') diff --git a/spec/query_parser_spec.rb b/spec/query_parser_spec.rb index 8e2de5b..f26c4a3 100644 --- a/spec/query_parser_spec.rb +++ b/spec/query_parser_spec.rb @@ -30,6 +30,135 @@ class QueryParserTester expect(@result['tag']).to eq '"tag 120"' end + it 'tests query with two columns connected with OR and search terms without quotes' do + @result = @evaluator.parse('tag:mta OR tag:12') + + @expected_array = [ + { 'tag' => 'mta' }, + { 'tag' => '12' } + ] + + expect(@result.count).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with two columns connected with OR and search terms with quotes' do + @result = @evaluator.parse('tag:mta OR tag:"tag 12"') + + @expected_array = [ + { 'tag' => 'mta' }, + { 'tag' => '"tag 12"' } + ] + + expect(@result.count).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with two columns connected with AND and search terms without quotes' do + @result = @evaluator.parse('tag:mta AND tag:12') + + @expected_array = [ + { 'tag' => 'mta' }, + { 'tag' => '12' } + ] + + expect(@result.count).to eq 1 + expect(@result[:OPERATOR_AND]).to eq @expected_array + end + + it 'tests query with two columns connected with AND and search terms with quotes' do + @result = @evaluator.parse('tag:mta and tag:"tag 12"') + + @expected_array = [ + { 'tag' => 'mta' }, + { 'tag' => '"tag 12"' } + ] + + expect(@result.count).to eq 1 + expect(@result[:OPERATOR_AND]).to eq @expected_array + end + + it 'tests simple query with brackets' do + @result = @evaluator.parse('(123)') + + expect(@result.count).to eq 1 + expect(@result[:DEFAULT_COLUMN]).to eq '123' + end + + it 'tests simple query with brackets and with a column name' do + @result = @evaluator.parse('(name:JF)') + + expect(@result.count).to eq 1 + expect(@result['name']).to eq 'JF' + end + + it 'tests query with OR operator in brackets' do + @result = @evaluator.parse('(name:JF or tag:mta)') + + @expected_array = [ + { 'name' => 'JF' }, + { 'tag' => 'mta' } + ] + + expect(@result.count).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with two simple brackets expressions' do + @result = @evaluator.parse('(name:JF) and (-456)') + + @expected_array = [ + { 'name' => 'JF' }, + { :DEFAULT_COLUMN => '-456' } + ] + + expect(@result.count).to eq 1 + expect(@result[:OPERATOR_AND]).to eq @expected_array + end + + it 'tests query with two brackets expressions' do + @result = @evaluator.parse('(name:JF or tag:"tag 0") and (-456)') + + @expected_array_part_1 = [ + { 'name' => 'JF' }, + { 'tag' => '"tag 0"' } + ] + + @expected_array_total = [ + {:OPERATOR_OR => @expected_array_part_1}, + {:DEFAULT_COLUMN => '-456'} + ] + + expect(@result.count).to eq 1 + expect(@result[:OPERATOR_AND]).to eq @expected_array_total + end + + it 'tests operator precedence' do + @result1 = @evaluator.parse('tag:mta or name:JF and 12_4') + @result2 = @evaluator.parse('tag:mta or (name:JF and 12_4)') + + expect(@result1).to eq @result2 + + expect(@result1.length).to eq 1 + + @expected_array_part_2 = [ + {'name' => 'JF'}, + {:DEFAULT_COLUMN => '12_4'} + ] + + @expected_array_total = [ + {'tag' => 'mta'}, + {:OPERATOR_AND => @expected_array_part_2} + ] + + expect(@result1[:OPERATOR_OR]).to eq @expected_array_total + + end + + # Tests to write : + # * query with multiple column names and search terms without logical operators + # * AND NOT, OR NOT tests + end end -- 2.47.3 From 6459d04cec1ea4e71999d36a4cd74b99d5562209 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Mon, 27 Jan 2020 08:17:33 +0100 Subject: [PATCH 4/4] add more rules to grammar and write more tests to cover new rules --- grammar.y | 2 + spec/query_parser_spec.rb | 214 +++++++++++++++++++++++++++++++++++++- 2 files changed, 212 insertions(+), 4 deletions(-) diff --git a/grammar.y b/grammar.y index 5484994..2eed955 100644 --- a/grammar.y +++ b/grammar.y @@ -14,7 +14,9 @@ class Query | TERM_WITHOUT_QUOTES COLON TERM_WITH_QUOTES { result = {val[0] => val[2]} } | expression OPERATOR_OR expression { result = {:OPERATOR_OR => [val[0], val[2]]} } | expression OPERATOR_AND expression { result = {:OPERATOR_AND => [val[0], val[2]]} } + | OPERATOR_NOT expression { result = {:OPERATOR_NOT => val[1]} } | L_BRACKET expression R_BRACKET { result = val[1] } + | expression expression { result = {:OPERATOR_OR => [val[0], val[1]]} } end ---- header diff --git a/spec/query_parser_spec.rb b/spec/query_parser_spec.rb index f26c4a3..58e3da7 100644 --- a/spec/query_parser_spec.rb +++ b/spec/query_parser_spec.rb @@ -133,7 +133,7 @@ class QueryParserTester expect(@result[:OPERATOR_AND]).to eq @expected_array_total end - it 'tests operator precedence' do + it 'tests operator precedence, or - and' do @result1 = @evaluator.parse('tag:mta or name:JF and 12_4') @result2 = @evaluator.parse('tag:mta or (name:JF and 12_4)') @@ -155,10 +155,216 @@ class QueryParserTester end - # Tests to write : - # * query with multiple column names and search terms without logical operators - # * AND NOT, OR NOT tests + it 'tests operator precedence, and - or' do + @result1 = @evaluator.parse('tag:mta and name:JF or 12_4') + @result2 = @evaluator.parse('(tag:mta and name:JF) or 12_4') + expect(@result1).to eq @result2 + + expect(@result1.length).to eq 1 + + @expected_array_part_2 = [ + {'tag' => 'mta'}, + {'name' => 'JF'} + ] + + @expected_array_total = [ + {:OPERATOR_AND => @expected_array_part_2}, + {:DEFAULT_COLUMN => '12_4'} + ] + + expect(@result1[:OPERATOR_OR]).to eq @expected_array_total + + end + + it 'tests simple query with not operator' do + @result = @evaluator.parse('not -54') + + expect(@result.length).to be 1 + expect(@result[:OPERATOR_NOT][:DEFAULT_COLUMN]).to eq '-54' + end + + it 'tests query with mixed NOT and AND logic operator' do + @result = @evaluator.parse('name:"some wild name" and not -123-456') + + @expected_and_array = [ + {'name' => '"some wild name"'}, + {:OPERATOR_NOT => {:DEFAULT_COLUMN => '-123-456'}} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_AND]).to eq @expected_and_array + end + + it 'tests query with mixed NOT, AND and OR logic operators' do + @result = @evaluator.parse('name:"some wild name" or not (tag:mta and -123-456)') + + @expected_and_array_operands = [ + {'tag' => 'mta'}, + {:DEFAULT_COLUMN => '-123-456'} + ] + + @expected_or_array_operands = [ + {'name' => '"some wild name"'}, + {:OPERATOR_NOT => {:OPERATOR_AND => @expected_and_array_operands}} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_or_array_operands + end + + it 'tests query with two default column search terms without quotes and without logical operators' do + @result = @evaluator.parse('id-5 -456') + + @expected_array = [ + {:DEFAULT_COLUMN => 'id-5'}, + {:DEFAULT_COLUMN => '-456'} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with two default column search terms with quotes and without logical operators' do + @result = @evaluator.parse('"id-5 no Q" -456') + + @expected_array = [ + {:DEFAULT_COLUMN => '"id-5 no Q"'}, + {:DEFAULT_COLUMN => '-456'} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with two default column search terms with quotes on both terms and without logical operators' do + @result = @evaluator.parse('"id-5 no Q" "wild id -456"') + + @expected_array = [ + {:DEFAULT_COLUMN => '"id-5 no Q"'}, + {:DEFAULT_COLUMN => '"wild id -456"'} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with one default column and one non defualt column, search terms without quotes and without logical operators' do + @result = @evaluator.parse('tag:mta -456') + + @expected_array = [ + {'tag' => 'mta'}, + {:DEFAULT_COLUMN => '-456'} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with one default column and one non defualt column, search terms with quotes and without logical operators' do + @result = @evaluator.parse('-1-23 name:"WILD name"') + + @expected_array = [ + {:DEFAULT_COLUMN => '-1-23'}, + {'name' => '"WILD name"'} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with two columns, search terms with quotes and without logical operators' do + @result = @evaluator.parse('tag:-1-23 name:"WILD name"') + + @expected_array = [ + {'tag' => '-1-23'}, + {'name' => '"WILD name"'} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with two columns, search terms with quotes on both and without logical operators' do + @result = @evaluator.parse('tag:"1 2 3" name:"WILD name"') + + @expected_array = [ + {'tag' => '"1 2 3"'}, + {'name' => '"WILD name"'} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with three columns, search terms with quotes on both and without logical operators' do + @result = @evaluator.parse('tag:"1 2 3" name:"WILD name" name:"ANOTHER wild name"') + + @expected_array = [ + {'tag' => '"1 2 3"'}, + {:OPERATOR_OR => [ + {'name' => '"WILD name"'}, + {'name' => '"ANOTHER wild name"'} + ]} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests complex query' do + @result = @evaluator.parse('(device-id:"with space" tag:mta no-quotes-id-123)'\ + 'or "id with quotes-5" and ( ("id with q 10" or "id with q 20")'\ + 'and ("id with Q 30" "id with Q 40") and not id-without-Q-50)') + + # (device-id:"with space" tag:mta no-quotes-id-123) or "id with quotes-5" and ( ("id with q 10" or "id with q 20") and ("id with Q 30" "id with Q 40") and not id-without-Q-50) + # _____________________A___________________________ or _______B___________ and __________________________________________________C______________________________________________ + # (____A1________________ ___A2__ ______A3________) or _______B___________ and ( ____________C1____________________ and ______________C2_______________ and ______C3___________) + + + @A = {:OPERATOR_OR => [ + {'device-id' => '"with space"'}, + {:OPERATOR_OR => [ + {'tag' => 'mta'}, + {:DEFAULT_COLUMN => 'no-quotes-id-123'}] + } + ]} + + @B = {:DEFAULT_COLUMN => '"id with quotes-5"'} + + + @C1 = {:OPERATOR_OR => [ + {:DEFAULT_COLUMN => '"id with q 10"'}, + {:DEFAULT_COLUMN => '"id with q 20"'} + ]} + + + @C2 = {:OPERATOR_OR => [ + {:DEFAULT_COLUMN => '"id with Q 30"'}, + {:DEFAULT_COLUMN => '"id with Q 40"'} + ]} + + @C3 = {:OPERATOR_NOT => {:DEFAULT_COLUMN => 'id-without-Q-50'}} + + @C = {:OPERATOR_AND => [ + {:OPERATOR_AND => [ + @C1, + @C2 + ]}, + @C3 + ]} + + @expected_final_array_result = [ + @A, + {:OPERATOR_AND => [ + @B, + @C + ]} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_final_array_result + end end end -- 2.47.3