From e5925c8209c5712f5bfadb267f7b4f2aae196c4a Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Wed, 5 Feb 2020 23:44:54 +0100 Subject: [PATCH 01/16] add model param to query generator --- lib/text_to_sql_query.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/text_to_sql_query.rb b/lib/text_to_sql_query.rb index d473f48..ab3320e 100644 --- a/lib/text_to_sql_query.rb +++ b/lib/text_to_sql_query.rb @@ -15,10 +15,11 @@ class TextToSqlQuery end end - def where_clause + def where_clause(model) @parser = Query.new @parsed_tree = @parser.parse(@text) - generate_sql @parsed_tree + generated_sql = generate_sql @parsed_tree + model.where generated_sql end private -- 2.47.3 From b9bfa004c6382f68bbdbbf4b941fbdd0d141a6d1 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Fri, 7 Feb 2020 11:40:03 +0100 Subject: [PATCH 02/16] fix rule for expression expression; fix tests --- lib/text_to_sql_query.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/text_to_sql_query.rb b/lib/text_to_sql_query.rb index ab3320e..d473f48 100644 --- a/lib/text_to_sql_query.rb +++ b/lib/text_to_sql_query.rb @@ -15,11 +15,10 @@ class TextToSqlQuery end end - def where_clause(model) + def where_clause @parser = Query.new @parsed_tree = @parser.parse(@text) - generated_sql = generate_sql @parsed_tree - model.where generated_sql + generate_sql @parsed_tree end private -- 2.47.3 From 66c01702c88915bbc5221524bab61170e92daebe Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Mon, 10 Feb 2020 14:32:37 +0100 Subject: [PATCH 03/16] fix rex specification and add tests for keyword detection --- lib/specification.rex | 6 +- spec/lib/query_lexer_spec.rb | 273 ++++++++++++++++++++ spec/lib/query_parser_spec.rb | 467 +++++++++++++++++++++++++++++++++- 3 files changed, 739 insertions(+), 7 deletions(-) diff --git a/lib/specification.rex b/lib/specification.rex index c92a37e..2fc8eaa 100644 --- a/lib/specification.rex +++ b/lib/specification.rex @@ -3,9 +3,9 @@ macro L_BRACKET \( R_BRACKET \) SPACE \ + # Space char - OPERATOR_OR (?i)or - OPERATOR_AND (?i)and - OPERATOR_NOT (?i)not + OPERATOR_OR (?i)\bor\b + OPERATOR_AND (?i)\band\b + OPERATOR_NOT (?i)\bnot\b TERM_WITH_QUOTES "([^"]*)" TERM_WITHOUT_QUOTES [a-zA-Z0-9\-_]+ COLON \: diff --git a/spec/lib/query_lexer_spec.rb b/spec/lib/query_lexer_spec.rb index f531899..ff68dfd 100644 --- a/spec/lib/query_lexer_spec.rb +++ b/spec/lib/query_lexer_spec.rb @@ -255,5 +255,278 @@ class QueryLexerTester expect(@result[2][0]).to eq :TERM_WITH_QUOTES expect(@result[2][1]).to eq '"tag with or and not inside"' end + + it 'tests query without column name containing AND word inside without quotes' do + @result = @evaluator.tokenize('land') + + expect(@result.length).to eq 1 + + expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[0][1]).to eq 'land' + end + + it 'tests query without column name containing AND word inside with quotes' do + @result = @evaluator.tokenize('"land"') + + expect(@result.length).to eq 1 + + expect(@result[0][0]).to eq :TERM_WITH_QUOTES + expect(@result[0][1]).to eq '"land"' + end + + it 'tests query without column name and with word starting with AND without quotes' do + @result = @evaluator.tokenize('andromeda') + + expect(@result.length).to eq 1 + + expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[0][1]).to eq 'andromeda' + end + + it 'tests query without column name and with word starting with AND with quotes' do + @result = @evaluator.tokenize('"andromeda"') + + expect(@result.length).to eq 1 + + expect(@result[0][0]).to eq :TERM_WITH_QUOTES + expect(@result[0][1]).to eq '"andromeda"' + end + + it 'tests query with column name containing AND word inside' do + @result = @evaluator.tokenize('land:some-search-term') + + expect(@result.length).to eq 3 + + expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[0][1]).to eq 'land' + expect(@result[1][0]).to eq :COLON + expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[2][1]).to eq 'some-search-term' + end + + it 'tests query with column name starting with AND' do + @result = @evaluator.tokenize('andromeda:some-search-term') + + expect(@result.length).to eq 3 + + expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[0][1]).to eq 'andromeda' + expect(@result[1][0]).to eq :COLON + expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[2][1]).to eq 'some-search-term' + end + + it 'tests query with search term without quotes containing AND word inside' do + @result = @evaluator.tokenize('name:land') + + expect(@result.length).to eq 3 + + expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[0][1]).to eq 'name' + expect(@result[1][0]).to eq :COLON + expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[2][1]).to eq 'land' + end + + it 'tests query with search term with quotes containing AND word inside' do + @result = @evaluator.tokenize('name:"land"') + + expect(@result.length).to eq 3 + + expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[0][1]).to eq 'name' + expect(@result[1][0]).to eq :COLON + expect(@result[2][0]).to eq :TERM_WITH_QUOTES + expect(@result[2][1]).to eq '"land"' + end + + it 'tests query with search term without quotes and search term starting with AND without quotes' do + @result = @evaluator.tokenize('name:andromeda') + + expect(@result.length).to eq 3 + + expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[0][1]).to eq 'name' + expect(@result[1][0]).to eq :COLON + expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[2][1]).to eq 'andromeda' + end + + it 'tests query with search term without quotes and search term starting with AND with quotes' do + @result = @evaluator.tokenize('name:"andrew"') + + expect(@result.length).to eq 3 + + expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[0][1]).to eq 'name' + expect(@result[1][0]).to eq :COLON + expect(@result[2][0]).to eq :TERM_WITH_QUOTES + expect(@result[2][1]).to eq '"andrew"' + end + + it 'tests query with search term in brackets with AND inside word' do + @result = @evaluator.tokenize('(land)') + + expect(@result.length).to eq 3 + + expect(@result[0][0]).to eq :L_BRACKET + expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[1][1]).to eq 'land' + expect(@result[2][0]).to eq :R_BRACKET + end + + it 'tests query with search term in brackets starting with AND' do + @result = @evaluator.tokenize('(andromeda)') + + expect(@result.length).to eq 3 + + expect(@result[0][0]).to eq :L_BRACKET + expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[1][1]).to eq 'andromeda' + expect(@result[2][0]).to eq :R_BRACKET + end + + it 'tests query with search term in brackets with AND inside word with quotes' do + @result = @evaluator.tokenize('("land")') + + expect(@result.length).to eq 3 + + expect(@result[0][0]).to eq :L_BRACKET + expect(@result[1][0]).to eq :TERM_WITH_QUOTES + expect(@result[1][1]).to eq '"land"' + expect(@result[2][0]).to eq :R_BRACKET + end + + it 'tests query with search term in brackets starting with AND with quotes' do + @result = @evaluator.tokenize('("andromeda")') + + expect(@result.length).to eq 3 + + expect(@result[0][0]).to eq :L_BRACKET + expect(@result[1][0]).to eq :TERM_WITH_QUOTES + expect(@result[1][1]).to eq '"andromeda"' + expect(@result[2][0]).to eq :R_BRACKET + end + + it 'tests query with multiple search terms with mixed and-or-not after dash and underscore' do + @result = @evaluator.tokenize('123-and-456 -or-2 -not_not_1') + + expect(@result.length).to eq 3 + + expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[0][1]).to eq '123-and-456' + expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[1][1]).to eq '-or-2' + expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[2][1]).to eq '-not_not_1' + end + + it 'tests query with multiple search terms starting with AND' do + @result = @evaluator.tokenize('land andrew and andromeda or orlando') + + expect(@result.length).to eq 6 + + expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[0][1]).to eq 'land' + expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[1][1]).to eq 'andrew' + expect(@result[2][0]).to eq :OPERATOR_AND + expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[3][1]).to eq 'andromeda' + expect(@result[4][0]).to eq :OPERATOR_OR + expect(@result[5][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[5][1]).to eq 'orlando' + end + + it 'tests query with multiple search terms with dash and underscore chars' do + @result = @evaluator.tokenize('land 123-andrew or -andrew- and not _ornela_ or ornela') + + expect(@result.length).to eq 9 + + expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[0][1]).to eq 'land' + expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[1][1]).to eq '123-andrew' + expect(@result[2][0]).to eq :OPERATOR_OR + expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[3][1]).to eq '-andrew-' + expect(@result[4][0]).to eq :OPERATOR_AND + expect(@result[5][0]).to eq :OPERATOR_NOT + expect(@result[6][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[6][1]).to eq '_ornela_' + expect(@result[7][0]).to eq :OPERATOR_OR + expect(@result[8][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[8][1]).to eq 'ornela' + + end + + it 'tests query with multiple search terms with dash and underscore chars under quotes' do + @result = @evaluator.tokenize('"land" "123-andrew" or "-andrew-"') + + expect(@result.length).to eq 4 + + expect(@result[0][0]).to eq :TERM_WITH_QUOTES + expect(@result[0][1]).to eq '"land"' + expect(@result[1][0]).to eq :TERM_WITH_QUOTES + expect(@result[1][1]).to eq '"123-andrew"' + expect(@result[2][0]).to eq :OPERATOR_OR + expect(@result[3][0]).to eq :TERM_WITH_QUOTES + expect(@result[3][1]).to eq '"-andrew-"' + end + + it 'tests query with multiple search terms with mixed and-or-not on start' do + @result = @evaluator.tokenize('sindy andora ornoty notary') + + expect(@result.length).to eq 4 + + expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[0][1]).to eq 'sindy' + expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[1][1]).to eq 'andora' + expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[2][1]).to eq 'ornoty' + expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[3][1]).to eq 'notary' + end + + it 'tests query with multiple search terms with mixed and-or-not on start and with logical operators between' do + @result = @evaluator.tokenize('sindy and andora-notary or not ornoty notary') + + expect(@result.length).to eq 7 + + expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[0][1]).to eq 'sindy' + expect(@result[1][0]).to eq :OPERATOR_AND + expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[2][1]).to eq 'andora-notary' + expect(@result[3][0]).to eq :OPERATOR_OR + expect(@result[4][0]).to eq :OPERATOR_NOT + expect(@result[5][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[5][1]).to eq 'ornoty' + expect(@result[6][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[6][1]).to eq 'notary' + end + + it 'tests query with multiple search terms with mixed and-or-not on start and with logical operators between' do + @result = @evaluator.tokenize('(sindy and andora-notary)or not(ornoty notary)') + + expect(@result.length).to eq 11 + + expect(@result[0][0]).to eq :L_BRACKET + expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[1][1]).to eq 'sindy' + expect(@result[2][0]).to eq :OPERATOR_AND + expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[3][1]).to eq 'andora-notary' + expect(@result[4][0]).to eq :R_BRACKET + expect(@result[5][0]).to eq :OPERATOR_OR + expect(@result[6][0]).to eq :OPERATOR_NOT + expect(@result[7][0]).to eq :L_BRACKET + expect(@result[8][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[8][1]).to eq 'ornoty' + expect(@result[9][0]).to eq :TERM_WITHOUT_QUOTES + expect(@result[9][1]).to eq 'notary' + expect(@result[10][0]).to eq :R_BRACKET + end end end diff --git a/spec/lib/query_parser_spec.rb b/spec/lib/query_parser_spec.rb index 16439fc..5ef78cd 100644 --- a/spec/lib/query_parser_spec.rb +++ b/spec/lib/query_parser_spec.rb @@ -133,7 +133,7 @@ class QueryParserTester expect(@result[:OPERATOR_AND]).to eq @expected_array_total end - it 'tests operator precedence' do + it 'tests operator precedence, or - and' do @result1 = @evaluator.parse('tag:mta or name:JF and 12_4') @result2 = @evaluator.parse('tag:mta or (name:JF and 12_4)') @@ -155,10 +155,469 @@ class QueryParserTester end - # Tests to write : - # * query with multiple column names and search terms without logical operators - # * AND NOT, OR NOT tests + it 'tests operator precedence, and - or' do + @result1 = @evaluator.parse('tag:mta and name:JF or 12_4') + @result2 = @evaluator.parse('(tag:mta and name:JF) or 12_4') + expect(@result1).to eq @result2 + + expect(@result1.length).to eq 1 + + @expected_array_part_2 = [ + {'tag' => 'mta'}, + {'name' => 'JF'} + ] + + @expected_array_total = [ + {:OPERATOR_AND => @expected_array_part_2}, + {:DEFAULT_COLUMN => '12_4'} + ] + + expect(@result1[:OPERATOR_OR]).to eq @expected_array_total + + end + + it 'tests simple query with not operator' do + @result = @evaluator.parse('not -54') + + expect(@result.length).to be 1 + expect(@result[:OPERATOR_NOT][:DEFAULT_COLUMN]).to eq '-54' + end + + it 'tests query with mixed NOT and AND logic operator' do + @result = @evaluator.parse('name:"some wild name" and not -123-456') + + @expected_and_array = [ + {'name' => '"some wild name"'}, + {:OPERATOR_NOT => {:DEFAULT_COLUMN => '-123-456'}} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_AND]).to eq @expected_and_array + end + + it 'tests query with mixed NOT, AND and OR logic operators' do + @result = @evaluator.parse('name:"some wild name" or not (tag:mta and -123-456)') + + @expected_and_array_operands = [ + {'tag' => 'mta'}, + {:DEFAULT_COLUMN => '-123-456'} + ] + + @expected_or_array_operands = [ + {'name' => '"some wild name"'}, + {:OPERATOR_NOT => {:OPERATOR_AND => @expected_and_array_operands}} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_or_array_operands + end + + it 'tests query with two default column search terms without quotes and without logical operators' do + @result = @evaluator.parse('id-5 -456') + + @expected_array = [ + {:DEFAULT_COLUMN => 'id-5'}, + {:DEFAULT_COLUMN => '-456'} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with two default column search terms with quotes and without logical operators' do + @result = @evaluator.parse('"id-5 no Q" -456') + + @expected_array = [ + {:DEFAULT_COLUMN => '"id-5 no Q"'}, + {:DEFAULT_COLUMN => '-456'} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with two default column search terms with quotes on both terms and without logical operators' do + @result = @evaluator.parse('"id-5 no Q" "wild id -456"') + + @expected_array = [ + {:DEFAULT_COLUMN => '"id-5 no Q"'}, + {:DEFAULT_COLUMN => '"wild id -456"'} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with one default column and one non defualt column, search terms without quotes and without logical operators' do + @result = @evaluator.parse('tag:mta -456') + + @expected_array = [ + {'tag' => 'mta'}, + {:DEFAULT_COLUMN => '-456'} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with one default column and one non defualt column, search terms with quotes and without logical operators' do + @result = @evaluator.parse('-1-23 name:"WILD name"') + + @expected_array = [ + {:DEFAULT_COLUMN => '-1-23'}, + {'name' => '"WILD name"'} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with two columns, search terms with quotes and without logical operators' do + @result = @evaluator.parse('tag:-1-23 name:"WILD name"') + + @expected_array = [ + {'tag' => '-1-23'}, + {'name' => '"WILD name"'} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with two columns, search terms with quotes on both and without logical operators' do + @result = @evaluator.parse('tag:"1 2 3" name:"WILD name"') + + @expected_array = [ + {'tag' => '"1 2 3"'}, + {'name' => '"WILD name"'} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests query with three columns, search terms with quotes on both and without logical operators' do + @result = @evaluator.parse('tag:"1 2 3" name:"WILD name" name:"ANOTHER wild name"') + + @expected_array = [ + {'tag' => '"1 2 3"'}, + {:OPERATOR_OR => [ + {'name' => '"WILD name"'}, + {'name' => '"ANOTHER wild name"'} + ]} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_array + end + + it 'tests complex query' do + @result = @evaluator.parse('(device-id:"with space" tag:mta no-quotes-id-123)'\ + 'or "id with quotes-5" and ( ("id with q 10" or "id with q 20")'\ + 'and ("id with Q 30" "id with Q 40") and not id-without-Q-50)') + + # (device-id:"with space" tag:mta no-quotes-id-123) or "id with quotes-5" and ( ("id with q 10" or "id with q 20") and ("id with Q 30" "id with Q 40") and not id-without-Q-50) + # _____________________A___________________________ or _______B___________ and __________________________________________________C______________________________________________ + # (____A1________________ ___A2__ ______A3________) or _______B___________ and ( ____________C1____________________ and ______________C2_______________ and ______C3___________) + + + @A = {:OPERATOR_OR => [ + {'device-id' => '"with space"'}, + {:OPERATOR_OR => [ + {'tag' => 'mta'}, + {:DEFAULT_COLUMN => 'no-quotes-id-123'}] + } + ]} + + @B = {:DEFAULT_COLUMN => '"id with quotes-5"'} + + + @C1 = {:OPERATOR_OR => [ + {:DEFAULT_COLUMN => '"id with q 10"'}, + {:DEFAULT_COLUMN => '"id with q 20"'} + ]} + + + @C2 = {:OPERATOR_OR => [ + {:DEFAULT_COLUMN => '"id with Q 30"'}, + {:DEFAULT_COLUMN => '"id with Q 40"'} + ]} + + @C3 = {:OPERATOR_NOT => {:DEFAULT_COLUMN => 'id-without-Q-50'}} + + @C = {:OPERATOR_AND => [ + {:OPERATOR_AND => [ + @C1, + @C2 + ]}, + @C3 + ]} + + @expected_final_array_result = [ + @A, + {:OPERATOR_AND => [ + @B, + @C + ]} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @expected_final_array_result + end + + it 'tests query without column name containing AND word inside without quotes' do + @result = @evaluator.parse('land') + + expect(@result.length).to eq 1 + expect(@result[:DEFAULT_COLUMN]).to eq 'land' + end + + it 'tests query without column name containing AND word inside with quotes' do + @result = @evaluator.parse('"land"') + + expect(@result.length).to eq 1 + expect(@result[:DEFAULT_COLUMN]).to eq '"land"' + end + + it 'tests query without column name and with word starting with AND without quotes' do + @result = @evaluator.parse('andromeda') + + expect(@result.length).to eq 1 + expect(@result[:DEFAULT_COLUMN]).to eq 'andromeda' + end + + it 'tests query without column name and with word starting with AND with quotes' do + @result = @evaluator.parse('"andromeda"') + + expect(@result.length).to eq 1 + expect(@result[:DEFAULT_COLUMN]).to eq '"andromeda"' + end + + it 'tests query with column name containing AND word inside' do + @result = @evaluator.parse('land:some-search-term') + + expect(@result.length).to eq 1 + expect(@result['land']).to eq 'some-search-term' + end + + it 'tests query with column name starting with AND' do + @result = @evaluator.parse('andromeda:some-search-term') + + expect(@result.length).to eq 1 + expect(@result['andromeda']).to eq 'some-search-term' + end + + it 'tests query with search term without quotes containing AND word inside' do + @result = @evaluator.parse('name:land') + + expect(@result.length).to eq 1 + expect(@result['name']).to eq 'land' + end + + it 'tests query with search term with quotes containing AND word inside' do + @result = @evaluator.parse('name:"land"') + + expect(@result.length).to eq 1 + expect(@result['name']).to eq '"land"' + end + + it 'tests query with search term without quotes and search term starting with AND without quotes' do + @result = @evaluator.parse('name:andromeda') + + expect(@result.length).to eq 1 + expect(@result['name']).to eq 'andromeda' + end + + it 'tests query with search term without quotes and search term starting with AND with quotes' do + @result = @evaluator.parse('name:"andrew"') + + expect(@result.length).to eq 1 + expect(@result['name']).to eq '"andrew"' + end + + it 'tests query with search term in brackets with AND inside word' do + @result = @evaluator.parse('(land)') + + expect(@result.length).to eq 1 + expect(@result[:DEFAULT_COLUMN]).to eq 'land' + end + + it 'tests query with search term in brackets starting with AND' do + @result = @evaluator.parse('(andromeda)') + + expect(@result.length).to eq 1 + expect(@result[:DEFAULT_COLUMN]).to eq 'andromeda' + end + + it 'tests query with multiple search terms with mixed and-or-not after dash and underscore' do + @result = @evaluator.parse('123-and-456 -or-2 -not_not_1') + + @or_array_1 = [ + {:DEFAULT_COLUMN => '-or-2'}, + {:DEFAULT_COLUMN => '-not_not_1'} + ] + + @last_or_array = [ + {:DEFAULT_COLUMN => '123-and-456'}, + {:OPERATOR_OR => @or_array_1} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @last_or_array + end + + it 'tests query with search term in brackets with AND inside word with quotes' do + @result = @evaluator.parse('("land")') + + expect(@result.length).to eq 1 + expect(@result[:DEFAULT_COLUMN]).to eq '"land"' + end + + it 'tests query with search term in brackets starting with AND with quotes' do + @result = @evaluator.parse('("andromeda")') + + expect(@result.length).to eq 1 + expect(@result[:DEFAULT_COLUMN]).to eq '"andromeda"' + end + + it 'tests query with multiple search terms starting with AND' do + @result = @evaluator.parse('land andrew and andromeda or orlando') + + @and_array = [ + {:DEFAULT_COLUMN=>'andrew'}, + {:DEFAULT_COLUMN=>'andromeda'} + ] + + @last_or_array = [ + {:OPERATOR_AND => @and_array}, + {:DEFAULT_COLUMN => 'orlando'} + ] + + @main_or_array = [ + {:DEFAULT_COLUMN => 'land'}, + {:OPERATOR_OR => @last_or_array} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @main_or_array + end + + it 'tests query with multiple search terms with dash and underscore chars' do + @result = @evaluator.parse('land 123-andrew or -andrew- and not _ornela_ or ornela') + + @and_array = [ + {:DEFAULT_COLUMN=>'-andrew-'}, + {:OPERATOR_NOT=>{:DEFAULT_COLUMN => '_ornela_'}} + ] + + @intra_or_array = [ + {:DEFAULT_COLUMN => '123-andrew'}, + {:OPERATOR_AND => @and_array} + ] + + @last_or_array = [ + {:OPERATOR_OR => @intra_or_array}, + {:DEFAULT_COLUMN => 'ornela'} + ] + + @main_or_array = [ + {:DEFAULT_COLUMN => 'land'}, + {:OPERATOR_OR => @last_or_array} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @main_or_array + + end + + it 'tests query with multiple search terms with dash and underscore chars under quotes' do + @result = @evaluator.parse('"land" "123-andrew" or "-andrew-"') + + + @last_or_array = [ + {:DEFAULT_COLUMN => '"123-andrew"'}, + {:DEFAULT_COLUMN => '"-andrew-"'} + ] + + @main_or_array = [ + {:DEFAULT_COLUMN => '"land"'}, + {:OPERATOR_OR => @last_or_array} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @main_or_array + end + + it 'tests query with multiple search terms with mixed and-or-not on start' do + @result = @evaluator.parse('sindy andora ornoty notary') + + @or_array_1 = [ + {:DEFAULT_COLUMN=>'ornoty'}, + {:DEFAULT_COLUMN=>'notary'} + ] + + @or_array_2 = [ + {:DEFAULT_COLUMN => 'andora'}, + {:OPERATOR_OR => @or_array_1} + ] + + @main_or_array = [ + {:DEFAULT_COLUMN => 'sindy'}, + {:OPERATOR_OR => @or_array_2} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @main_or_array + end + + it 'tests query with multiple search terms with mixed and-or-not on start and with logical operators between' do + @result = @evaluator.parse('sindy and andora-notary or not ornoty notary') + + @and_array = [ + {:DEFAULT_COLUMN=>'sindy'}, + {:DEFAULT_COLUMN=>'andora-notary'} + ] + + @not_or_array = [ + {:DEFAULT_COLUMN => 'ornoty'}, + {:DEFAULT_COLUMN => 'notary'} + ] + + @main_or_array = [ + {:OPERATOR_AND => @and_array}, + {:OPERATOR_NOT => { + :OPERATOR_OR => @not_or_array + }} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_OR]).to eq @main_or_array + end + + it 'tests query with multiple search terms with mixed and-or-not on start and with logical operators between' do + @result = @evaluator.parse('(sindy or andora-notary)and not(ornoty notary)') + + @or_array_1 = [ + {:DEFAULT_COLUMN=>'sindy'}, + {:DEFAULT_COLUMN=>'andora-notary'} + ] + + @not_or_array = [ + {:DEFAULT_COLUMN => 'ornoty'}, + {:DEFAULT_COLUMN => 'notary'} + ] + + @main_and_array = [ + {:OPERATOR_OR => @or_array_1}, + {:OPERATOR_NOT => { + :OPERATOR_OR => @not_or_array + }} + ] + + expect(@result.length).to eq 1 + expect(@result[:OPERATOR_AND]).to eq @main_and_array + end end end -- 2.47.3 From b2cdcfbd615a9dd6119f61f47b6cf628388451d1 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Mon, 10 Feb 2020 14:32:54 +0100 Subject: [PATCH 04/16] update lexer --- lib/lexer.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/lexer.rb b/lib/lexer.rb index 80cb51f..aa87d79 100644 --- a/lib/lexer.rb +++ b/lib/lexer.rb @@ -66,13 +66,13 @@ class Query < Racc::Parser when (text = @ss.scan(/\)/)) action { return [:R_BRACKET, text] } - when (text = @ss.scan(/(?i)or/)) + when (text = @ss.scan(/(?i)\bor\b/)) action { return [:OPERATOR_OR, text] } - when (text = @ss.scan(/(?i)and/)) + when (text = @ss.scan(/(?i)\band\b/)) action { return [:OPERATOR_AND, text] } - when (text = @ss.scan(/(?i)not/)) + when (text = @ss.scan(/(?i)\bnot\b/)) action { return [:OPERATOR_NOT, text] } when (text = @ss.scan(/"([^"]*)"/)) -- 2.47.3 From d96817e3f8cac43d3e5ccdcb0bef4d030b16aa23 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Mon, 10 Feb 2020 14:33:03 +0100 Subject: [PATCH 05/16] fix Gemfile --- Gemfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 2283ebe..42f51f3 100644 --- a/Gemfile +++ b/Gemfile @@ -23,6 +23,6 @@ group :development do gem 'rubocop', '~> 0.54', require: false gem 'rubocop-rspec', '~> 1.23', require: false gem 'simplecov', '~> 0.16', require: false - gem "rexical", '~> 1.0', require: false - gem "racc", '~> 1.4', require: false + gem 'rexical', '~> 1.0', require: false + gem 'racc', '~> 1.4', require: false end -- 2.47.3 From 21b736420b0cf2a744dc9387acd991edd5ef7f44 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Mon, 10 Feb 2020 14:47:29 +0100 Subject: [PATCH 06/16] add tests to sql generator for keyword detection --- spec/lib/text_to_sql_query_spec.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spec/lib/text_to_sql_query_spec.rb b/spec/lib/text_to_sql_query_spec.rb index c8f39c0..85df3dc 100644 --- a/spec/lib/text_to_sql_query_spec.rb +++ b/spec/lib/text_to_sql_query_spec.rb @@ -57,5 +57,10 @@ describe TextToSqlQuery do # tests complex query it { expect(described_class.new('(device_id:"with space" tags:mta no-quotes-id-123) or "id with quotes-5" and ( ("id with q 10" or "id with q 20") and ("id with Q 30" "id with Q 40") and not id-without-Q-50)', [:'players.name', :'players.value', :'players.device_id'], :"players.device_id", { tags: 'tags.name' }).where_clause).to eq(['((players.device_id ILIKE ? OR (tags.name ILIKE ? OR players.device_id ILIKE ?)) OR (players.device_id ILIKE ? AND (((players.device_id ILIKE ? OR players.device_id ILIKE ?) AND (players.device_id ILIKE ? OR players.device_id ILIKE ?)) AND NOT players.device_id ILIKE ?)))', '%with space%', '%mta%', '%no-quotes-id-123%', '%id with quotes-5%', '%id with q 10%', '%id with q 20%', '%id with Q 30%', '%id with Q 40%', '%id-without-Q-50%']) } + # tests query with multiple search terms with mixed and-or-not after dash and underscore + it { expect(described_class.new('123-and-456 -or-2 -not_not_1', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id').where_clause).to eq(['(players.device_id ILIKE ? OR (players.device_id ILIKE ? OR players.device_id ILIKE ?))', '%123-and-456%', '%-or-2%', '%-not\_not\_1%'])} + + # tests query with multiple search terms with mixed and-or-not after dash and underscore + it { expect(described_class.new('andrew or ornela', [:'players.title', :'players.tag', :'players.device_id'], :'players.device_id').where_clause).to eq(['(players.device_id ILIKE ? OR players.device_id ILIKE ?)', '%andrew%', '%ornela%'])} end end -- 2.47.3 From fe7dcbcd6fdaee414b11a4d1fc9b26050e7a82f9 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Mon, 10 Feb 2020 14:47:53 +0100 Subject: [PATCH 07/16] add required import --- spec/lib/pg_searchable_spec.rb | 1 + spec/lib/text_to_regex_query_spec.rb | 22 ---------------------- spec/lib/text_to_tsquery_spec.rb | 2 ++ 3 files changed, 3 insertions(+), 22 deletions(-) delete mode 100644 spec/lib/text_to_regex_query_spec.rb diff --git a/spec/lib/pg_searchable_spec.rb b/spec/lib/pg_searchable_spec.rb index 5a80c00..08c67ff 100644 --- a/spec/lib/pg_searchable_spec.rb +++ b/spec/lib/pg_searchable_spec.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true +require_relative '../../lib/pg_searchable_regex' describe PgSearchable do include_examples 'pg_search', VectorModel diff --git a/spec/lib/text_to_regex_query_spec.rb b/spec/lib/text_to_regex_query_spec.rb deleted file mode 100644 index 57cc291..0000000 --- a/spec/lib/text_to_regex_query_spec.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -describe TextToRegexQuery do - include_examples 'pg_search', SimpleVectorModel - describe '.new' do - # just default - it { expect(described_class.new('some-default-value', [:"players.name"], :"players.name").where_clause(SimpleVectorModel)).to eq([' players.name like ? OR 1<>1', '%some-default-value%']) } - - # default and named - it { expect(described_class.new('name:hamo id:1', [:"players.name"], :"players.name").where_clause(SimpleVectorModel)).to eq([' players.name like ? OR players.name like ? OR 1<>1', '%id:1%', '%hamo%']) } - - # escape characters - it { expect(described_class.new('name:hamo id:1',[:"players.name"], :"players.name").where_clause(SimpleVectorModel)).to eq([' players.name like ? OR players.name like ? OR 1<>1', '%id:1%', '%hamo%']) } - - # default and explicit with underscore - it { expect(described_class.new('device_id:-123', [:"players.name", :"players.device_id"], :"players.device_id").where_clause(SimpleVectorModel)).to eq([" players.device_id like ? OR 1<>1", "%-123%"]) } - - # default and explicit with underscore with multiple columns - it { expect(described_class.new('name:bla device_id:-123', [:"players.name", :"players.device_id"], :"players.device_id").where_clause(SimpleVectorModel)).to eq([" players.name like ? OR players.device_id like ? OR 1<>1", "%bla%", "%-123%"]) } - - end -end diff --git a/spec/lib/text_to_tsquery_spec.rb b/spec/lib/text_to_tsquery_spec.rb index 330899b..99e396a 100644 --- a/spec/lib/text_to_tsquery_spec.rb +++ b/spec/lib/text_to_tsquery_spec.rb @@ -1,4 +1,6 @@ # frozen_string_literal: true +require_relative '../../lib/text_to_tsquery' + describe TextToTsquery do describe '.new' do -- 2.47.3 From f696a7cd20e4f14b75fc2952f169c297df8e7468 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Mon, 10 Feb 2020 16:42:59 +0100 Subject: [PATCH 08/16] add CAST AS TEXT; change how default_field is assigned --- lib/pg_searchable_regex.rb | 2 +- lib/text_to_sql_query.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pg_searchable_regex.rb b/lib/pg_searchable_regex.rb index 4387373..dc1b18d 100644 --- a/lib/pg_searchable_regex.rb +++ b/lib/pg_searchable_regex.rb @@ -36,7 +36,7 @@ module PgSearchable @ts_skip_cache_update = skip_callback @ts_wildcard = wildcard @ts_joins = joins - @default_field = (default_field.to_sym || fields.first) + @default_field = default_field.to_s.empty? ? fields.first : default_field.to_sym ts_add_scope end diff --git a/lib/text_to_sql_query.rb b/lib/text_to_sql_query.rb index d473f48..c464349 100644 --- a/lib/text_to_sql_query.rb +++ b/lib/text_to_sql_query.rb @@ -29,7 +29,7 @@ class TextToSqlQuery case first_key when :DEFAULT_COLUMN escaped_node_value = handle_special_chars node_value - ["#{@default_field.to_s} ILIKE ?", "%#{escaped_node_value}%"] + ["CAST(#{@default_field.to_s} AS TEXT) ILIKE ?", "%#{escaped_node_value}%"] when :OPERATOR_OR generate_expression_for_logical_operator(:OR, node_value) when :OPERATOR_AND @@ -53,7 +53,7 @@ class TextToSqlQuery if mapping.nil? raise "Unknown field '#{first_key.to_s}'" else - ["#{mapping.to_s} ILIKE ?", "%#{escaped_node_value}%"] + ["CAST(#{mapping.to_s} AS TEXT) ILIKE ?", "%#{escaped_node_value}%"] end end end -- 2.47.3 From ebc21919bf52a10ad6a3d45214aea4c2a584cdb8 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Tue, 11 Feb 2020 10:31:23 +0100 Subject: [PATCH 09/16] update models to reflect expected field names --- spec/support/models.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/support/models.rb b/spec/support/models.rb index f18beb4..a5208c7 100755 --- a/spec/support/models.rb +++ b/spec/support/models.rb @@ -2,7 +2,7 @@ class VectorModel < ActiveRecord::Base include PgSearchable - pg_search fields: %i[id name value], cache: :search_cache + pg_search fields: %i[vector_models.id vector_models.name vector_models.value], cache: :search_cache end class VectorModelWithoutCallback < ActiveRecord::Base @@ -32,7 +32,7 @@ end class SimpleVectorModel < ActiveRecord::Base self.table_name = :vector_models include PgSearchable - pg_search fields: %i[id name value], cache: :search_cache, language: :simple + pg_search fields: %i[vector_models.id vector_models.name vector_models.value], cache: :search_cache, language: :simple end class SimplePlayerModel < ActiveRecord::Base -- 2.47.3 From d307811f5ffa1f7cae0e315ceb0ea608e3ea81a4 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Tue, 11 Feb 2020 10:31:39 +0100 Subject: [PATCH 10/16] add integration tests for simple model --- spec/lib/pg_searchable_new_spec.rb | 101 +++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 spec/lib/pg_searchable_new_spec.rb diff --git a/spec/lib/pg_searchable_new_spec.rb b/spec/lib/pg_searchable_new_spec.rb new file mode 100644 index 0000000..9775582 --- /dev/null +++ b/spec/lib/pg_searchable_new_spec.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true +require_relative '../../lib/pg_searchable_regex' + +describe PgSearchable do + describe 'pg_search' do + + describe 'properties' do + describe 'scope' do + it 'defaults to "scope_search"' do + expect(VectorModel).to respond_to(:scope_search) + end + + it 'can use a different scope name' do + expect(VectorModelWithCustomSearchScope).to respond_to(:fulltext) + end + + it 'doesnt pollutes the default method name if customized' do + expect(VectorModelWithCustomSearchScope).not_to respond_to(:scope_search) + end + end + + describe 'language' do + it 'defaults to english lexemes' do + record = VectorModel.create name: 'something', value: 'amazing' + expect(VectorModel.scope_search('value:amaz')).to include(record) + end + + it 'can be changed to simple to avoid lexeme truncation' do + record = SimpleVectorModel.create name: 'something', value: 'amazing' + expect(SimpleVectorModel.scope_search('value:amazings')).not_to include(record) + end + end + end + + describe 'single model' do + it 'searches only default column if no column name is used' do + record1 = VectorModel.create name: 'hamo', value: '-45' + record2 = VectorModel.create name: 'meho', value: '120' + + expect(VectorModel.scope_search('45')).to be_empty + expect(VectorModel.scope_search("#{record2.id}")).to contain_exactly(record2) + end + + it 'searches column declared in search term' do + record1 = VectorModel.create name: 'hamo', value: '-45' + record2 = VectorModel.create name: 'meho', value: '120' + + expect(VectorModel.scope_search('name:45')).to be_empty + expect(VectorModel.scope_search('value:120')).to contain_exactly(record2) + end + + it 'searches column declared in search term with only partial match' do + record1 = VectorModel.create name: 'hamo', value: '-45' + record2 = VectorModel.create name: 'meho', value: '120' + + expect(VectorModel.scope_search('value:-4')).to contain_exactly(record1) + expect(VectorModel.scope_search('value:12')).to contain_exactly(record2) + end + + it 'handles multiple search terms without logical operator between' do + record1 = VectorModel.create name: 'hamo', value: '-45' + record2 = VectorModel.create name: 'meho', value: '120' + record3 = VectorModel.create name: 'atif', value: '80' + + expect(VectorModel.scope_search('value:-4 name:meho')).to contain_exactly(record1, record2) + end + + it 'handles multiple search terms with AND operator between' do + record1 = VectorModel.create name: 'hamo', value: '-45' + record2 = VectorModel.create name: 'meho', value: '120' + record3 = VectorModel.create name: 'atif', value: '80' + + expect(VectorModel.scope_search('value:-4 AND name:meho')).to be_empty + end + + it 'handles search term with NOT operator' do + record1 = VectorModel.create name: 'hamo', value: '-45' + record2 = VectorModel.create name: 'meho', value: '120' + record3 = VectorModel.create name: 'atif', value: '80' + + expect(VectorModel.scope_search('NOT value:0')).to contain_exactly(record1) + end + + it 'handles multiple search terms with mixed logical operators between without brackets' do + record1 = VectorModel.create name: 'hamo', value: '-45' + record2 = VectorModel.create name: 'meho', value: '120' + record3 = VectorModel.create name: 'atif', value: '80' + + expect(VectorModel.scope_search('name:hamo AND value:120 OR value:80')).to contain_exactly(record3) + end + + it 'handles multiple search terms with mixed logical operators between with brackets' do + record1 = VectorModel.create name: 'hamo', value: '-45' + record2 = VectorModel.create name: 'meho', value: '120' + record3 = VectorModel.create name: 'atif', value: '80' + + expect(VectorModel.scope_search('name:hamo AND (value:120 OR value:80)')).to be_empty + end + end + end +end -- 2.47.3 From 94767d2b240daf7ee96fedce365d512be4b78255 Mon Sep 17 00:00:00 2001 From: Bilal Catic Date: Tue, 11 Feb 2020 11:17:29 +0100 Subject: [PATCH 11/16] fix sql query generator tests to include CAST --- coverage/index.html | 2424 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 2186 insertions(+), 238 deletions(-) diff --git a/coverage/index.html b/coverage/index.html index cb2a0c4..55faeff 100644 --- a/coverage/index.html +++ b/coverage/index.html @@ -1,7 +1,7 @@ - Code coverage for Pg searchable + Code coverage for Gem @@ -14,27 +14,27 @@ loading