require_relative '../../lib/lexer' class QueryLexerTester describe 'Testing the Lexer' do before do @evaluator = Query.new end it 'tests bracket expression' do @result = @evaluator.tokenize('()') expect(@result.length).to eq 2 expect(@result[0][0]).to eq :L_BRACKET expect(@result[1][0]).to eq :R_BRACKET end it 'tests bracket expression with spaces' do @result = @evaluator.tokenize(' ( ) ') expect(@result.length).to eq 2 expect(@result[0][0]).to eq :L_BRACKET expect(@result[1][0]).to eq :R_BRACKET end it 'tests expression with OR operator' do @result = @evaluator.tokenize('() or () OR ()') expect(@result.length).to eq 8 expect(@result[0][0]).to eq :L_BRACKET expect(@result[1][0]).to eq :R_BRACKET expect(@result[2][0]).to eq :OPERATOR_OR expect(@result[3][0]).to eq :L_BRACKET expect(@result[4][0]).to eq :R_BRACKET expect(@result[5][0]).to eq :OPERATOR_OR expect(@result[6][0]).to eq :L_BRACKET expect(@result[7][0]).to eq :R_BRACKET end it 'tests expression with AND operator' do @result = @evaluator.tokenize('() AND () and ()') expect(@result.length).to eq 8 expect(@result[0][0]).to eq :L_BRACKET expect(@result[1][0]).to eq :R_BRACKET expect(@result[2][0]).to eq :OPERATOR_AND expect(@result[3][0]).to eq :L_BRACKET expect(@result[4][0]).to eq :R_BRACKET expect(@result[5][0]).to eq :OPERATOR_AND expect(@result[6][0]).to eq :L_BRACKET expect(@result[7][0]).to eq :R_BRACKET end it 'tests expression with NOT OR and NOT AND operator' do @result = @evaluator.tokenize('() NOT or () not AND ()') expect(@result.length).to eq 10 expect(@result[0][0]).to eq :L_BRACKET expect(@result[1][0]).to eq :R_BRACKET expect(@result[2][0]).to eq :OPERATOR_NOT expect(@result[3][0]).to eq :OPERATOR_OR expect(@result[4][0]).to eq :L_BRACKET expect(@result[5][0]).to eq :R_BRACKET expect(@result[6][0]).to eq :OPERATOR_NOT expect(@result[7][0]).to eq :OPERATOR_AND expect(@result[8][0]).to eq :L_BRACKET expect(@result[9][0]).to eq :R_BRACKET end it 'tests search term under quotes' do @result = @evaluator.tokenize('"123-456"') expect(@result.length).to eq 1 expect(@result[0][0]).to eq :TERM_WITH_QUOTES expect(@result[0][1]).to eq '"123-456"' end it 'tests term without quotes' do @result = @evaluator.tokenize('device_id') expect(@result.length).to eq 1 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'device_id' end it 'tests integer term without quotes' do @result = @evaluator.tokenize('123') expect(@result.length).to eq 1 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq '123' end it 'tests multiple terms without quotes' do @result = @evaluator.tokenize('device_id tag 123-456 name123') expect(@result.length).to eq 4 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'device_id' expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[1][1]).to eq 'tag' expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[2][1]).to eq '123-456' expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[3][1]).to eq 'name123' end it 'tests simple query with column name and search term without quotes' do @result = @evaluator.tokenize('name:JF') expect(@result.length).to eq 3 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'name' expect(@result[1][0]).to eq :COLON expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[2][1]).to eq 'JF' end it 'tests simple query with two columns with name and search terms without quotes' do @result = @evaluator.tokenize('name:JF tag:mta') expect(@result.length).to eq 6 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'name' expect(@result[1][0]).to eq :COLON expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[2][1]).to eq 'JF' expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[3][1]).to eq 'tag' expect(@result[4][0]).to eq :COLON expect(@result[5][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[5][1]).to eq 'mta' end it 'tests simple query with column name and search term with quotes' do @result = @evaluator.tokenize('name:"name with space"') expect(@result.length).to eq 3 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'name' expect(@result[1][0]).to eq :COLON expect(@result[2][0]).to eq :TERM_WITH_QUOTES expect(@result[2][1]).to eq '"name with space"' end it 'tests search term with quotes containing non alphanumerical characters' do @result = @evaluator.tokenize('"|*|/\()#-!=<>&$"') expect(@result.length).to eq 1 expect(@result[0][0]).to eq :TERM_WITH_QUOTES expect(@result[0][1]).to eq '"|*|/\()#-!=<>&$"' end it 'tests simple query in brackets' do @result = @evaluator.tokenize('(name:"name with space")') expect(@result.length).to eq 5 expect(@result[0][0]).to eq :L_BRACKET expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[1][1]).to eq 'name' expect(@result[2][0]).to eq :COLON expect(@result[3][0]).to eq :TERM_WITH_QUOTES expect(@result[3][1]).to eq '"name with space"' expect(@result[4][0]).to eq :R_BRACKET end it 'tests multiple query wtih brackets' do @result = @evaluator.tokenize('(name:"name with space") or (tag:mta)') expect(@result.length).to eq 11 expect(@result[0][0]).to eq :L_BRACKET expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[1][1]).to eq 'name' expect(@result[2][0]).to eq :COLON expect(@result[3][0]).to eq :TERM_WITH_QUOTES expect(@result[3][1]).to eq '"name with space"' expect(@result[4][0]).to eq :R_BRACKET expect(@result[5][0]).to eq :OPERATOR_OR expect(@result[6][0]).to eq :L_BRACKET expect(@result[7][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[7][1]).to eq 'tag' expect(@result[8][0]).to eq :COLON expect(@result[9][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[9][1]).to eq 'mta' expect(@result[10][0]).to eq :R_BRACKET end it 'tests complex query' do @result = @evaluator.tokenize('(device-id:"with space" tag:mta no-quotes-id-123)'\ 'or "id with quotes-5" and ( ("id with q 10" or "id with q 20")'\ 'and ("id with Q 30" "id with Q 40") and not id-without-Q-50)') expect(@result.length).to eq 27 expect(@result[0][0]).to eq :L_BRACKET expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[1][1]).to eq 'device-id' expect(@result[2][0]).to eq :COLON expect(@result[3][0]).to eq :TERM_WITH_QUOTES expect(@result[3][1]).to eq '"with space"' expect(@result[4][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[4][1]).to eq 'tag' expect(@result[5][0]).to eq :COLON expect(@result[6][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[6][1]).to eq 'mta' expect(@result[7][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[7][1]).to eq 'no-quotes-id-123' expect(@result[8][0]).to eq :R_BRACKET expect(@result[9][0]).to eq :OPERATOR_OR expect(@result[10][0]).to eq :TERM_WITH_QUOTES expect(@result[10][1]).to eq '"id with quotes-5"' expect(@result[11][0]).to eq :OPERATOR_AND expect(@result[12][0]).to eq :L_BRACKET expect(@result[13][0]).to eq :L_BRACKET expect(@result[14][0]).to eq :TERM_WITH_QUOTES expect(@result[14][1]).to eq '"id with q 10"' expect(@result[15][0]).to eq :OPERATOR_OR expect(@result[16][0]).to eq :TERM_WITH_QUOTES expect(@result[16][1]).to eq '"id with q 20"' expect(@result[17][0]).to eq :R_BRACKET expect(@result[18][0]).to eq :OPERATOR_AND expect(@result[19][0]).to eq :L_BRACKET expect(@result[20][0]).to eq :TERM_WITH_QUOTES expect(@result[20][1]).to eq '"id with Q 30"' expect(@result[21][0]).to eq :TERM_WITH_QUOTES expect(@result[21][1]).to eq '"id with Q 40"' expect(@result[22][0]).to eq :R_BRACKET expect(@result[23][0]).to eq :OPERATOR_AND expect(@result[24][0]).to eq :OPERATOR_NOT expect(@result[25][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[25][1]).to eq 'id-without-Q-50' expect(@result[26][0]).to eq :R_BRACKET end it 'tests query with -or-, -and- and -not- words inside quoted expression' do @result = @evaluator.tokenize('tag:"tag with or and not inside"') expect(@result.length).to eq 3 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'tag' expect(@result[1][0]).to eq :COLON expect(@result[2][0]).to eq :TERM_WITH_QUOTES expect(@result[2][1]).to eq '"tag with or and not inside"' end it 'tests query without column name containing AND word inside without quotes' do @result = @evaluator.tokenize('land') expect(@result.length).to eq 1 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'land' end it 'tests query without column name containing AND word inside with quotes' do @result = @evaluator.tokenize('"land"') expect(@result.length).to eq 1 expect(@result[0][0]).to eq :TERM_WITH_QUOTES expect(@result[0][1]).to eq '"land"' end it 'tests query without column name and with word starting with AND without quotes' do @result = @evaluator.tokenize('andromeda') expect(@result.length).to eq 1 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'andromeda' end it 'tests query without column name and with word starting with AND with quotes' do @result = @evaluator.tokenize('"andromeda"') expect(@result.length).to eq 1 expect(@result[0][0]).to eq :TERM_WITH_QUOTES expect(@result[0][1]).to eq '"andromeda"' end it 'tests query with column name containing AND word inside' do @result = @evaluator.tokenize('land:some-search-term') expect(@result.length).to eq 3 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'land' expect(@result[1][0]).to eq :COLON expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[2][1]).to eq 'some-search-term' end it 'tests query with column name starting with AND' do @result = @evaluator.tokenize('andromeda:some-search-term') expect(@result.length).to eq 3 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'andromeda' expect(@result[1][0]).to eq :COLON expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[2][1]).to eq 'some-search-term' end it 'tests query with search term without quotes containing AND word inside' do @result = @evaluator.tokenize('name:land') expect(@result.length).to eq 3 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'name' expect(@result[1][0]).to eq :COLON expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[2][1]).to eq 'land' end it 'tests query with search term with quotes containing AND word inside' do @result = @evaluator.tokenize('name:"land"') expect(@result.length).to eq 3 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'name' expect(@result[1][0]).to eq :COLON expect(@result[2][0]).to eq :TERM_WITH_QUOTES expect(@result[2][1]).to eq '"land"' end it 'tests query with search term without quotes and search term starting with AND without quotes' do @result = @evaluator.tokenize('name:andromeda') expect(@result.length).to eq 3 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'name' expect(@result[1][0]).to eq :COLON expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[2][1]).to eq 'andromeda' end it 'tests query with search term without quotes and search term starting with AND with quotes' do @result = @evaluator.tokenize('name:"andrew"') expect(@result.length).to eq 3 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'name' expect(@result[1][0]).to eq :COLON expect(@result[2][0]).to eq :TERM_WITH_QUOTES expect(@result[2][1]).to eq '"andrew"' end it 'tests query with search term in brackets with AND inside word' do @result = @evaluator.tokenize('(land)') expect(@result.length).to eq 3 expect(@result[0][0]).to eq :L_BRACKET expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[1][1]).to eq 'land' expect(@result[2][0]).to eq :R_BRACKET end it 'tests query with search term in brackets starting with AND' do @result = @evaluator.tokenize('(andromeda)') expect(@result.length).to eq 3 expect(@result[0][0]).to eq :L_BRACKET expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[1][1]).to eq 'andromeda' expect(@result[2][0]).to eq :R_BRACKET end it 'tests query with search term in brackets with AND inside word with quotes' do @result = @evaluator.tokenize('("land")') expect(@result.length).to eq 3 expect(@result[0][0]).to eq :L_BRACKET expect(@result[1][0]).to eq :TERM_WITH_QUOTES expect(@result[1][1]).to eq '"land"' expect(@result[2][0]).to eq :R_BRACKET end it 'tests query with search term in brackets starting with AND with quotes' do @result = @evaluator.tokenize('("andromeda")') expect(@result.length).to eq 3 expect(@result[0][0]).to eq :L_BRACKET expect(@result[1][0]).to eq :TERM_WITH_QUOTES expect(@result[1][1]).to eq '"andromeda"' expect(@result[2][0]).to eq :R_BRACKET end it 'tests query with multiple search terms with mixed and-or-not after dash and underscore' do @result = @evaluator.tokenize('123-and-456 -or-2 -not_not_1') expect(@result.length).to eq 3 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq '123-and-456' expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[1][1]).to eq '-or-2' expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[2][1]).to eq '-not_not_1' end it 'tests query with multiple search terms starting with AND' do @result = @evaluator.tokenize('land andrew and andromeda or orlando') expect(@result.length).to eq 6 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'land' expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[1][1]).to eq 'andrew' expect(@result[2][0]).to eq :OPERATOR_AND expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[3][1]).to eq 'andromeda' expect(@result[4][0]).to eq :OPERATOR_OR expect(@result[5][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[5][1]).to eq 'orlando' end it 'tests query with multiple search terms with dash and underscore chars' do @result = @evaluator.tokenize('land 123-andrew or -andrew- and not _ornela_ or ornela') expect(@result.length).to eq 9 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'land' expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[1][1]).to eq '123-andrew' expect(@result[2][0]).to eq :OPERATOR_OR expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[3][1]).to eq '-andrew-' expect(@result[4][0]).to eq :OPERATOR_AND expect(@result[5][0]).to eq :OPERATOR_NOT expect(@result[6][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[6][1]).to eq '_ornela_' expect(@result[7][0]).to eq :OPERATOR_OR expect(@result[8][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[8][1]).to eq 'ornela' end it 'tests query with multiple search terms with dash and underscore chars under quotes' do @result = @evaluator.tokenize('"land" "123-andrew" or "-andrew-"') expect(@result.length).to eq 4 expect(@result[0][0]).to eq :TERM_WITH_QUOTES expect(@result[0][1]).to eq '"land"' expect(@result[1][0]).to eq :TERM_WITH_QUOTES expect(@result[1][1]).to eq '"123-andrew"' expect(@result[2][0]).to eq :OPERATOR_OR expect(@result[3][0]).to eq :TERM_WITH_QUOTES expect(@result[3][1]).to eq '"-andrew-"' end it 'tests query with multiple search terms with mixed and-or-not on start' do @result = @evaluator.tokenize('sindy andora ornoty notary') expect(@result.length).to eq 4 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'sindy' expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[1][1]).to eq 'andora' expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[2][1]).to eq 'ornoty' expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[3][1]).to eq 'notary' end it 'tests query with multiple search terms with mixed and-or-not on start and with logical operators between' do @result = @evaluator.tokenize('sindy and andora-notary or not ornoty notary') expect(@result.length).to eq 7 expect(@result[0][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[0][1]).to eq 'sindy' expect(@result[1][0]).to eq :OPERATOR_AND expect(@result[2][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[2][1]).to eq 'andora-notary' expect(@result[3][0]).to eq :OPERATOR_OR expect(@result[4][0]).to eq :OPERATOR_NOT expect(@result[5][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[5][1]).to eq 'ornoty' expect(@result[6][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[6][1]).to eq 'notary' end it 'tests query with multiple search terms with mixed and-or-not on start and with logical operators between' do @result = @evaluator.tokenize('(sindy and andora-notary)or not(ornoty notary)') expect(@result.length).to eq 11 expect(@result[0][0]).to eq :L_BRACKET expect(@result[1][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[1][1]).to eq 'sindy' expect(@result[2][0]).to eq :OPERATOR_AND expect(@result[3][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[3][1]).to eq 'andora-notary' expect(@result[4][0]).to eq :R_BRACKET expect(@result[5][0]).to eq :OPERATOR_OR expect(@result[6][0]).to eq :OPERATOR_NOT expect(@result[7][0]).to eq :L_BRACKET expect(@result[8][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[8][1]).to eq 'ornoty' expect(@result[9][0]).to eq :TERM_WITHOUT_QUOTES expect(@result[9][1]).to eq 'notary' expect(@result[10][0]).to eq :R_BRACKET end end end