old-gem/lib/specification.rex

class Query
macro
    L_BRACKET                       \(
    R_BRACKET                       \)
    SPACE                           \ +     # Space char
    OPERATOR_OR                     (?i)or
    OPERATOR_AND                    (?i)and
    OPERATOR_NOT                    (?i)not
    TERM_WITH_QUOTES                "([^"]*)"
    TERM_WITHOUT_QUOTES             [a-zA-Z0-9\-_]+
    COLON                           \:



rule
    {SPACE}                         # No action
    {L_BRACKET}                     { return [:L_BRACKET, text] }
    {R_BRACKET}                     { return [:R_BRACKET, text] }
    {OPERATOR_OR}                   { return [:OPERATOR_OR, text] }
    {OPERATOR_AND}                  { return [:OPERATOR_AND, text] }
    {OPERATOR_NOT}                  { return [:OPERATOR_NOT, text] }
    {TERM_WITH_QUOTES}              { return [:TERM_WITH_QUOTES, text] }
    {TERM_WITHOUT_QUOTES}           { return [:TERM_WITHOUT_QUOTES, text] }
    {COLON}                         { return [:COLON, text] }

inner
  def tokenize(code)
    scan_setup(code)
    tokens = []
    while token = next_token
      tokens << token
    end
    tokens
  end
end