Upstream sync

coverage/.last_run.json

@@ -1,5 +1,5 @@
 {
   "result": {
-    "covered_percent": 66.92
+    "covered_percent": 78.1
   }
 }

coverage/.resultset.json

@@ -9,69 +9,73 @@
         1,
         1,
         null,
-        0,
+        1,
-        0,
+        1,
         null,
-        0,
+        1,
+        13,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        1,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        13,
+        13,
+        13,
+        13,
+        13,
+        13,
+        13,
+        13,
+        13,
+        13,
+        null,
+        null,
+        1,
+        13,
+        13,
+        24,
+        23,
+        null,
+        null,
+        null,
+        null,
+        1,
+        24,
+        null,
+        24,
+        null,
+        null,
+        1,
         0,
         null,
         null,
+        1,
+        16,
         null,
         null,
+        1,
+        13,
         null,
-        0,
+        null,
+        1,
         0,
         null,
         null,
-        null,
+        1,
-        null,
-        null,
-        null,
-        null,
-        null,
-        null,
-        null,
-        null,
-        0,
-        0,
-        0,
-        0,
-        0,
-        0,
-        0,
-        0,
-        0,
-        0,
-        null,
-        null,
-        0,
-        0,
-        0,
-        null,
-        null,
-        null,
-        0,
-        0,
-        0,
-        null,
-        null,
-        0,
-        0,
-        null,
-        null,
-        0,
-        0,
-        null,
-        null,
-        0,
-        0,
-        null,
-        null,
-        0,
-        0,
-        null,
-        null,
-        0,
         0,
         0,
         0,
@@ -181,57 +185,121 @@
         null
       ],
       "/home/hamo/projects/toptal/outfrontmedia/pg_searchable/lib/text_to_sql_query.rb": [
+        1,
         null,
+        1,
+        1,
+        44,
+        44,
+        44,
+        44,
+        114,
+        114,
+        114,
         null,
+        44,
+        13,
         null,
         null,
         null,
+        1,
+        44,
+        44,
+        44,
         null,
         null,
+        1,
         null,
+        1,
+        128,
+        128,
+        128,
         null,
+        23,
+        23,
         null,
+        22,
         null,
+        15,
         null,
+        10,
         null,
+        10,
+        0,
         null,
         null,
+        10,
+        10,
         null,
+        10,
         null,
         null,
         null,
+        58,
+        58,
+        58,
+        2,
         null,
+        56,
         null,
         null,
         null,
         null,
+        1,
+        37,
+        0,
         null,
         null,
+        37,
+        37,
         null,
+        37,
+        0,
         null,
         null,
+        37,
+        0,
         null,
         null,
+        37,
+        37,
         null,
+        37,
+        37,
         null,
+        37,
         null,
         null,
+        1,
+        81,
+        81,
+        81,
+        81,
+        81,
         null,
+        null
+      ],
+      "/home/hamo/projects/toptal/outfrontmedia/pg_searchable/lib/parser.rb": [
         null,
         null,
         null,
         null,
         null,
         null,
+        1,
         null,
+        1,
         null,
+        1,
         null,
+        1,
         null,
         null,
         null,
         null,
         null,
         null,
+        1,
         null,
         null,
         null,
@@ -240,6 +308,7 @@
         null,
         null,
         null,
+        1,
         null,
         null,
         null,
@@ -248,22 +317,29 @@
         null,
         null,
         null,
+        1,
         null,
         null,
         null,
+        1,
         null,
         null,
         null,
+        1,
         null,
         null,
         null,
+        1,
         null,
         null,
         null,
+        1,
         null,
         null,
+        1,
         null,
         null,
+        1,
         null,
         null,
         null,
@@ -273,9 +349,255 @@
         null,
         null,
         null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        null,
+        1,
+        null,
+        1,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        null,
+        1,
+        null,
+        1,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        0,
+        null,
+        null,
+        null
+      ],
+      "/home/hamo/projects/toptal/outfrontmedia/pg_searchable/lib/lexer.rb": [
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        1,
+        1,
+        1,
+        null,
+        1,
+        null,
+        1,
+        1,
+        1,
+        null,
+        1,
+        132,
+        132,
+        132,
+        null,
+        null,
+        1,
+        702,
+        null,
+        null,
+        1,
+        94,
+        94,
+        null,
+        1,
+        null,
+        1,
+        0,
+        0,
+        0,
+        null,
+        null,
+        null,
+        1,
+        0,
+        0,
+        null,
+        null,
+        null,
+        1,
+        834,
+        null,
+        null,
+        703,
+        703,
+        null,
+        null,
+        1,
+        939,
+        939,
+        939,
+        null,
+        null,
+        939,
+        null,
+        null,
+        702,
+        106,
+        null,
+        649,
+        106,
+        null,
+        596,
+        84,
+        null,
+        554,
+        84,
+        null,
+        512,
+        46,
+        null,
+        489,
+        132,
+        null,
+        423,
+        610,
+        null,
+        118,
+        236,
+        null,
+        null,
+        null,
+        0,
+        0,
+        939,
+        null,
+        null,
+        0,
+        null,
+        237,
+        null,
+        null,
+        1,
+        38,
+        38,
+        38,
+        174,
+        null,
+        38,
+        null,
         null
       ]
     },
-    "timestamp": 1580827025
+    "timestamp": 1583928139
   }
 }

lib/pg_searchable_regex.rb

@@ -42,14 +42,17 @@ module PgSearchable
 
     def ts_add_scope
       class_eval do
-        scope ts_scope_method, ->(value) { ts_search(value) }
+        scope ts_scope_method, ->(value) do
+          resulting_ids = ts_search(value).map(&:id)
+          where(id: resulting_ids)
+        end
       end
     end
 
     def ts_search(value)
       return if @ts_search_fields.blank? || value.blank?
       includes(@ts_joins).references(:all).where(
-      TextToSqlQuery.new(value, @ts_search_fields, @default_field, @ts_search_fields_mappings).where_clause)
+        TextToSqlQuery.new(value, @ts_search_fields, @default_field, @ts_search_fields_mappings).where_clause).distinct
     end
 
     def should_update_cache_field?

lib/text_to_regex_query.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-# transforms "english like" text queries into a where clause with regex
-# https://www.postgresql.org/docs/9.5/textsearch-controls.html#TEXTSEARCH-PARSING-QUERIES
-
-class TextToRegexQuery
-  def initialize(text, fields, default_field, fields_mappings = {})
-    @text = text.to_s.strip
-    @fields = fields.map(&:to_sym)
-    @default_field = default_field.to_sym
-    @fields_mappings = fields_mappings.merge(@fields.reduce({}) do |mappings, field|
-      table_name, field_name = field.to_s.split(".")
-      mappings[field_name.to_sym] = field
-      mappings 
-    end)
-  end
-
-  def where_clause(query)
-    @cleared_text = @text.dup
-    @column_chunks = []
-    remove_duplicated_spaces
-    extract_columns
-    escape_special_characters
-    generate_where_clause(query)
-  end
-
-private
-
-  def remove_duplicated_spaces
-    @cleared_text.gsub!(/\s+/, ' ')
-  end
-
-  def escape_special_characters
-    @cleared_text.gsub!(/\_/, '\_')
-    @cleared_text.tr!('\\', '\\')
-    @cleared_text.gsub!(/%/, '\%')
-  end
-
-  def extract_columns
-    column_search_term_pairs = @cleared_text.scan(/([A-Za-z0-9_]+:[\w\_-]+)/)
-
-    @column_chunks = (column_search_term_pairs.flatten.map do |pair|
-      column, term = pair.split(':')
-      next unless @fields_mappings.include?(column.to_sym) 
-      @cleared_text.gsub!(pair, '')
-      { @fields_mappings[column.to_sym] => term }
-    end).compact
-    unless @cleared_text.strip.empty?
-      @column_chunks = [{ @default_field.to_s => @cleared_text.strip }] + @column_chunks
-    end
-    @column_chunks
-  end
-
-  def generate_where_clause(query)
-    where_clause = ''
-    columns = @column_chunks.map { |c| c.keys.first } 
-    values = @column_chunks.map { |c| c.values.first }
-
-    columns.each do |column|
-      quoted_column = '"' + column.to_s.gsub(".",'"."') + '"'
-      where_clause += "#{quoted_column} ILIKE ? OR "
-    end
-    where_clause += " 1<>1 "
-    regexed_values = values.map { |v| "%#{v}%" } 
-    query.where([where_clause] + regexed_values) 
-  end
-end

spec/lib/pg_searchable_new_spec.rb

@@ -136,6 +136,16 @@ describe PgSearchable do
         expect(DynamicModelWithTagValues.scope_search('tag:red value:"not"')).to contain_exactly(record1, record2)
       end
 
+      it 'can find models without tags' do
+        record1 = DynamicModelWithTagValues.create name: 'something', value: 'amazing'
+        record2 = DynamicModelWithTagValues.create name: 'new record', value: 'not so amazing'
+        Tag.create(taggable: record1, value: 'green')
+
+        expect(DynamicModelWithTagValues.scope_search('tag:green or value:"not"')).to contain_exactly(record1, record2)
+      end
+
+
+
       it 'can search in referenced column and in model columns with multiple search terms connected with logical operators' do
         record1 = DynamicModelWithTagValues.create name: 'something', value: 'amazing'
         record2 = DynamicModelWithTagValues.create name: 'new record', value: 'not so amazing'

spec/lib/pg_searchable_spec.rb

@@ -1,116 +0,0 @@
-# frozen_string_literal: true
-require_relative '../../lib/pg_searchable_regex'
-
-describe PgSearchable do
-  include_examples 'pg_search', VectorModel
-  include_examples 'pg_search', VectorWithCustomPrimaryKeyModel
-  include_examples 'pg_search', VectorWithCustomCallback
-  include_examples 'pg_search', SimpleVectorModel
-  include_examples 'pg_search', VectorWithoutWildcardModel
-  include_examples 'pg_search', VectorModelWithCustomSearchScope, 'fulltext'
-  include_examples 'pg_search', VectorModelWithTagValues
-  include_examples 'pg_search', DynamicModel
-  include_examples 'pg_search', DynamicModelWithTagValues
-  include_examples 'pg_search', DynamicModelWithCategory
-  include_examples 'pg_search', DynamicModelWithSectionsTrhough
-
-  describe 'pg_search' do
-    describe 'joins' do
-      it 'can dynamically query compound relation' do
-        record = DynamicModelWithCategory.create(name: 'something', value: 'amazing')
-        category = Category.create(name: 'searchable')
-        Tag.create(value: 'impressive', category: category, taggable: record)
-        expect(DynamicModelWithCategory.scope_search('searchable')).to include(record)
-      end
-
-      it 'can use has_many :through relation' do
-        record = DynamicModelWithSectionsTrhough.create(name: 'something', value: 'amazing')
-        tag = Tag.create(value: 'impressive', taggable: record)
-        Section.create(name: 'searchable', tag: tag)
-        expect(DynamicModelWithSectionsTrhough.scope_search('searchable')).to include(record)
-      end
-    end
-
-    describe 'properties' do
-      describe 'skip_callback' do
-        context 'when enabled' do
-          let(:record) { VectorModel.create(name: 'something', value: 'amazing') }
-
-          it 'can find the record after it updates' do
-            record.update(name: 'cookie')
-            expect(VectorModel.scope_search('cookie')).to include(record)
-          end
-        end
-
-        context 'when disabled' do
-          let(:record) { VectorModelWithoutCallback.create(name: 'something', value: 'amazing') }
-
-          it 'cannot find the record after it updates' do
-            record.update(name: 'cookie')
-            expect(VectorModelWithoutCallback.scope_search('cookie')).not_to include(record)
-          end
-
-          it 'can find the record after manually calling .update_pg_search_cache' do
-            record.update(name: 'cookie')
-            record.update_pg_search_cache
-            expect(VectorModelWithoutCallback.scope_search('cookie')).to include(record)
-          end
-        end
-      end
-
-      describe 'scope' do
-        it 'defaults to "scope_search"' do
-          expect(VectorModel).to respond_to(:scope_search)
-        end
-
-        it 'can use a different scope name' do
-          expect(VectorModelWithCustomSearchScope).to respond_to(:fulltext)
-        end
-
-        it 'doesnt pollutes the default method name if customized' do
-          expect(VectorModelWithCustomSearchScope).not_to respond_to(:scope_search)
-        end
-      end
-
-      describe 'language' do
-        it 'defaults to english lexemes' do
-          record = VectorModel.create name: 'something', value: 'amazing'
-          expect(VectorModel.scope_search('amaz')).to include(record)
-        end
-
-        it 'can be changed to simple to avoid lexeme truncation' do
-          record = SimpleVectorModel.create name: 'something', value: 'amazing'
-          expect(SimpleVectorModel.scope_search('amazings')).not_to include(record)
-        end
-      end
-
-      describe 'wildcard' do
-        it 'by default uses it' do
-          record = VectorModel.create name: '12345', value: 'amazing'
-          expect(VectorModel.scope_search('123')).to include(record)
-        end
-
-        it 'can be set to false' do
-          record = VectorWithoutWildcardModel.create name: '12345', value: 'amazing'
-          expect(VectorWithoutWildcardModel.scope_search('123')).not_to include(record)
-        end
-      end
-
-      describe 'tags' do
-        it 'allow indexing fields of other associations' do
-          record = DynamicModelWithTagValues.create name: 'something', value: 'amazing'
-          Tag.create(taggable: record, value: 'red')
-          expect(DynamicModelWithTagValues.scope_search('red')).to include(record)
-        end
-      end
-
-      describe 'external_cache_data' do
-        it 'can index external data using a method' do
-          record = VectorModelWithTagValues.create name: 'something', value: 'amazing'
-          Tag.create(taggable: record, value: 'red')
-          expect(VectorModelWithTagValues.scope_search('red')).to include(record)
-        end
-      end
-    end
-  end
-end

spec/lib/text_to_tsquery_spec.rb

@@ -1,82 +0,0 @@
-# frozen_string_literal: true
-require_relative '../../lib/text_to_tsquery'
-
-
-describe TextToTsquery do
-  describe '.new' do
-    # partial match
-    it { expect(described_class.new('A').tsquery).to eq('A:*') }
-    it { expect(described_class.new('A', wildcard: false).tsquery).to eq('A:') }
-    it { expect(described_class.new('  A  ').tsquery).to eq('A:*') }
-
-    # AND operations
-    it { expect(described_class.new('A B').tsquery).to eq('A:*&B:*') }
-    it { expect(described_class.new('A B C').tsquery).to eq('A:*&B:*&C:*') }
-    it { expect(described_class.new('A and B').tsquery).to eq('A:*&B:*') }
-    it { expect(described_class.new('A AND B').tsquery).to eq('A:*&B:*') }
-    it { expect(described_class.new('A & B').tsquery).to eq('A:*&B:*') }
-    it { expect(described_class.new('A && B').tsquery).to eq('A:*&B:*') }
-    it { expect(described_class.new('A & B && C and D AND E F').tsquery).to eq('A:*&B:*&C:*&D:*&E:*&F:*') }
-
-    # OR operations
-    it { expect(described_class.new('A or B').tsquery).to eq('A:*|B:*') }
-    it { expect(described_class.new('A or B', wildcard: false).tsquery).to eq('A:|B:') }
-    it { expect(described_class.new('A OR B').tsquery).to eq('A:*|B:*') }
-    it { expect(described_class.new('A OR B', wildcard: false).tsquery).to eq('A:|B:') }
-    it { expect(described_class.new('A | B').tsquery).to eq('A:*|B:*') }
-    it { expect(described_class.new('A | B', wildcard: false).tsquery).to eq('A:|B:') }
-    it { expect(described_class.new('A || B').tsquery).to eq('A:*|B:*') }
-    it { expect(described_class.new('A || B', wildcard: false).tsquery).to eq('A:|B:') }
-    it { expect(described_class.new('A or or B').tsquery).to eq('A:*|B:*') }
-    it { expect(described_class.new('A or or B', wildcard: false).tsquery).to eq('A:|B:') }
-    it { expect(described_class.new('A | B || C or D OR E').tsquery).to eq('A:*|B:*|C:*|D:*|E:*') }
-    it { expect(described_class.new('A | B || C or D OR E', wildcard: false).tsquery).to eq('A:|B:|C:|D:|E:') }
-
-    # () Precedence
-    it { expect(described_class.new('(A)').tsquery).to eq('(A:*)') }
-    it { expect(described_class.new('(A)', wildcard: false).tsquery).to eq('(A:)') }
-    it { expect(described_class.new('(A B)').tsquery).to eq('(A:*&B:*)') }
-    it { expect(described_class.new('(A B)', wildcard: false).tsquery).to eq('(A:&B:)') }
-    it { expect(described_class.new('A (B !C)').tsquery).to eq('A:*&(B:*&!C)') }
-    it { expect(described_class.new('A (B !C)', wildcard: false).tsquery).to eq('A:&(B:&!C)') }
-    it { expect(described_class.new('(A AND B) OR C').tsquery).to eq('(A:*&B:*)|C:*') }
-    it { expect(described_class.new('(A AND B) OR C', wildcard: false).tsquery).to eq('(A:&B:)|C:') }
-    it { expect(described_class.new('A AND (B OR C)').tsquery).to eq('A:*&(B:*|C:*)') }
-    it { expect(described_class.new('A AND (B OR C)', wildcard: false).tsquery).to eq('A:&(B:|C:)') }
-    it { expect(described_class.new('(A & B) || C').tsquery).to eq('(A:*&B:*)|C:*') }
-    it { expect(described_class.new('(A & B) || C', wildcard: false).tsquery).to eq('(A:&B:)|C:') }
-    it { expect(described_class.new('A && (B | C)').tsquery).to eq('A:*&(B:*|C:*)') }
-    it { expect(described_class.new('A && (B | C)', wildcard: false).tsquery).to eq('A:&(B:|C:)') }
-    it { expect(described_class.new('A && !D (B | C | !E)').tsquery).to eq('A:*&!D&(B:*|C:*|!E)') }
-    it { expect(described_class.new('A && !D (B | C | !E)', wildcard: false).tsquery).to eq('A:&!D&(B:|C:|!E)') }
-
-    # Exact Matches
-    it { expect(described_class.new('"A"').tsquery).to eq("'A'") }
-    it { expect(described_class.new('"A B"').tsquery).to eq("'A B'") }
-    it { expect(described_class.new('"A&B"').tsquery).to eq("'A&B'") }
-    it { expect(described_class.new('"-A|B"').tsquery).to eq("'-A|B'") }
-    it { expect(described_class.new('"A-B"').tsquery).to eq("'A-B'") }
-    it { expect(described_class.new('"A" B').tsquery).to eq("'A'&B:*") }
-    it { expect(described_class.new('"A" B', wildcard: false).tsquery).to eq("'A'&B:") }
-    it { expect(described_class.new('"A B" C').tsquery).to eq("'A B'&C:*") }
-    it { expect(described_class.new('"A B" C', wildcard: false).tsquery).to eq("'A B'&C:") }
-    it { expect(described_class.new('("A B" or C) and D').tsquery).to eq("('A B'|C:*)&D:*") }
-    it { expect(described_class.new('("A B" or C) and D', wildcard: false).tsquery).to eq("('A B'|C:)&D:") }
-
-    describe 'validations' do
-      it { expect { described_class.new('(') }.to raise_error(ArgumentError, /parenthesis/) }
-    end
-  end
-
-  describe '.valid_search_parenthesis?' do
-    it { expect(described_class.valid_search_parenthesis?('')).to eq true }
-    it { expect(described_class.valid_search_parenthesis?('()')).to eq true }
-    it { expect(described_class.valid_search_parenthesis?('()()')).to eq true }
-    it { expect(described_class.valid_search_parenthesis?('(()())')).to eq true }
-    it { expect(described_class.valid_search_parenthesis?('((())())')).to eq true }
-    it { expect(described_class.valid_search_parenthesis?('(')).to eq false }
-    it { expect(described_class.valid_search_parenthesis?(')(')).to eq false }
-    it { expect(described_class.valid_search_parenthesis?('())')).to eq false }
-    it { expect(described_class.valid_search_parenthesis?('((()())')).to eq false }
-  end
-end

version.rb

@@ -3,7 +3,7 @@
 module Version
   MAJOR = 1
   MINOR = 0
-  PATCH = 24
+  PATCH = 27
 
   def self.to_s
     [MAJOR, MINOR, PATCH].compact.join('.')