senaduka/old-gem
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Upstream sync

Browse Source
This commit was merged in pull request #1.
This commit is contained in:
Senad Uka
2020-04-16 10:04:13 +02:00
parent 3de6bd22d8
commit afbfdb87cd
9 changed files with 686 additions and 598 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
coverage/.last_run.json
View File

@@ -1,5 +1,5 @@
{
"result": {
"covered_percent": 66.92
"covered_percent": 78.1
}
}

430
coverage/.resultset.json
View File

@@ -9,69 +9,73 @@
1,
1,
null,
0,
0,
1,
1,
null,
0,
1,
13,
null,
null,
null,
null,
null,
1,
1,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
13,
13,
13,
13,
13,
13,
13,
13,
13,
13,
null,
null,
1,
13,
13,
24,
23,
null,
null,
null,
null,
1,
24,
null,
24,
null,
null,
1,
0,
null,
null,
1,
16,
null,
null,
1,
13,
null,
0,
null,
1,
0,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
null,
null,
0,
0,
0,
null,
null,
null,
0,
0,
0,
null,
null,
0,
0,
null,
null,
0,
0,
null,
null,
0,
0,
null,
null,
0,
0,
null,
null,
0,
1,
0,
0,
0,
@@ -181,57 +185,121 @@
null
],
"/home/hamo/projects/toptal/outfrontmedia/pg_searchable/lib/text_to_sql_query.rb": [
1,
null,
1,
1,
44,
44,
44,
44,
114,
114,
114,
null,
44,
13,
null,
null,
null,
1,
44,
44,
44,
null,
null,
1,
null,
1,
128,
128,
128,
null,
23,
23,
null,
22,
null,
15,
null,
10,
null,
10,
0,
null,
null,
10,
10,
null,
10,
null,
null,
null,
58,
58,
58,
2,
null,
56,
null,
null,
null,
null,
1,
37,
0,
null,
null,
37,
37,
null,
37,
0,
null,
null,
37,
0,
null,
null,
37,
37,
null,
37,
37,
null,
37,
null,
null,
1,
81,
81,
81,
81,
81,
null,
null
],
"/home/hamo/projects/toptal/outfrontmedia/pg_searchable/lib/parser.rb": [
null,
null,
null,
null,
null,
null,
1,
null,
1,
null,
1,
null,
1,
null,
null,
null,
null,
null,
null,
1,
null,
null,
null,
@@ -240,6 +308,7 @@
null,
null,
null,
1,
null,
null,
null,
@@ -248,22 +317,29 @@
null,
null,
null,
1,
null,
null,
null,
1,
null,
null,
null,
1,
null,
null,
null,
1,
null,
null,
null,
1,
null,
null,
1,
null,
null,
1,
null,
null,
null,
@@ -273,9 +349,255 @@
null,
null,
null,
null,
null,
null,
null,
1,
null,
1,
null,
1,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
1,
null,
1,
null,
1,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
1,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
1,
null,
null,
null,
null,
null,
null,
null,
1,
null,
null,
null,
null,
null,
null,
1,
null,
null,
null,
null,
null,
null,
1,
null,
null,
null,
null,
null,
null,
1,
null,
null,
null,
null,
null,
null,
1,
null,
null,
null,
null,
null,
null,
1,
null,
null,
null,
null,
null,
null,
1,
null,
null,
null,
null,
null,
null,
1,
null,
null,
null,
null,
null,
null,
1,
null,
null,
null,
null,
null,
null,
1,
null,
null,
null,
null,
null,
null,
1,
0,
null,
null,
null
],
"/home/hamo/projects/toptal/outfrontmedia/pg_searchable/lib/lexer.rb": [
null,
null,
null,
null,
null,
null,
1,
1,
1,
null,
1,
null,
1,
1,
1,
null,
1,
132,
132,
132,
null,
null,
1,
702,
null,
null,
1,
94,
94,
null,
1,
null,
1,
0,
0,
0,
null,
null,
null,
1,
0,
0,
null,
null,
null,
1,
834,
null,
null,
703,
703,
null,
null,
1,
939,
939,
939,
null,
null,
939,
null,
null,
702,
106,
null,
649,
106,
null,
596,
84,
null,
554,
84,
null,
512,
46,
null,
489,
132,
null,
423,
610,
null,
118,
236,
null,
null,
null,
0,
0,
939,
null,
null,
0,
null,
237,
null,
null,
1,
38,
38,
38,
174,
null,
38,
null,
null
]
},
"timestamp": 1580827025
"timestamp": 1583928139
}
}

568
coverage/index.html
View File

File diff suppressed because it is too large Load Diff

7
lib/pg_searchable_regex.rb
View File

@@ -42,14 +42,17 @@ module PgSearchable
def ts_add_scope
class_eval do
scope ts_scope_method, ->(value) { ts_search(value) }
scope ts_scope_method, ->(value) do
resulting_ids = ts_search(value).map(&:id)
where(id: resulting_ids)
end
end
end
def ts_search(value)
return if @ts_search_fields.blank? || value.blank?
includes(@ts_joins).references(:all).where(
TextToSqlQuery.new(value, @ts_search_fields, @default_field, @ts_search_fields_mappings).where_clause)
TextToSqlQuery.new(value, @ts_search_fields, @default_field, @ts_search_fields_mappings).where_clause).distinct
end
def should_update_cache_field?

67
lib/text_to_regex_query.rb
View File

@@ -1,67 +0,0 @@
# frozen_string_literal: true
# transforms "english like" text queries into a where clause with regex
# https://www.postgresql.org/docs/9.5/textsearch-controls.html#TEXTSEARCH-PARSING-QUERIES
class TextToRegexQuery
def initialize(text, fields, default_field, fields_mappings = {})
@text = text.to_s.strip
@fields = fields.map(&:to_sym)
@default_field = default_field.to_sym
@fields_mappings = fields_mappings.merge(@fields.reduce({}) do |mappings, field|
table_name, field_name = field.to_s.split(".")
mappings[field_name.to_sym] = field
mappings
end)
end
def where_clause(query)
@cleared_text = @text.dup
@column_chunks = []
remove_duplicated_spaces
extract_columns
escape_special_characters
generate_where_clause(query)
end
private
def remove_duplicated_spaces
@cleared_text.gsub!(/\s+/, ' ')
end
def escape_special_characters
@cleared_text.gsub!(/\_/, '\_')
@cleared_text.tr!('\\', '\\')
@cleared_text.gsub!(/%/, '\%')
end
def extract_columns
column_search_term_pairs = @cleared_text.scan(/([A-Za-z0-9_]+:[\w\_-]+)/)
@column_chunks = (column_search_term_pairs.flatten.map do |pair|
column, term = pair.split(':')
next unless @fields_mappings.include?(column.to_sym)
@cleared_text.gsub!(pair, '')
{ @fields_mappings[column.to_sym] => term }
end).compact
unless @cleared_text.strip.empty?
@column_chunks = [{ @default_field.to_s => @cleared_text.strip }] + @column_chunks
end
@column_chunks
end
def generate_where_clause(query)
where_clause = ''
columns = @column_chunks.map { |c| c.keys.first }
values = @column_chunks.map { |c| c.values.first }
columns.each do |column|
quoted_column = '"' + column.to_s.gsub(".",'"."') + '"'
where_clause += "#{quoted_column} ILIKE ? OR "
end
where_clause += " 1<>1 "
regexed_values = values.map { |v| "%#{v}%" }
query.where([where_clause] + regexed_values)
end
end

10
spec/lib/pg_searchable_new_spec.rb
View File

@@ -136,6 +136,16 @@ describe PgSearchable do
expect(DynamicModelWithTagValues.scope_search('tag:red value:"not"')).to contain_exactly(record1, record2)
end
it 'can find models without tags' do
record1 = DynamicModelWithTagValues.create name: 'something', value: 'amazing'
record2 = DynamicModelWithTagValues.create name: 'new record', value: 'not so amazing'
Tag.create(taggable: record1, value: 'green')
expect(DynamicModelWithTagValues.scope_search('tag:green or value:"not"')).to contain_exactly(record1, record2)
end
it 'can search in referenced column and in model columns with multiple search terms connected with logical operators' do
record1 = DynamicModelWithTagValues.create name: 'something', value: 'amazing'
record2 = DynamicModelWithTagValues.create name: 'new record', value: 'not so amazing'

116
spec/lib/pg_searchable_spec.rb
View File

@@ -1,116 +0,0 @@
# frozen_string_literal: true
require_relative '../../lib/pg_searchable_regex'
describe PgSearchable do
include_examples 'pg_search', VectorModel
include_examples 'pg_search', VectorWithCustomPrimaryKeyModel
include_examples 'pg_search', VectorWithCustomCallback
include_examples 'pg_search', SimpleVectorModel
include_examples 'pg_search', VectorWithoutWildcardModel
include_examples 'pg_search', VectorModelWithCustomSearchScope, 'fulltext'
include_examples 'pg_search', VectorModelWithTagValues
include_examples 'pg_search', DynamicModel
include_examples 'pg_search', DynamicModelWithTagValues
include_examples 'pg_search', DynamicModelWithCategory
include_examples 'pg_search', DynamicModelWithSectionsTrhough
describe 'pg_search' do
describe 'joins' do
it 'can dynamically query compound relation' do
record = DynamicModelWithCategory.create(name: 'something', value: 'amazing')
category = Category.create(name: 'searchable')
Tag.create(value: 'impressive', category: category, taggable: record)
expect(DynamicModelWithCategory.scope_search('searchable')).to include(record)
end
it 'can use has_many :through relation' do
record = DynamicModelWithSectionsTrhough.create(name: 'something', value: 'amazing')
tag = Tag.create(value: 'impressive', taggable: record)
Section.create(name: 'searchable', tag: tag)
expect(DynamicModelWithSectionsTrhough.scope_search('searchable')).to include(record)
end
end
describe 'properties' do
describe 'skip_callback' do
context 'when enabled' do
let(:record) { VectorModel.create(name: 'something', value: 'amazing') }
it 'can find the record after it updates' do
record.update(name: 'cookie')
expect(VectorModel.scope_search('cookie')).to include(record)
end
end
context 'when disabled' do
let(:record) { VectorModelWithoutCallback.create(name: 'something', value: 'amazing') }
it 'cannot find the record after it updates' do
record.update(name: 'cookie')
expect(VectorModelWithoutCallback.scope_search('cookie')).not_to include(record)
end
it 'can find the record after manually calling .update_pg_search_cache' do
record.update(name: 'cookie')
record.update_pg_search_cache
expect(VectorModelWithoutCallback.scope_search('cookie')).to include(record)
end
end
end
describe 'scope' do
it 'defaults to "scope_search"' do
expect(VectorModel).to respond_to(:scope_search)
end
it 'can use a different scope name' do
expect(VectorModelWithCustomSearchScope).to respond_to(:fulltext)
end
it 'doesnt pollutes the default method name if customized' do
expect(VectorModelWithCustomSearchScope).not_to respond_to(:scope_search)
end
end
describe 'language' do
it 'defaults to english lexemes' do
record = VectorModel.create name: 'something', value: 'amazing'
expect(VectorModel.scope_search('amaz')).to include(record)
end
it 'can be changed to simple to avoid lexeme truncation' do
record = SimpleVectorModel.create name: 'something', value: 'amazing'
expect(SimpleVectorModel.scope_search('amazings')).not_to include(record)
end
end
describe 'wildcard' do
it 'by default uses it' do
record = VectorModel.create name: '12345', value: 'amazing'
expect(VectorModel.scope_search('123')).to include(record)
end
it 'can be set to false' do
record = VectorWithoutWildcardModel.create name: '12345', value: 'amazing'
expect(VectorWithoutWildcardModel.scope_search('123')).not_to include(record)
end
end
describe 'tags' do
it 'allow indexing fields of other associations' do
record = DynamicModelWithTagValues.create name: 'something', value: 'amazing'
Tag.create(taggable: record, value: 'red')
expect(DynamicModelWithTagValues.scope_search('red')).to include(record)
end
end
describe 'external_cache_data' do
it 'can index external data using a method' do
record = VectorModelWithTagValues.create name: 'something', value: 'amazing'
Tag.create(taggable: record, value: 'red')
expect(VectorModelWithTagValues.scope_search('red')).to include(record)
end
end
end
end
end

82
spec/lib/text_to_tsquery_spec.rb
View File

@@ -1,82 +0,0 @@
# frozen_string_literal: true
require_relative '../../lib/text_to_tsquery'
describe TextToTsquery do
describe '.new' do
# partial match
it { expect(described_class.new('A').tsquery).to eq('A:*') }
it { expect(described_class.new('A', wildcard: false).tsquery).to eq('A:') }
it { expect(described_class.new(' A ').tsquery).to eq('A:*') }
# AND operations
it { expect(described_class.new('A B').tsquery).to eq('A:*&B:*') }
it { expect(described_class.new('A B C').tsquery).to eq('A:*&B:*&C:*') }
it { expect(described_class.new('A and B').tsquery).to eq('A:*&B:*') }
it { expect(described_class.new('A AND B').tsquery).to eq('A:*&B:*') }
it { expect(described_class.new('A & B').tsquery).to eq('A:*&B:*') }
it { expect(described_class.new('A && B').tsquery).to eq('A:*&B:*') }
it { expect(described_class.new('A & B && C and D AND E F').tsquery).to eq('A:*&B:*&C:*&D:*&E:*&F:*') }
# OR operations
it { expect(described_class.new('A or B').tsquery).to eq('A:*|B:*') }
it { expect(described_class.new('A or B', wildcard: false).tsquery).to eq('A:|B:') }
it { expect(described_class.new('A OR B').tsquery).to eq('A:*|B:*') }
it { expect(described_class.new('A OR B', wildcard: false).tsquery).to eq('A:|B:') }
it { expect(described_class.new('A | B').tsquery).to eq('A:*|B:*') }
it { expect(described_class.new('A | B', wildcard: false).tsquery).to eq('A:|B:') }
it { expect(described_class.new('A || B').tsquery).to eq('A:*|B:*') }
it { expect(described_class.new('A || B', wildcard: false).tsquery).to eq('A:|B:') }
it { expect(described_class.new('A or or B').tsquery).to eq('A:*|B:*') }
it { expect(described_class.new('A or or B', wildcard: false).tsquery).to eq('A:|B:') }
it { expect(described_class.new('A | B || C or D OR E').tsquery).to eq('A:*|B:*|C:*|D:*|E:*') }
it { expect(described_class.new('A | B || C or D OR E', wildcard: false).tsquery).to eq('A:|B:|C:|D:|E:') }
# () Precedence
it { expect(described_class.new('(A)').tsquery).to eq('(A:*)') }
it { expect(described_class.new('(A)', wildcard: false).tsquery).to eq('(A:)') }
it { expect(described_class.new('(A B)').tsquery).to eq('(A:*&B:*)') }
it { expect(described_class.new('(A B)', wildcard: false).tsquery).to eq('(A:&B:)') }
it { expect(described_class.new('A (B !C)').tsquery).to eq('A:*&(B:*&!C)') }
it { expect(described_class.new('A (B !C)', wildcard: false).tsquery).to eq('A:&(B:&!C)') }
it { expect(described_class.new('(A AND B) OR C').tsquery).to eq('(A:*&B:*)|C:*') }
it { expect(described_class.new('(A AND B) OR C', wildcard: false).tsquery).to eq('(A:&B:)|C:') }
it { expect(described_class.new('A AND (B OR C)').tsquery).to eq('A:*&(B:*|C:*)') }
it { expect(described_class.new('A AND (B OR C)', wildcard: false).tsquery).to eq('A:&(B:|C:)') }
it { expect(described_class.new('(A & B) || C').tsquery).to eq('(A:*&B:*)|C:*') }
it { expect(described_class.new('(A & B) || C', wildcard: false).tsquery).to eq('(A:&B:)|C:') }
it { expect(described_class.new('A && (B | C)').tsquery).to eq('A:*&(B:*|C:*)') }
it { expect(described_class.new('A && (B | C)', wildcard: false).tsquery).to eq('A:&(B:|C:)') }
it { expect(described_class.new('A && !D (B | C | !E)').tsquery).to eq('A:*&!D&(B:*|C:*|!E)') }
it { expect(described_class.new('A && !D (B | C | !E)', wildcard: false).tsquery).to eq('A:&!D&(B:|C:|!E)') }
# Exact Matches
it { expect(described_class.new('"A"').tsquery).to eq("'A'") }
it { expect(described_class.new('"A B"').tsquery).to eq("'A B'") }
it { expect(described_class.new('"A&B"').tsquery).to eq("'A&B'") }
it { expect(described_class.new('"-A|B"').tsquery).to eq("'-A|B'") }
it { expect(described_class.new('"A-B"').tsquery).to eq("'A-B'") }
it { expect(described_class.new('"A" B').tsquery).to eq("'A'&B:*") }
it { expect(described_class.new('"A" B', wildcard: false).tsquery).to eq("'A'&B:") }
it { expect(described_class.new('"A B" C').tsquery).to eq("'A B'&C:*") }
it { expect(described_class.new('"A B" C', wildcard: false).tsquery).to eq("'A B'&C:") }
it { expect(described_class.new('("A B" or C) and D').tsquery).to eq("('A B'|C:*)&D:*") }
it { expect(described_class.new('("A B" or C) and D', wildcard: false).tsquery).to eq("('A B'|C:)&D:") }
describe 'validations' do
it { expect { described_class.new('(') }.to raise_error(ArgumentError, /parenthesis/) }
end
end
describe '.valid_search_parenthesis?' do
it { expect(described_class.valid_search_parenthesis?('')).to eq true }
it { expect(described_class.valid_search_parenthesis?('()')).to eq true }
it { expect(described_class.valid_search_parenthesis?('()()')).to eq true }
it { expect(described_class.valid_search_parenthesis?('(()())')).to eq true }
it { expect(described_class.valid_search_parenthesis?('((())())')).to eq true }
it { expect(described_class.valid_search_parenthesis?('(')).to eq false }
it { expect(described_class.valid_search_parenthesis?(')(')).to eq false }
it { expect(described_class.valid_search_parenthesis?('())')).to eq false }
it { expect(described_class.valid_search_parenthesis?('((()())')).to eq false }
end
end

2
version.rb
View File

@@ -3,7 +3,7 @@
module Version
MAJOR = 1
MINOR = 0
PATCH = 24
PATCH = 27
def self.to_s
[MAJOR, MINOR, PATCH].compact.join('.')