Initial commit

This commit is contained in:
2024-08-27 20:33:44 +02:00
commit 1f1832267d
14794 changed files with 1599592 additions and 0 deletions

View File

@@ -0,0 +1,208 @@
from datetime import date
from io import StringIO
from django.core import management
from wagtail.search.query import MATCH_ALL
from wagtail.search.tests.test_backends import BackendTests
from wagtail.test.search import models
class ElasticsearchCommonSearchBackendTests(BackendTests):
def test_search_with_spaces_only(self):
# Search for some space characters and hope it doesn't crash
results = self.backend.search(" ", models.Book)
# Queries are lazily evaluated, force it to run
list(results)
# Didn't crash, yay!
def test_filter_with_unsupported_lookup_type(self):
"""
Not all lookup types are supported by the Elasticsearch backends
"""
from wagtail.search.backends.base import FilterError
with self.assertRaises(FilterError):
list(
self.backend.search(
"Hello", models.Book.objects.filter(title__iregex="h(ea)llo")
)
)
def test_partial_search(self):
results = self.backend.autocomplete("Java", models.Book)
self.assertUnsortedListEqual(
[r.title for r in results],
["JavaScript: The Definitive Guide", "JavaScript: The good parts"],
)
def test_disabled_partial_search(self):
results = self.backend.search("Java", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
def test_disabled_partial_search_with_whole_term(self):
# Making sure that there isn't a different reason why the above test
# returned no results
results = self.backend.search("JavaScript", models.Book)
self.assertUnsortedListEqual(
[r.title for r in results],
["JavaScript: The Definitive Guide", "JavaScript: The good parts"],
)
def test_child_partial_search(self):
# Note: Expands to "Westeros". Which is in a field on Novel.setting
results = self.backend.autocomplete("Wes", models.Book)
self.assertUnsortedListEqual(
[r.title for r in results],
["A Game of Thrones", "A Storm of Swords", "A Clash of Kings"],
)
def test_ascii_folding(self):
book = models.Book.objects.create(
title="Ĥéllø", publication_date=date(2017, 10, 19), number_of_pages=1
)
index = self.backend.get_index_for_model(models.Book)
index.add_item(book)
index.refresh()
results = self.backend.autocomplete("Hello", models.Book)
self.assertUnsortedListEqual([r.title for r in results], ["Ĥéllø"])
def test_query_analyser(self):
# This is testing that fields that use edgengram_analyzer as their index analyser do not
# have it also as their query analyser
results = self.backend.search("JavaScript", models.Book)
self.assertUnsortedListEqual(
[r.title for r in results],
["JavaScript: The Definitive Guide", "JavaScript: The good parts"],
)
# Even though they both start with "Java", this should not match the "JavaScript" books
results = self.backend.search("JavaBeans", models.Book)
self.assertSetEqual({r.title for r in results}, set())
def test_search_with_hyphen(self):
"""
This tests that punctuation characters are treated the same
way in both indexing and querying.
See: https://github.com/wagtail/wagtail/issues/937
"""
book = models.Book.objects.create(
title="Harry Potter and the Half-Blood Prince",
publication_date=date(2009, 7, 15),
number_of_pages=607,
)
index = self.backend.get_index_for_model(models.Book)
index.add_item(book)
index.refresh()
results = self.backend.search("Half-Blood", models.Book)
self.assertUnsortedListEqual(
[r.title for r in results],
[
"Harry Potter and the Half-Blood Prince",
],
)
def test_and_operator_with_single_field(self):
# Testing for bug #1859
results = self.backend.search(
"JavaScript", models.Book, operator="and", fields=["title"]
)
self.assertUnsortedListEqual(
[r.title for r in results],
["JavaScript: The Definitive Guide", "JavaScript: The good parts"],
)
def test_update_index_command_schema_only(self):
management.call_command(
"update_index",
backend_name=self.backend_name,
schema_only=True,
stdout=StringIO(),
)
# This should not give any results
results = self.backend.search(MATCH_ALL, models.Book)
self.assertSetEqual(set(results), set())
def test_more_than_ten_results(self):
# #3431 reported that Elasticsearch only sends back 10 results if the results set is not sliced
results = self.backend.search(MATCH_ALL, models.Book)
self.assertEqual(len(results), 14)
def test_more_than_one_hundred_results(self):
# Tests that fetching more than 100 results uses the scroll API
books = []
for i in range(150):
books.append(
models.Book.objects.create(
title=f"Book {i}",
publication_date=date(2017, 10, 21),
number_of_pages=i,
)
)
index = self.backend.get_index_for_model(models.Book)
index.add_items(models.Book, books)
index.refresh()
results = self.backend.search(MATCH_ALL, models.Book)
self.assertEqual(len(results), 164)
def test_slice_more_than_one_hundred_results(self):
books = []
for i in range(150):
books.append(
models.Book.objects.create(
title=f"Book {i}",
publication_date=date(2017, 10, 21),
number_of_pages=i,
)
)
index = self.backend.get_index_for_model(models.Book)
index.add_items(models.Book, books)
index.refresh()
results = self.backend.search(MATCH_ALL, models.Book)[10:120]
self.assertEqual(len(results), 110)
def test_slice_to_next_page(self):
# ES scroll API doesn't support offset. The implementation has an optimisation
# which will skip the first page if the first result is on the second page
books = []
for i in range(150):
books.append(
models.Book.objects.create(
title=f"Book {i}",
publication_date=date(2017, 10, 21),
number_of_pages=i,
)
)
index = self.backend.get_index_for_model(models.Book)
index.add_items(models.Book, books)
index.refresh()
results = self.backend.search(MATCH_ALL, models.Book)[110:]
self.assertEqual(len(results), 54)
def test_cannot_filter_on_date_parts_other_than_year(self):
# Filtering by date not supported, should throw a FilterError
from wagtail.search.backends.base import FilterError
in_jan = models.Book.objects.filter(publication_date__month=1)
with self.assertRaises(FilterError):
self.backend.search(MATCH_ALL, in_jan)

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,62 @@
import unittest
from django.test import TestCase
from django.test.utils import override_settings
from .test_backends import BackendTests
@override_settings(
WAGTAILSEARCH_BACKENDS={
"default": {
"BACKEND": "wagtail.search.backends.database.fallback",
}
}
)
class TestDBBackend(BackendTests, TestCase):
backend_path = "wagtail.search.backends.database.fallback"
# Doesn't support ranking
@unittest.expectedFailure
def test_ranking(self):
super().test_ranking()
# Doesn't support ranking
@unittest.expectedFailure
def test_annotate_score(self):
super().test_annotate_score()
# Doesn't support ranking
@unittest.expectedFailure
def test_annotate_score_with_slice(self):
super().test_annotate_score_with_slice()
# Doesn't support ranking
@unittest.expectedFailure
def test_search_boosting_on_related_fields(self):
super().test_search_boosting_on_related_fields()
# Doesn't support searching specific fields
@unittest.expectedFailure
def test_search_child_class_field_from_parent(self):
super().test_search_child_class_field_from_parent()
# Doesn't support searching related fields
@unittest.expectedFailure
def test_search_on_related_fields(self):
super().test_search_on_related_fields()
# Doesn't support searching callable fields
@unittest.expectedFailure
def test_search_callable_field(self):
super().test_search_callable_field()
# Database backend always uses `icontains`, so always autocomplete
@unittest.expectedFailure
def test_incomplete_plain_text(self):
super().test_incomplete_plain_text()
# Database backend doesn't support Boost() query class
@unittest.expectedFailure
def test_boost(self):
super().test_boost()

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,15 @@
import unittest
from django.test import TestCase
from .elasticsearch_common_tests import ElasticsearchCommonSearchBackendTests
try:
from elasticsearch import VERSION as ELASTICSEARCH_VERSION
except ImportError:
ELASTICSEARCH_VERSION = (0, 0, 0)
@unittest.skipIf(ELASTICSEARCH_VERSION[0] != 8, "Elasticsearch 8 required")
class TestElasticsearch8SearchBackend(ElasticsearchCommonSearchBackendTests, TestCase):
backend_path = "wagtail.search.backends.elasticsearch8"

View File

@@ -0,0 +1,226 @@
from datetime import date
from unittest import mock
from django.test import TestCase, override_settings
from wagtail.models import Page
from wagtail.search import index
from wagtail.test.search import models
from wagtail.test.testapp.models import SimplePage
from wagtail.test.utils import WagtailTestUtils
class TestGetIndexedInstance(TestCase):
fixtures = ["search"]
def test_gets_instance(self):
obj = models.Author.objects.get(id=1)
# Should just return the object
indexed_instance = index.get_indexed_instance(obj)
self.assertEqual(indexed_instance, obj)
def test_gets_specific_class(self):
obj = models.Novel.objects.get(id=1)
# Running the command with the parent class should find the specific class again
indexed_instance = index.get_indexed_instance(obj.book_ptr)
self.assertEqual(indexed_instance, obj)
def test_blocks_not_in_indexed_objects(self):
obj = models.Novel(
title="Don't index me!",
publication_date=date(2017, 10, 18),
number_of_pages=100,
)
obj.save()
# We've told it not to index anything with the title "Don't index me"
# get_indexed_instance should return None
indexed_instance = index.get_indexed_instance(obj.book_ptr)
self.assertIsNone(indexed_instance)
@mock.patch("wagtail.search.tests.DummySearchBackend", create=True)
@override_settings(
WAGTAILSEARCH_BACKENDS={
"default": {"BACKEND": "wagtail.search.tests.DummySearchBackend"}
}
)
class TestInsertOrUpdateObject(WagtailTestUtils, TestCase):
def test_inserts_object(self, backend):
obj = models.Book.objects.create(
title="Test", publication_date=date(2017, 10, 18), number_of_pages=100
)
backend().reset_mock()
index.insert_or_update_object(obj)
backend().add.assert_called_with(obj)
def test_doesnt_insert_unsaved_object(self, backend):
obj = models.Book(
title="Test", publication_date=date(2017, 10, 18), number_of_pages=100
)
backend().reset_mock()
index.insert_or_update_object(obj)
self.assertFalse(backend().add.mock_calls)
def test_converts_to_specific_page(self, backend):
root_page = Page.objects.get(id=1)
page = root_page.add_child(
instance=SimplePage(title="test", slug="test", content="test")
)
# Convert page into a generic "Page" object and add it into the index
unspecific_page = page.page_ptr
backend().reset_mock()
index.insert_or_update_object(unspecific_page)
# It should be automatically converted back to the specific version
backend().add.assert_called_with(page)
def test_catches_index_error(self, backend):
obj = models.Book.objects.create(
title="Test", publication_date=date(2017, 10, 18), number_of_pages=100
)
backend().add.side_effect = ValueError("Test")
backend().reset_mock()
with self.assertLogs("wagtail.search.index", level="ERROR") as cm:
index.insert_or_update_object(obj)
self.assertEqual(len(cm.output), 1)
self.assertIn(
"Exception raised while adding <Book: Test> into the 'default' search backend",
cm.output[0],
)
self.assertIn("Traceback (most recent call last):", cm.output[0])
self.assertIn("ValueError: Test", cm.output[0])
@mock.patch("wagtail.search.tests.DummySearchBackend", create=True)
@override_settings(
WAGTAILSEARCH_BACKENDS={
"default": {"BACKEND": "wagtail.search.tests.DummySearchBackend"}
}
)
class TestRemoveObject(WagtailTestUtils, TestCase):
def test_removes_object(self, backend):
obj = models.Book.objects.create(
title="Test", publication_date=date(2017, 10, 18), number_of_pages=100
)
backend().reset_mock()
index.remove_object(obj)
backend().delete.assert_called_with(obj)
def test_removes_unsaved_object(self, backend):
obj = models.Book(
title="Test", publication_date=date(2017, 10, 18), number_of_pages=100
)
backend().reset_mock()
index.remove_object(obj)
backend().delete.assert_called_with(obj)
def test_catches_index_error(self, backend):
obj = models.Book.objects.create(
title="Test", publication_date=date(2017, 10, 18), number_of_pages=100
)
backend().reset_mock()
backend().delete.side_effect = ValueError("Test")
with self.assertLogs("wagtail.search.index", level="ERROR") as cm:
index.remove_object(obj)
self.assertEqual(len(cm.output), 1)
self.assertIn(
"Exception raised while deleting <Book: Test> from the 'default' search backend",
cm.output[0],
)
self.assertIn("Traceback (most recent call last):", cm.output[0])
self.assertIn("ValueError: Test", cm.output[0])
@mock.patch("wagtail.search.tests.DummySearchBackend", create=True)
@override_settings(
WAGTAILSEARCH_BACKENDS={
"default": {"BACKEND": "wagtail.search.tests.DummySearchBackend"}
}
)
class TestSignalHandlers(WagtailTestUtils, TestCase):
def test_index_on_create(self, backend):
backend().reset_mock()
obj = models.Book.objects.create(
title="Test", publication_date=date(2017, 10, 18), number_of_pages=100
)
backend().add.assert_called_with(obj)
def test_index_on_update(self, backend):
obj = models.Book.objects.create(
title="Test", publication_date=date(2017, 10, 18), number_of_pages=100
)
backend().reset_mock()
obj.title = "Updated test"
obj.save()
self.assertEqual(backend().add.call_count, 1)
indexed_object = backend().add.call_args[0][0]
self.assertEqual(indexed_object.title, "Updated test")
def test_index_on_delete(self, backend):
obj = models.Book.objects.create(
title="Test", publication_date=date(2017, 10, 18), number_of_pages=100
)
backend().reset_mock()
obj.delete()
backend().delete.assert_called_with(obj)
def test_do_not_index_fields_omitted_from_update_fields(self, backend):
obj = models.Book.objects.create(
title="Test", publication_date=date(2017, 10, 18), number_of_pages=100
)
backend().reset_mock()
obj.title = "Updated test"
obj.publication_date = date(2001, 10, 19)
obj.save(update_fields=["title"])
self.assertEqual(backend().add.call_count, 1)
indexed_object = backend().add.call_args[0][0]
self.assertEqual(indexed_object.title, "Updated test")
self.assertEqual(indexed_object.publication_date, date(2017, 10, 18))
@mock.patch("wagtail.search.tests.DummySearchBackend", create=True)
@override_settings(
WAGTAILSEARCH_BACKENDS={
"default": {"BACKEND": "wagtail.search.tests.DummySearchBackend"}
}
)
class TestSignalHandlersSearchDisabled(TestCase, WagtailTestUtils):
def test_index_on_create_and_update(self, backend):
obj = models.UnindexedBook.objects.create(
title="Test", publication_date=date(2017, 10, 18), number_of_pages=100
)
self.assertEqual(backend().add.call_count, 0)
self.assertIsNone(backend().add.call_args)
backend().reset_mock()
obj.title = "Updated test"
obj.save()
self.assertEqual(backend().add.call_count, 0)
self.assertIsNone(backend().add.call_args)

View File

@@ -0,0 +1,157 @@
from contextlib import contextmanager
from django.core import checks
from django.test import TestCase
from wagtail.models import Page
from wagtail.search import index
from wagtail.test.search import models
from wagtail.test.testapp.models import (
TaggedChildPage,
TaggedGrandchildPage,
TaggedPage,
)
@contextmanager
def patch_search_fields(model, new_search_fields):
"""
A context manager to allow testing of different search_fields configurations
without permanently changing the models' search_fields.
"""
old_search_fields = model.search_fields
model.search_fields = new_search_fields
yield
model.search_fields = old_search_fields
class TestContentTypeNames(TestCase):
def test_base_content_type_name(self):
name = models.Novel.indexed_get_toplevel_content_type()
self.assertEqual(name, "searchtests_book")
def test_qualified_content_type_name(self):
name = models.Novel.indexed_get_content_type()
self.assertEqual(name, "searchtests_book_searchtests_novel")
class TestSearchFields(TestCase):
def make_dummy_type(self, search_fields):
return type("DummyType", (index.Indexed,), {"search_fields": search_fields})
def get_checks_result(warning_id=None):
"""Run Django checks on any with the 'search' tag used when registering the check"""
checks_result = checks.run_checks()
if warning_id:
return [warning for warning in checks_result if warning.id == warning_id]
return checks_result
def test_basic(self):
cls = self.make_dummy_type(
[
index.SearchField("test", boost=100),
index.FilterField("filter_test"),
]
)
self.assertEqual(len(cls.get_search_fields()), 2)
self.assertEqual(len(cls.get_searchable_search_fields()), 1)
self.assertEqual(len(cls.get_filterable_search_fields()), 1)
def test_overriding(self):
# If there are two fields with the same type and name
# the last one should override all the previous ones. This ensures that the
# standard convention of:
#
# class SpecificPageType(Page):
# search_fields = Page.search_fields + [some_other_definitions]
#
# ...causes the definitions in some_other_definitions to override Page.search_fields
# as intended.
cls = self.make_dummy_type(
[
index.SearchField("test", boost=100),
index.SearchField("test"),
]
)
self.assertEqual(len(cls.get_search_fields()), 1)
self.assertEqual(len(cls.get_searchable_search_fields()), 1)
self.assertEqual(len(cls.get_filterable_search_fields()), 0)
field = cls.get_search_fields()[0]
self.assertIsInstance(field, index.SearchField)
# Boost should be reset to the default if it's not specified by the override
self.assertIsNone(field.boost)
def test_different_field_types_dont_override(self):
# A search and filter field with the same name should be able to coexist
cls = self.make_dummy_type(
[
index.SearchField("test", boost=100),
index.FilterField("test"),
]
)
self.assertEqual(len(cls.get_search_fields()), 2)
self.assertEqual(len(cls.get_searchable_search_fields()), 1)
self.assertEqual(len(cls.get_filterable_search_fields()), 1)
def test_checking_search_fields(self):
with patch_search_fields(
models.Book, models.Book.search_fields + [index.SearchField("foo")]
):
expected_errors = [
checks.Warning(
"Book.search_fields contains non-existent field 'foo'",
obj=models.Book,
id="wagtailsearch.W004",
)
]
errors = models.Book.check()
self.assertEqual(errors, expected_errors)
def test_checking_core_page_fields_are_indexed(self):
"""Run checks to ensure that when core page fields are missing we get a warning"""
# first confirm that errors show as TaggedPage (in test models) has no Page.search_fields
errors = [
error for error in checks.run_checks() if error.id == "wagtailsearch.W001"
]
# should only ever get this warning on the sub-classes of the page model
self.assertEqual(
[TaggedPage, TaggedChildPage, TaggedGrandchildPage],
[error.obj for error in errors],
)
for error in errors:
self.assertEqual(
error.msg,
"Core Page fields missing in `search_fields`",
)
self.assertIn(
"Page model search fields `search_fields = Page.search_fields + [...]`",
error.hint,
)
# second check that we get no errors when setting up the models correctly
with patch_search_fields(
TaggedPage, Page.search_fields + TaggedPage.search_fields
):
errors = [
error
for error in checks.run_checks()
if error.id == "wagtailsearch.W001"
]
self.assertEqual([], errors)
# third check that we get no errors when disabling all model search
with patch_search_fields(TaggedPage, []):
errors = [
error
for error in checks.run_checks()
if error.id == "wagtailsearch.W001"
]
self.assertEqual([], errors)

View File

@@ -0,0 +1,95 @@
import unittest
from unittest import skip
from django.db import connection
from django.test.testcases import TransactionTestCase
from django.test.utils import override_settings
from wagtail.search.query import Not, PlainText
from wagtail.search.tests.test_backends import BackendTests
from wagtail.test.search import models
@unittest.skipUnless(connection.vendor == "mysql", "The current database is not MySQL")
@override_settings(
WAGTAILSEARCH_BACKENDS={
"default": {
"BACKEND": "wagtail.search.backends.database.mysql.mysql",
}
}
)
class TestMySQLSearchBackend(BackendTests, TransactionTestCase):
backend_path = "wagtail.search.backends.database.mysql.mysql"
# Overrides parent method, because there's a slight difference in what the MySQL backend supports/accepts as search queries.
def test_not(self):
all_other_titles = {
"A Clash of Kings",
"A Game of Thrones",
"A Storm of Swords",
"Foundation",
"Learning Python",
"The Hobbit",
"The Two Towers",
"The Fellowship of the Ring",
"The Return of the King",
"The Rust Programming Language",
"Two Scoops of Django 1.11",
"Programming Rust",
}
results = self.backend.search(
Not(PlainText("javascript")), models.Book.objects.all()
)
self.assertSetEqual({r.title for r in results}, all_other_titles)
results = self.backend.search(
~PlainText("javascript"), models.Book.objects.all()
)
self.assertSetEqual({r.title for r in results}, all_other_titles)
# Tests multiple words
results = self.backend.search(
~PlainText("javascript the"), models.Book.objects.all()
)
# NOTE: The difference with the parent method is here. As we're querying NOT 'javascript the', all entries containing both words should be excluded, but MySQL doesn't index stopwords in FULLTEXT indexes by default, so the JavaScript books won't match the query, since the 'the' word is excluded from the index. Therefore, both books will get returned.
self.assertSetEqual(
{r.title for r in results},
all_other_titles
| {"JavaScript: The Definitive Guide", "JavaScript: The good parts"},
)
# Tests multiple words too, but this time the second word is not a stopword
results = self.backend.search(
~PlainText("javascript parts"), models.Book.objects.all()
)
self.assertSetEqual(
{r.title for r in results},
all_other_titles | {"JavaScript: The Definitive Guide"},
)
@skip(
"The MySQL backend doesn't support choosing individual fields for the search, only (body, title) or (autocomplete) fields may be searched."
)
def test_search_on_individual_field(self):
return super().test_search_on_individual_field()
@skip("The MySQL backend doesn't support boosting.")
def test_search_boosting_on_related_fields(self):
return super().test_search_boosting_on_related_fields()
@skip("The MySQL backend doesn't support boosting.")
def test_boost(self):
return super().test_boost()
@skip("The MySQL backend doesn't score annotations.")
def test_annotate_score(self):
return super().test_annotate_score()
@skip("The MySQL backend doesn't score annotations.")
def test_annotate_score_with_slice(self):
return super().test_annotate_score_with_slice()
@skip("The MySQL backend doesn't guarantee correct ranking of results.")
def test_ranking(self):
return super().test_ranking()

View File

@@ -0,0 +1,74 @@
from django.conf import settings
from django.test import TestCase
from wagtail.models import Page
from wagtail.search.backends import get_search_backend
from wagtail.search.backends.base import BaseSearchQueryCompiler, BaseSearchResults
class PageSearchTests:
# A TestCase with this class mixed in will be dynamically created
# for each search backend defined in WAGTAILSEARCH_BACKENDS, with the backend name available
# as self.backend_name
fixtures = ["test.json"]
def setUp(self):
self.backend = get_search_backend(self.backend_name)
self.reset_index()
for page in Page.objects.all():
self.backend.add(page)
self.refresh_index()
def reset_index(self):
if self.backend.rebuilder_class:
index = self.backend.get_index_for_model(Page)
rebuilder = self.backend.rebuilder_class(index)
index = rebuilder.start()
index.add_model(Page)
rebuilder.finish()
def refresh_index(self):
index = self.backend.get_index_for_model(Page)
if index:
index.refresh()
def test_order_by_title(self):
list(
Page.objects.order_by("title").search(
"blah", order_by_relevance=False, backend=self.backend_name
)
)
def test_search_specific_queryset(self):
list(Page.objects.specific().search("bread", backend=self.backend_name))
def test_search_specific_queryset_with_fields(self):
list(
Page.objects.specific().search(
"bread", fields=["title"], backend=self.backend_name
)
)
for backend_name in settings.WAGTAILSEARCH_BACKENDS.keys():
test_name = str("Test%sBackend" % backend_name.title())
globals()[test_name] = type(
test_name,
(
PageSearchTests,
TestCase,
),
{"backend_name": backend_name},
)
class TestBaseSearchResults(TestCase):
def test_get_item_no_results(self):
# Ensure that, if there are no results, we do not attempt to get the entire search index.
base_search_results = BaseSearchResults(
"BackendIrrelevant", BaseSearchQueryCompiler
)
obj = base_search_results[0:0]
self.assertEqual(obj.start, 0)
self.assertEqual(obj.stop, 0)

View File

@@ -0,0 +1,224 @@
import unittest
from django.db import connection
from django.test import TestCase
from django.test.utils import override_settings
from wagtail.search.query import Phrase
from wagtail.search.tests.test_backends import BackendTests
from wagtail.test.search import models
@unittest.skipUnless(
connection.vendor == "postgresql", "The current database is not PostgreSQL"
)
@override_settings(
WAGTAILSEARCH_BACKENDS={
"default": {
"BACKEND": "wagtail.search.backends.database.postgres.postgres",
}
}
)
class TestPostgresSearchBackend(BackendTests, TestCase):
backend_path = "wagtail.search.backends.database.postgres.postgres"
def test_weights(self):
from ..backends.database.postgres.weights import (
BOOSTS_WEIGHTS,
WEIGHTS_VALUES,
determine_boosts_weights,
get_weight,
)
self.assertListEqual(
BOOSTS_WEIGHTS, [(10, "A"), (2, "B"), (0.5, "C"), (0.25, "D")]
)
self.assertListEqual(WEIGHTS_VALUES, [0.025, 0.05, 0.2, 1.0])
self.assertEqual(get_weight(15), "A")
self.assertEqual(get_weight(10), "A")
self.assertEqual(get_weight(9.9), "B")
self.assertEqual(get_weight(2), "B")
self.assertEqual(get_weight(1.9), "C")
self.assertEqual(get_weight(0), "D")
self.assertEqual(get_weight(-1), "D")
self.assertListEqual(
determine_boosts_weights([1]), [(1, "A"), (0, "B"), (0, "C"), (0, "D")]
)
self.assertListEqual(
determine_boosts_weights([-1]), [(-1, "A"), (-1, "B"), (-1, "C"), (-1, "D")]
)
self.assertListEqual(
determine_boosts_weights([-1, 1, 2]),
[(2, "A"), (1, "B"), (-1, "C"), (-1, "D")],
)
self.assertListEqual(
determine_boosts_weights([0, 1, 2, 3]),
[(3, "A"), (2, "B"), (1, "C"), (0, "D")],
)
self.assertListEqual(
determine_boosts_weights([0, 0.25, 0.75, 1, 1.5]),
[(1.5, "A"), (1, "B"), (0.5, "C"), (0, "D")],
)
self.assertListEqual(
determine_boosts_weights([0, 1, 2, 3, 4, 5, 6]),
[(6, "A"), (4, "B"), (2, "C"), (0, "D")],
)
self.assertListEqual(
determine_boosts_weights([-2, -1, 0, 1, 2, 3, 4]),
[(4, "A"), (2, "B"), (0, "C"), (-2, "D")],
)
def test_search_tsquery_chars(self):
"""
Checks that tsquery characters are correctly escaped
and do not generate a PostgreSQL syntax error.
"""
# Simple quote should be escaped inside each tsquery term.
results = self.backend.search("L'amour piqué par une abeille", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
results = self.backend.search("'starting quote", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
results = self.backend.search("ending quote'", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
results = self.backend.search("double quo''te", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
results = self.backend.search("triple quo'''te", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
# Now suffixes.
results = self.backend.search("Something:B", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
results = self.backend.search("Something:*", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
results = self.backend.search("Something:A*BCD", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
# Now the AND operator.
results = self.backend.search("first & second", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
# Now the OR operator.
results = self.backend.search("first | second", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
# Now the NOT operator.
results = self.backend.search("first & !second", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
# Now the phrase operator.
results = self.backend.search("first <-> second", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
def test_autocomplete_tsquery_chars(self):
"""
Checks that tsquery characters are correctly escaped
and do not generate a PostgreSQL syntax error.
"""
# Simple quote should be escaped inside each tsquery term.
results = self.backend.autocomplete(
"L'amour piqué par une abeille", models.Book
)
self.assertUnsortedListEqual([r.title for r in results], [])
results = self.backend.autocomplete("'starting quote", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
results = self.backend.autocomplete("ending quote'", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
results = self.backend.autocomplete("double quo''te", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
results = self.backend.autocomplete("triple quo'''te", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
# Backslashes should be escaped inside each tsquery term.
results = self.backend.autocomplete("backslash\\", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
# Now suffixes.
results = self.backend.autocomplete("Something:B", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
results = self.backend.autocomplete("Something:*", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
results = self.backend.autocomplete("Something:A*BCD", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
# Now the AND operator.
results = self.backend.autocomplete("first & second", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
# Now the OR operator.
results = self.backend.autocomplete("first | second", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
# Now the NOT operator.
results = self.backend.autocomplete("first & !second", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
# Now the phrase operator.
results = self.backend.autocomplete("first <-> second", models.Book)
self.assertUnsortedListEqual([r.title for r in results], [])
def test_index_without_upsert(self):
# Test the add_items code path for Postgres 9.4, where upsert is not available
self.backend.reset_index()
index = self.backend.get_index_for_model(models.Book)
index._enable_upsert = False
index.add_items(models.Book, models.Book.objects.all())
results = self.backend.search("JavaScript", models.Book)
self.assertUnsortedListEqual(
[r.title for r in results],
["JavaScript: The good parts", "JavaScript: The Definitive Guide"],
)
@unittest.skipUnless(
connection.vendor == "postgresql", "The current database is not PostgreSQL"
)
@override_settings(
WAGTAILSEARCH_BACKENDS={
"default": {
"BACKEND": "wagtail.search.backends.database.postgres.postgres",
"SEARCH_CONFIG": "dutch",
}
}
)
class TestPostgresLanguageTextSearch(TestCase):
backend_path = "wagtail.search.backends.database.postgres.postgres"
def setUp(self):
# get search backend by backend_path
BackendTests.setUp(self)
book = models.Book.objects.create(
title="Nu is beter dan nooit",
publication_date="1999-05-01",
number_of_pages=333,
)
self.backend.add(book)
self.book = book
def test_search_language_plain_text(self):
results = self.backend.search("Nu is beter dan nooit", models.Book)
self.assertEqual(list(results), [self.book])
results = self.backend.search("is beter", models.Book)
self.assertEqual(list(results), [self.book])
# search deals even with variations
results = self.backend.search("zijn beter", models.Book)
self.assertEqual(list(results), [self.book])
# search deals even when there are minor typos
results = self.backend.search("zij beter dan", models.Book)
self.assertEqual(list(results), [self.book])
def test_search_language_phrase_text(self):
results = self.backend.search(Phrase("Nu is beter"), models.Book)
self.assertEqual(list(results), [self.book])
results = self.backend.search(Phrase("Nu zijn beter"), models.Book)
self.assertEqual(list(results), [self.book])

View File

@@ -0,0 +1,332 @@
from django.test import SimpleTestCase, TestCase
from wagtail.search.query import And, Or, Phrase, PlainText
from wagtail.search.utils import (
balanced_reduce,
normalise_query_string,
parse_query_string,
separate_filters_from_query,
)
class TestQueryStringNormalisation(TestCase):
def test_truncation(self):
test_querystring = "a" * 1000
result = normalise_query_string(test_querystring)
self.assertEqual(len(result), 255)
def test_no_truncation(self):
test_querystring = "a" * 10
result = normalise_query_string(test_querystring)
self.assertEqual(len(result), 10)
class TestSeparateFiltersFromQuery(SimpleTestCase):
def test_only_query(self):
filters, query = separate_filters_from_query("hello world")
self.assertDictEqual(filters.dict(), {})
self.assertEqual(query, "hello world")
def test_filter(self):
filters, query = separate_filters_from_query("author:foo")
self.assertDictEqual(filters.dict(), {"author": "foo"})
self.assertEqual(query, "")
def test_filter_with_quotation_mark(self):
filters, query = separate_filters_from_query('author:"foo bar"')
self.assertDictEqual(filters.dict(), {"author": "foo bar"})
self.assertEqual(query, "")
def test_filter_and_query(self):
filters, query = separate_filters_from_query("author:foo hello world")
self.assertDictEqual(filters.dict(), {"author": "foo"})
self.assertEqual(query, "hello world")
def test_filter_with_quotation_mark_and_query(self):
filters, query = separate_filters_from_query('author:"foo bar" hello world')
self.assertDictEqual(filters.dict(), {"author": "foo bar"})
self.assertEqual(query, "hello world")
def test_filter_with_unclosed_quotation_mark_and_query(self):
filters, query = separate_filters_from_query('author:"foo bar hello world')
self.assertDictEqual(filters.dict(), {})
self.assertEqual(query, 'author:"foo bar hello world')
def test_two_filters_and_query(self):
filters, query = separate_filters_from_query(
'author:"foo bar" hello world bar:beer'
)
self.assertDictEqual(filters.dict(), {"author": "foo bar", "bar": "beer"})
self.assertEqual(query, "hello world")
def test_two_filters_with_quotation_marks_and_query(self):
filters, query = separate_filters_from_query(
'author:"foo bar" hello world bar:"two beers"'
)
self.assertDictEqual(filters.dict(), {"author": "foo bar", "bar": "two beers"})
self.assertEqual(query, "hello world")
filters, query = separate_filters_from_query(
"author:'foo bar' hello world bar:'two beers'"
)
self.assertDictEqual(filters.dict(), {"author": "foo bar", "bar": "two beers"})
self.assertEqual(query, "hello world")
def test_return_list_of_multiple_instances_for_same_filter_key(self):
filters, query = separate_filters_from_query(
'foo:test1 hello world foo:test2 foo:"test3" foo2:test4'
)
self.assertDictEqual(filters.dict(), {"foo": "test3", "foo2": "test4"})
self.assertListEqual(filters.getlist("foo"), ["test1", "test2", "test3"])
self.assertEqual(query, "hello world")
class TestParseQueryString(SimpleTestCase):
def test_simple_query(self):
filters, query = parse_query_string("hello world")
self.assertDictEqual(filters.dict(), {})
self.assertEqual(repr(query), repr(PlainText("hello world")))
def test_with_phrase(self):
filters, query = parse_query_string('"hello world"')
self.assertDictEqual(filters.dict(), {})
self.assertEqual(repr(query), repr(Phrase("hello world")))
filters, query = parse_query_string("'hello world'")
self.assertDictEqual(filters.dict(), {})
self.assertEqual(repr(query), repr(Phrase("hello world")))
def test_with_simple_and_phrase(self):
filters, query = parse_query_string('this is simple "hello world"')
self.assertDictEqual(filters.dict(), {})
self.assertEqual(
repr(query), repr(And([PlainText("this is simple"), Phrase("hello world")]))
)
filters, query = parse_query_string("this is simple 'hello world'")
self.assertDictEqual(filters.dict(), {})
self.assertEqual(
repr(query), repr(And([PlainText("this is simple"), Phrase("hello world")]))
)
def test_operator(self):
filters, query = parse_query_string(
'this is simple "hello world"', operator="or"
)
self.assertDictEqual(filters.dict(), {})
self.assertEqual(
repr(query),
repr(
Or([PlainText("this is simple", operator="or"), Phrase("hello world")])
),
)
filters, query = parse_query_string(
"this is simple 'hello world'", operator="or"
)
self.assertDictEqual(filters.dict(), {})
self.assertEqual(
repr(query),
repr(
Or([PlainText("this is simple", operator="or"), Phrase("hello world")])
),
)
def test_with_phrase_unclosed(self):
filters, query = parse_query_string('"hello world')
self.assertDictEqual(filters.dict(), {})
self.assertEqual(repr(query), repr(Phrase("hello world")))
filters, query = parse_query_string("'hello world")
self.assertDictEqual(filters.dict(), {})
self.assertEqual(repr(query), repr(Phrase("hello world")))
def test_phrase_with_filter(self):
filters, query = parse_query_string('"hello world" author:"foo bar" bar:beer')
self.assertDictEqual(filters.dict(), {"author": "foo bar", "bar": "beer"})
self.assertEqual(repr(query), repr(Phrase("hello world")))
filters, query = parse_query_string("'hello world' author:'foo bar' bar:beer")
self.assertDictEqual(filters.dict(), {"author": "foo bar", "bar": "beer"})
self.assertEqual(repr(query), repr(Phrase("hello world")))
def test_long_queries(self):
filters, query = parse_query_string("0" * 60_000)
self.assertEqual(filters.dict(), {})
self.assertEqual(repr(query), repr(PlainText("0" * 60_000)))
filters, _ = parse_query_string(f'{"a" * 60_000}:"foo bar"')
self.assertEqual(filters.dict(), {"a" * 60_000: "foo bar"})
def test_long_filter_value(self):
filters, _ = parse_query_string(f'foo:ba{"r" * 60_000}')
self.assertEqual(filters.dict(), {"foo": f'ba{"r" * 60_000}'})
def test_joined_filters(self):
filters, query = parse_query_string("foo:bar:baz")
self.assertEqual(filters.dict(), {"foo": "bar"})
self.assertEqual(repr(query), repr(PlainText(":baz")))
filters, query = parse_query_string("foo:'bar':baz")
self.assertEqual(filters.dict(), {"foo": "bar"})
self.assertEqual(repr(query), repr(PlainText(":baz")))
filters, query = parse_query_string("foo:'bar:baz'")
self.assertEqual(filters.dict(), {"foo": "bar:baz"})
def test_multiple_phrases(self):
filters, query = parse_query_string('"hello world" "hi earth"')
self.assertEqual(
repr(query), repr(And([Phrase("hello world"), Phrase("hi earth")]))
)
filters, query = parse_query_string("'hello world' 'hi earth'")
self.assertEqual(
repr(query), repr(And([Phrase("hello world"), Phrase("hi earth")]))
)
def test_mixed_phrases_with_filters(self):
filters, query = parse_query_string(
""""lord of the rings" army_1:"elves" army_2:'humans'"""
)
self.assertDictEqual(filters.dict(), {"army_1": "elves", "army_2": "humans"})
self.assertEqual(
repr(query),
repr(Phrase("lord of the rings")),
)
class TestBalancedReduce(SimpleTestCase):
# For simple values, this should behave exactly the same as Pythons reduce()
# So I've copied its tests: https://github.com/python/cpython/blob/21cdb711e3b1975398c54141e519ead02670610e/Lib/test/test_functools.py#L771
def test_reduce(self):
class Squares:
def __init__(self, max):
self.max = max
self.sofar = []
def __len__(self):
return len(self.sofar)
def __getitem__(self, i):
if not 0 <= i < self.max:
raise IndexError
n = len(self.sofar)
while n <= i:
self.sofar.append(n * n)
n += 1
return self.sofar[i]
def add(x, y):
return x + y
self.assertEqual(balanced_reduce(add, ["a", "b", "c"], ""), "abc")
self.assertEqual(
balanced_reduce(add, [["a", "c"], [], ["d", "w"]], []), ["a", "c", "d", "w"]
)
self.assertEqual(balanced_reduce(lambda x, y: x * y, range(2, 8), 1), 5040)
self.assertEqual(
balanced_reduce(lambda x, y: x * y, range(2, 21), 1), 2432902008176640000
)
self.assertEqual(balanced_reduce(add, Squares(10)), 285)
self.assertEqual(balanced_reduce(add, Squares(10), 0), 285)
self.assertEqual(balanced_reduce(add, Squares(0), 0), 0)
self.assertRaises(TypeError, balanced_reduce)
self.assertRaises(TypeError, balanced_reduce, 42, 42)
self.assertRaises(TypeError, balanced_reduce, 42, 42, 42)
self.assertEqual(
balanced_reduce(42, "1"), "1"
) # func is never called with one item
self.assertEqual(
balanced_reduce(42, "", "1"), "1"
) # func is never called with one item
self.assertRaises(TypeError, balanced_reduce, 42, (42, 42))
self.assertRaises(
TypeError, balanced_reduce, add, []
) # arg 2 must not be empty sequence with no initial value
self.assertRaises(TypeError, balanced_reduce, add, "")
self.assertRaises(TypeError, balanced_reduce, add, ())
self.assertRaises(TypeError, balanced_reduce, add, object())
class TestFailingIter:
def __iter__(self):
raise RuntimeError
self.assertRaises(RuntimeError, balanced_reduce, add, TestFailingIter())
self.assertIsNone(balanced_reduce(add, [], None))
self.assertEqual(balanced_reduce(add, [], 42), 42)
class BadSeq:
def __getitem__(self, index):
raise ValueError
self.assertRaises(ValueError, balanced_reduce, 42, BadSeq())
# Test reduce()'s use of iterators.
def test_iterator_usage(self):
class SequenceClass:
def __init__(self, n):
self.n = n
def __getitem__(self, i):
if 0 <= i < self.n:
return i
else:
raise IndexError
from operator import add
self.assertEqual(balanced_reduce(add, SequenceClass(5)), 10)
self.assertEqual(balanced_reduce(add, SequenceClass(5), 42), 52)
self.assertRaises(TypeError, balanced_reduce, add, SequenceClass(0))
self.assertEqual(balanced_reduce(add, SequenceClass(0), 42), 42)
self.assertEqual(balanced_reduce(add, SequenceClass(1)), 0)
self.assertEqual(balanced_reduce(add, SequenceClass(1), 42), 42)
d = {"one": 1, "two": 2, "three": 3}
self.assertEqual(balanced_reduce(add, d), "".join(d.keys()))
# This test is specific to balanced_reduce
def test_is_balanced(self):
# Tests that balanced_reduce returns the object as a balanced tree
class CombinedNode:
def __init__(self, a, b):
self.a = a
self.b = b
def __repr__(self):
return f"({self.a} {self.b})"
self.assertEqual(
repr(
balanced_reduce(CombinedNode, ["A", "B", "C", "D", "E", "F", "G", "H"])
),
"(((A B) (C D)) ((E F) (G H)))",
# Note: functools.reduce will return '(((((((A B) C) D) E) F) G) H)'
)

View File

@@ -0,0 +1,102 @@
from django.test import TestCase
from wagtail.search import index
from wagtail.test.search.models import Book, Novel
from wagtail.test.testapp.models import Advert, ManyToManyBlogPage
class TestSelectOnQuerySet(TestCase):
def test_select_on_queryset_with_foreign_key(self):
fields = index.RelatedFields(
"protagonist",
[
index.SearchField("name"),
],
)
queryset = fields.select_on_queryset(Novel.objects.all())
# ForeignKey should be select_related
self.assertFalse(queryset._prefetch_related_lookups)
self.assertIn("protagonist", queryset.query.select_related)
def test_select_on_queryset_with_one_to_one(self):
fields = index.RelatedFields(
"book_ptr",
[
index.SearchField("title"),
],
)
queryset = fields.select_on_queryset(Novel.objects.all())
# OneToOneField should be select_related
self.assertFalse(queryset._prefetch_related_lookups)
self.assertIn("book_ptr", queryset.query.select_related)
def test_select_on_queryset_with_many_to_many(self):
fields = index.RelatedFields(
"adverts",
[
index.SearchField("title"),
],
)
queryset = fields.select_on_queryset(ManyToManyBlogPage.objects.all())
# ManyToManyField should be prefetch_related
self.assertIn("adverts", queryset._prefetch_related_lookups)
self.assertFalse(queryset.query.select_related)
def test_select_on_queryset_with_reverse_foreign_key(self):
fields = index.RelatedFields(
"categories", [index.RelatedFields("category", [index.SearchField("name")])]
)
queryset = fields.select_on_queryset(ManyToManyBlogPage.objects.all())
# reverse ForeignKey should be prefetch_related
self.assertIn("categories", queryset._prefetch_related_lookups)
self.assertFalse(queryset.query.select_related)
def test_select_on_queryset_with_reverse_one_to_one(self):
fields = index.RelatedFields(
"novel",
[
index.SearchField("subtitle"),
],
)
queryset = fields.select_on_queryset(Book.objects.all())
# reverse OneToOneField should be select_related
self.assertFalse(queryset._prefetch_related_lookups)
self.assertIn("novel", queryset.query.select_related)
def test_select_on_queryset_with_reverse_many_to_many(self):
fields = index.RelatedFields(
"manytomanyblogpage",
[
index.SearchField("title"),
],
)
queryset = fields.select_on_queryset(Advert.objects.all())
# reverse ManyToManyField should be prefetch_related
self.assertIn("manytomanyblogpage", queryset._prefetch_related_lookups)
self.assertFalse(queryset.query.select_related)
def test_select_on_queryset_with_taggable_manager(self):
fields = index.RelatedFields(
"tags",
[
index.SearchField("name"),
],
)
queryset = fields.select_on_queryset(Novel.objects.all())
# Tags should be prefetch_related
self.assertIn("tags", queryset._prefetch_related_lookups)
self.assertFalse(queryset.query.select_related)

View File

@@ -0,0 +1,52 @@
import sqlite3
import unittest
from unittest import skip
from django.db import connection
from django.test.testcases import TestCase
from django.test.utils import override_settings
from wagtail.search.backends.database.sqlite.utils import fts5_available
from wagtail.search.tests.test_backends import BackendTests
@unittest.skipUnless(
connection.vendor == "sqlite", "The current database is not SQLite"
)
@unittest.skipIf(
sqlite3.sqlite_version_info < (3, 19, 0), "This SQLite version is not supported"
)
@unittest.skipUnless(fts5_available(), "The SQLite fts5 extension is not available")
@override_settings(
WAGTAILSEARCH_BACKENDS={
"default": {
"BACKEND": "wagtail.search.backends.database.sqlite.sqlite",
}
}
)
class TestSQLiteSearchBackend(BackendTests, TestCase):
backend_path = "wagtail.search.backends.database.sqlite.sqlite"
@skip("The SQLite backend doesn't support boosting.")
def test_search_boosting_on_related_fields(self):
return super().test_search_boosting_on_related_fields()
@skip("The SQLite backend doesn't support boosting.")
def test_boost(self):
return super().test_boost()
@skip("The SQLite backend doesn't score annotations.")
def test_annotate_score(self):
return super().test_annotate_score()
@skip("The SQLite backend doesn't score annotations.")
def test_annotate_score_with_slice(self):
return super().test_annotate_score_with_slice()
@skip("The SQLite backend doesn't support searching on specified fields.")
def test_autocomplete_with_fields_arg(self):
return super().test_autocomplete_with_fields_arg()
@skip("The SQLite backend doesn't guarantee correct ranking of results.")
def test_ranking(self):
return super().test_ranking()