Initial commit

This commit is contained in:
2024-08-27 20:33:44 +02:00
commit 1f1832267d
14794 changed files with 1599592 additions and 0 deletions

View File

@@ -0,0 +1,13 @@
from django.apps import AppConfig
from django.utils.translation import gettext_lazy as _
from wagtail.contrib.frontend_cache.signal_handlers import register_signal_handlers
class WagtailFrontendCacheAppConfig(AppConfig):
name = "wagtail.contrib.frontend_cache"
label = "wagtailfrontendcache"
verbose_name = _("Wagtail frontend cache")
def ready(self):
register_signal_handlers()

View File

@@ -0,0 +1,5 @@
from .base import * # noqa
from .azure import * # noqa
from .http import * # noqa
from .cloudflare import * # noqa
from .cloudfront import * # noqa

View File

@@ -0,0 +1,179 @@
import logging
from urllib.parse import urlsplit, urlunsplit
from django.core.exceptions import ImproperlyConfigured
from .base import BaseBackend
logger = logging.getLogger("wagtail.frontendcache")
__all__ = ["AzureBaseBackend", "AzureFrontDoorBackend", "AzureCdnBackend"]
class AzureBaseBackend(BaseBackend):
def __init__(self, params):
super().__init__(params)
self._credentials = params.pop("CREDENTIALS", None)
self._subscription_id = params.pop("SUBSCRIPTION_ID", None)
try:
self._resource_group_name = params.pop("RESOURCE_GROUP_NAME")
except KeyError:
raise ImproperlyConfigured(
"The setting 'WAGTAILFRONTENDCACHE' requires 'RESOURCE_GROUP_NAME' to be specified."
)
self._custom_headers = params.pop("CUSTOM_HEADERS", None)
def purge_batch(self, urls):
self._purge_content([self._get_path(url) for url in urls])
def purge(self, url):
self.purge_batch([url])
def _get_default_credentials(self):
try:
from azure.identity import DefaultAzureCredential
except ImportError:
return
return DefaultAzureCredential()
def _get_credentials(self):
"""
Use credentials object set by user. If not set, use the one configured
in the current environment.
"""
user_credentials = self._credentials
if user_credentials:
return user_credentials
return self._get_default_credentials()
def _get_default_subscription_id(self):
"""
Obtain subscription ID directly from Azure.
"""
try:
from azure.mgmt.resource import SubscriptionClient
except ImportError:
return ""
credential = self._get_credentials()
subscription_client = SubscriptionClient(credential)
subscription = next(subscription_client.subscriptions.list())
return subscription.subscription_id
def _get_subscription_id(self):
"""
Use subscription ID set in the user configuration. If not set, try to
retrieve one from Azure directly.
"""
user_subscription_id = self._subscription_id
if user_subscription_id:
return user_subscription_id
return self._get_default_subscription_id()
def _get_client_kwargs(self):
return {
"credential": self._get_credentials(),
"subscription_id": self._get_subscription_id(),
}
def _get_path(self, url):
"""
Split netloc from the URL and return path only.
"""
# Delete scheme and netloc from the URL, that will result in only path being
# left.
url_parts = ("",) * 2 + urlsplit(url)[2:]
return urlunsplit(url_parts)
def _get_client(self):
"""
Get Azure client instance.
"""
klass = self._get_client_class()
kwargs = self._get_client_kwargs()
return klass(**kwargs)
def _get_purge_kwargs(self, paths):
"""
Get keyword arguments passes to Azure purge content calls.
"""
return {
"resource_group_name": self._resource_group_name,
"custom_headers": self._custom_headers,
"content_paths": paths,
}
def _purge_content(self, paths):
from msrest.exceptions import HttpOperationError
client = self._get_client()
try:
self._make_purge_call(client, paths)
except HttpOperationError as exception:
for path in paths:
logger.exception(
"Couldn't purge '%s' from %s cache. HttpOperationError: %r",
path,
type(self).__name__,
exception.response,
)
class AzureFrontDoorBackend(AzureBaseBackend):
def __init__(self, params):
super().__init__(params)
try:
self._front_door_name = params.pop("FRONT_DOOR_NAME")
except KeyError:
raise ImproperlyConfigured(
"The setting 'WAGTAILFRONTENDCACHE' requires 'FRONT_DOOR_NAME' to be specified."
)
self._front_door_service_url = params.pop("FRONT_DOOR_SERVICE_URL", None)
def _get_client_class(self):
from azure.mgmt.frontdoor import FrontDoorManagementClient
return FrontDoorManagementClient
def _get_client_kwargs(self):
kwargs = super()._get_client_kwargs()
kwargs.setdefault("base_url", self._front_door_service_url)
return kwargs
def _make_purge_call(self, client, paths):
return client.endpoints.purge_content(
**self._get_purge_kwargs(paths),
front_door_name=self._front_door_name,
)
class AzureCdnBackend(AzureBaseBackend):
def __init__(self, params):
super().__init__(params)
try:
self._cdn_profile_name = params.pop("CDN_PROFILE_NAME")
self._cdn_endpoint_name = params.pop("CDN_ENDPOINT_NAME")
except KeyError:
raise ImproperlyConfigured(
"The setting 'WAGTAILFRONTENDCACHE' requires 'CDN_PROFILE_NAME' and 'CDN_ENDPOINT_NAME' to be specified."
)
self._cdn_service_url = params.pop("CDN_SERVICE_URL", None)
def _get_client_class(self):
from azure.mgmt.cdn import CdnManagementClient
return CdnManagementClient
def _get_client_kwargs(self):
kwargs = super()._get_client_kwargs()
kwargs.setdefault("base_url", self._cdn_service_url)
return kwargs
def _make_purge_call(self, client, paths):
return client.endpoints.purge_content(
**self._get_purge_kwargs(paths),
profile_name=self._cdn_profile_name,
endpoint_name=self._cdn_endpoint_name,
)

View File

@@ -0,0 +1,28 @@
import logging
from django.http.request import validate_host
logger = logging.getLogger("wagtail.frontendcache")
__all__ = ["BaseBackend"]
class BaseBackend:
def __init__(self, params):
# If unspecified, invalidate all hosts
self.hostnames = params.get("HOSTNAMES", ["*"])
def purge(self, url) -> None:
raise NotImplementedError
def purge_batch(self, urls) -> None:
# Fallback for backends that do not support batch purging
for url in urls:
self.purge(url)
def invalidates_hostname(self, hostname) -> bool:
"""
Can `hostname` be invalidated by this backend?
"""
return validate_host(hostname, self.hostnames)

View File

@@ -0,0 +1,114 @@
import logging
import requests
from django.core.exceptions import ImproperlyConfigured
from .base import BaseBackend
logger = logging.getLogger("wagtail.frontendcache")
__all__ = ["CloudflareBackend"]
class CloudflareBackend(BaseBackend):
CHUNK_SIZE = 30
def __init__(self, params):
super().__init__(params)
self.cloudflare_email = params.pop("EMAIL", None)
self.cloudflare_api_key = params.pop("TOKEN", None) or params.pop(
"API_KEY", None
)
self.cloudflare_token = params.pop("BEARER_TOKEN", None)
self.cloudflare_zoneid = params.pop("ZONEID")
self.cloudflare_purge_endpoint_url = (
"https://api.cloudflare.com/client/v4/zones/{}/purge_cache".format(
self.cloudflare_zoneid
)
)
if (
(not self.cloudflare_email and self.cloudflare_api_key)
or (self.cloudflare_email and not self.cloudflare_api_key)
or (
not any(
[
self.cloudflare_email,
self.cloudflare_api_key,
self.cloudflare_token,
]
)
)
):
raise ImproperlyConfigured(
"The setting 'WAGTAILFRONTENDCACHE' requires both 'EMAIL' and 'API_KEY', or 'BEARER_TOKEN' to be specified."
)
def _purge_urls(self, urls):
try:
purge_url = (
"https://api.cloudflare.com/client/v4/zones/{}/purge_cache".format(
self.cloudflare_zoneid
)
)
headers = {"Content-Type": "application/json"}
if self.cloudflare_token:
headers["Authorization"] = f"Bearer {self.cloudflare_token}"
else:
headers["X-Auth-Email"] = self.cloudflare_email
headers["X-Auth-Key"] = self.cloudflare_api_key
data = {"files": urls}
response = requests.delete(
purge_url,
json=data,
headers=headers,
)
try:
response_json = response.json()
except ValueError:
if response.status_code != 200:
response.raise_for_status()
else:
for url in urls:
logger.error(
"Couldn't purge '%s' from Cloudflare. Unexpected JSON parse error.",
url,
)
except requests.exceptions.HTTPError as e:
for url in urls:
logging.exception(
"Couldn't purge '%s' from Cloudflare. HTTPError: %d",
url,
e.response.status_code,
)
return
if response_json["success"] is False:
error_messages = ", ".join(
[str(err["message"]) for err in response_json["errors"]]
)
for url in urls:
logger.error(
"Couldn't purge '%s' from Cloudflare. Cloudflare errors '%s'",
url,
error_messages,
)
return
def purge_batch(self, urls):
# Break the batched URLs in to chunks to fit within Cloudflare's maximum size for
# the purge_cache call (https://api.cloudflare.com/#zone-purge-files-by-url)
for i in range(0, len(urls), self.CHUNK_SIZE):
chunk = urls[i : i + self.CHUNK_SIZE]
self._purge_urls(chunk)
def purge(self, url):
self._purge_urls([url])

View File

@@ -0,0 +1,99 @@
import logging
import uuid
from collections import defaultdict
from urllib.parse import urlparse
from warnings import warn
from django.core.exceptions import ImproperlyConfigured
from wagtail.utils.deprecation import RemovedInWagtail70Warning
from .base import BaseBackend
logger = logging.getLogger("wagtail.frontendcache")
__all__ = ["CloudfrontBackend"]
class CloudfrontBackend(BaseBackend):
def __init__(self, params):
import boto3
super().__init__(params)
self.client = boto3.client(
"cloudfront",
aws_access_key_id=params.get("AWS_ACCESS_KEY_ID"),
aws_secret_access_key=params.get("AWS_SECRET_ACCESS_KEY"),
aws_session_token=params.get("AWS_SESSION_TOKEN"),
)
try:
self.cloudfront_distribution_id = params.pop("DISTRIBUTION_ID")
except KeyError:
raise ImproperlyConfigured(
"The setting 'WAGTAILFRONTENDCACHE' requires the object 'DISTRIBUTION_ID'."
)
# Add known hostnames for hostname validation (if not already defined)
# RemovedInWagtail70Warning
if isinstance(self.cloudfront_distribution_id, dict):
if "HOSTNAMES" in params:
self.hostnames.extend(self.cloudfront_distribution_id.keys())
else:
self.hostnames = list(self.cloudfront_distribution_id.keys())
def purge_batch(self, urls):
paths_by_distribution_id = defaultdict(list)
for url in urls:
url_parsed = urlparse(url)
distribution_id = None
if isinstance(self.cloudfront_distribution_id, dict):
warn(
"Using a `DISTRIBUTION_ID` mapping is deprecated - use `HOSTNAMES` in combination with multiple backends instead.",
category=RemovedInWagtail70Warning,
)
host = url_parsed.hostname
if host in self.cloudfront_distribution_id:
distribution_id = self.cloudfront_distribution_id.get(host)
else:
logger.warning(
"Couldn't purge '%s' from CloudFront. Hostname '%s' not found in the DISTRIBUTION_ID mapping",
url,
host,
)
else:
distribution_id = self.cloudfront_distribution_id
if distribution_id:
paths_by_distribution_id[distribution_id].append(url_parsed.path)
for distribution_id, paths in paths_by_distribution_id.items():
self._create_invalidation(distribution_id, paths)
def purge(self, url):
self.purge_batch([url])
def _create_invalidation(self, distribution_id, paths):
import botocore
try:
self.client.create_invalidation(
DistributionId=distribution_id,
InvalidationBatch={
"Paths": {"Quantity": len(paths), "Items": paths},
"CallerReference": str(uuid.uuid4()),
},
)
except botocore.exceptions.ClientError as e:
for path in paths:
logger.error(
"Couldn't purge path '%s' from CloudFront (DistributionId=%s). ClientError: %s %s",
path,
distribution_id,
e.response["Error"]["Code"],
e.response["Error"]["Message"],
)

View File

@@ -0,0 +1,64 @@
import logging
from urllib.error import HTTPError, URLError
from urllib.parse import urlsplit, urlunsplit
from urllib.request import Request, urlopen
from wagtail import __version__
from .base import BaseBackend
logger = logging.getLogger("wagtail.frontendcache")
__all__ = ["PurgeRequest", "HTTPBackend"]
class PurgeRequest(Request):
def get_method(self):
return "PURGE"
class HTTPBackend(BaseBackend):
def __init__(self, params):
super().__init__(params)
location_url_parsed = urlsplit(params.pop("LOCATION"))
self.cache_scheme = location_url_parsed.scheme
self.cache_netloc = location_url_parsed.netloc
def purge(self, url):
url_parsed = urlsplit(url)
host = url_parsed.hostname
# Append port to host if it is set in the original URL
if url_parsed.port:
host += ":" + str(url_parsed.port)
request = PurgeRequest(
url=urlunsplit(
[
self.cache_scheme,
self.cache_netloc,
url_parsed.path,
url_parsed.query,
url_parsed.fragment,
]
),
headers={
"Host": host,
"User-Agent": "Wagtail-frontendcache/" + __version__,
},
)
try:
urlopen(request)
except HTTPError as e:
logger.error(
"Couldn't purge '%s' from HTTP cache. HTTPError: %d %s",
url,
e.code,
e.reason,
)
except URLError as e:
logger.error(
"Couldn't purge '%s' from HTTP cache. URLError: %s", url, e.reason
)

View File

@@ -0,0 +1,23 @@
from django.apps import apps
from wagtail.contrib.frontend_cache.utils import purge_page_from_cache
from wagtail.signals import page_published, page_unpublished
def page_published_signal_handler(instance, **kwargs):
purge_page_from_cache(instance)
def page_unpublished_signal_handler(instance, **kwargs):
purge_page_from_cache(instance)
def register_signal_handlers():
# Get list of models that are page types
Page = apps.get_model("wagtailcore", "Page")
indexed_models = [model for model in apps.get_models() if issubclass(model, Page)]
# Loop through list and register signal handlers for each one
for model in indexed_models:
page_published.connect(page_published_signal_handler, sender=model)
page_unpublished.connect(page_unpublished_signal_handler, sender=model)

View File

@@ -0,0 +1,710 @@
from unittest import mock
from urllib.error import HTTPError, URLError
import requests
from azure.mgmt.cdn import CdnManagementClient
from azure.mgmt.frontdoor import FrontDoorManagementClient
from django.core.exceptions import ImproperlyConfigured
from django.test import SimpleTestCase, TestCase
from django.test.utils import override_settings
from wagtail.contrib.frontend_cache.backends import (
AzureCdnBackend,
AzureFrontDoorBackend,
BaseBackend,
CloudflareBackend,
CloudfrontBackend,
HTTPBackend,
)
from wagtail.contrib.frontend_cache.utils import get_backends
from wagtail.models import Page
from wagtail.test.testapp.models import EventIndex
from wagtail.utils.deprecation import RemovedInWagtail70Warning
from .utils import (
PurgeBatch,
purge_page_from_cache,
purge_pages_from_cache,
purge_url_from_cache,
purge_urls_from_cache,
)
class TestBackendConfiguration(SimpleTestCase):
def test_default(self):
backends = get_backends()
self.assertEqual(len(backends), 0)
def test_varnish(self):
backends = get_backends(
backend_settings={
"varnish": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.HTTPBackend",
"LOCATION": "http://localhost:8000",
},
}
)
self.assertEqual(set(backends.keys()), {"varnish"})
self.assertIsInstance(backends["varnish"], HTTPBackend)
self.assertEqual(backends["varnish"].cache_scheme, "http")
self.assertEqual(backends["varnish"].cache_netloc, "localhost:8000")
def test_cloudflare(self):
backends = get_backends(
backend_settings={
"cloudflare": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.CloudflareBackend",
"EMAIL": "test@test.com",
"API_KEY": "this is the api key",
"ZONEID": "this is a zone id",
"BEARER_TOKEN": "this is a bearer token",
},
}
)
self.assertEqual(set(backends.keys()), {"cloudflare"})
self.assertIsInstance(backends["cloudflare"], CloudflareBackend)
self.assertEqual(backends["cloudflare"].cloudflare_email, "test@test.com")
self.assertEqual(
backends["cloudflare"].cloudflare_api_key, "this is the api key"
)
self.assertEqual(
backends["cloudflare"].cloudflare_token, "this is a bearer token"
)
def test_cloudfront(self):
backends = get_backends(
backend_settings={
"cloudfront": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.CloudfrontBackend",
"DISTRIBUTION_ID": "frontend",
"AWS_ACCESS_KEY_ID": "my-access-key-id",
"AWS_SECRET_ACCESS_KEY": "my-secret-access-key",
},
}
)
self.assertEqual(set(backends.keys()), {"cloudfront"})
self.assertIsInstance(backends["cloudfront"], CloudfrontBackend)
self.assertEqual(backends["cloudfront"].cloudfront_distribution_id, "frontend")
credentials = backends["cloudfront"].client._request_signer._credentials
self.assertEqual(credentials.method, "explicit")
self.assertEqual(credentials.access_key, "my-access-key-id")
self.assertEqual(credentials.secret_key, "my-secret-access-key")
def test_azure_cdn(self):
backends = get_backends(
backend_settings={
"azure_cdn": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.AzureCdnBackend",
"RESOURCE_GROUP_NAME": "test-resource-group",
"CDN_PROFILE_NAME": "wagtail-io-profile",
"CDN_ENDPOINT_NAME": "wagtail-io-endpoint",
},
}
)
self.assertEqual(set(backends.keys()), {"azure_cdn"})
self.assertIsInstance(backends["azure_cdn"], AzureCdnBackend)
self.assertEqual(
backends["azure_cdn"]._resource_group_name, "test-resource-group"
)
self.assertEqual(backends["azure_cdn"]._cdn_profile_name, "wagtail-io-profile")
self.assertEqual(
backends["azure_cdn"]._cdn_endpoint_name, "wagtail-io-endpoint"
)
def test_azure_front_door(self):
backends = get_backends(
backend_settings={
"azure_front_door": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.AzureFrontDoorBackend",
"RESOURCE_GROUP_NAME": "test-resource-group",
"FRONT_DOOR_NAME": "wagtail-io-front-door",
},
}
)
self.assertEqual(set(backends.keys()), {"azure_front_door"})
self.assertIsInstance(backends["azure_front_door"], AzureFrontDoorBackend)
self.assertEqual(
backends["azure_front_door"]._resource_group_name, "test-resource-group"
)
self.assertEqual(
backends["azure_front_door"]._front_door_name, "wagtail-io-front-door"
)
def test_azure_cdn_get_client(self):
mock_credentials = mock.MagicMock()
backends = get_backends(
backend_settings={
"azure_cdn": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.AzureCdnBackend",
"RESOURCE_GROUP_NAME": "test-resource-group",
"CDN_PROFILE_NAME": "wagtail-io-profile",
"CDN_ENDPOINT_NAME": "wagtail-io-endpoint",
"SUBSCRIPTION_ID": "fake-subscription-id",
"CREDENTIALS": mock_credentials,
},
}
)
self.assertEqual(set(backends.keys()), {"azure_cdn"})
client = backends["azure_cdn"]._get_client()
self.assertIsInstance(client, CdnManagementClient)
self.assertEqual(client._config.subscription_id, "fake-subscription-id")
self.assertIs(client._config.credential, mock_credentials)
def test_azure_front_door_get_client(self):
mock_credentials = mock.MagicMock()
backends = get_backends(
backend_settings={
"azure_front_door": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.AzureFrontDoorBackend",
"RESOURCE_GROUP_NAME": "test-resource-group",
"FRONT_DOOR_NAME": "wagtail-io-fake-front-door-name",
"SUBSCRIPTION_ID": "fake-subscription-id",
"CREDENTIALS": mock_credentials,
},
}
)
client = backends["azure_front_door"]._get_client()
self.assertEqual(set(backends.keys()), {"azure_front_door"})
self.assertIsInstance(client, FrontDoorManagementClient)
self.assertEqual(client._config.subscription_id, "fake-subscription-id")
self.assertIs(client._config.credential, mock_credentials)
@mock.patch(
"wagtail.contrib.frontend_cache.backends.azure.AzureCdnBackend._make_purge_call"
)
def test_azure_cdn_purge(self, make_purge_call_mock):
backends = get_backends(
backend_settings={
"azure_cdn": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.AzureCdnBackend",
"RESOURCE_GROUP_NAME": "test-resource-group",
"CDN_PROFILE_NAME": "wagtail-io-profile",
"CDN_ENDPOINT_NAME": "wagtail-io-endpoint",
"CREDENTIALS": "Fake credentials",
},
}
)
self.assertEqual(set(backends.keys()), {"azure_cdn"})
self.assertIsInstance(backends["azure_cdn"], AzureCdnBackend)
# purge()
backends["azure_cdn"].purge(
"http://www.wagtail.org/home/events/christmas/?test=1"
)
make_purge_call_mock.assert_called_once()
call_args = tuple(make_purge_call_mock.call_args)[0]
self.assertEqual(len(call_args), 2)
self.assertIsInstance(call_args[0], CdnManagementClient)
self.assertEqual(call_args[1], ["/home/events/christmas/?test=1"])
make_purge_call_mock.reset_mock()
# purge_batch()
backends["azure_cdn"].purge_batch(
[
"http://www.wagtail.org/home/events/christmas/?test=1",
"http://torchbox.com/blog/",
]
)
make_purge_call_mock.assert_called_once()
call_args = tuple(make_purge_call_mock.call_args)[0]
self.assertIsInstance(call_args[0], CdnManagementClient)
self.assertEqual(call_args[1], ["/home/events/christmas/?test=1", "/blog/"])
@mock.patch(
"wagtail.contrib.frontend_cache.backends.azure.AzureFrontDoorBackend._make_purge_call"
)
def test_azure_front_door_purge(self, make_purge_call_mock):
backends = get_backends(
backend_settings={
"azure_front_door": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.AzureFrontDoorBackend",
"RESOURCE_GROUP_NAME": "test-resource-group",
"FRONT_DOOR_NAME": "wagtail-io-front-door",
"CREDENTIALS": "Fake credentials",
},
}
)
self.assertEqual(set(backends.keys()), {"azure_front_door"})
self.assertIsInstance(backends["azure_front_door"], AzureFrontDoorBackend)
# purge()
backends["azure_front_door"].purge(
"http://www.wagtail.org/home/events/christmas/?test=1"
)
make_purge_call_mock.assert_called_once()
call_args = tuple(make_purge_call_mock.call_args)[0]
self.assertIsInstance(call_args[0], FrontDoorManagementClient)
self.assertEqual(call_args[1], ["/home/events/christmas/?test=1"])
make_purge_call_mock.reset_mock()
# purge_batch()
backends["azure_front_door"].purge_batch(
[
"http://www.wagtail.org/home/events/christmas/?test=1",
"http://torchbox.com/blog/",
]
)
make_purge_call_mock.assert_called_once()
call_args = tuple(make_purge_call_mock.call_args)[0]
self.assertIsInstance(call_args[0], FrontDoorManagementClient)
self.assertEqual(call_args[1], ["/home/events/christmas/?test=1", "/blog/"])
def test_http(self):
"""Test that `HTTPBackend.purge` works when urlopen succeeds"""
self._test_http_with_side_effect(urlopen_side_effect=None)
def test_http_httperror(self):
"""Test that `HTTPBackend.purge` can handle `HTTPError`"""
http_error = HTTPError(
url="http://localhost:8000/home/events/christmas/",
code=500,
msg="Internal Server Error",
hdrs={},
fp=None,
)
with self.assertLogs(level="ERROR") as log_output:
self._test_http_with_side_effect(urlopen_side_effect=http_error)
self.assertIn(
"Couldn't purge 'http://www.wagtail.org/home/events/christmas/' from HTTP cache. HTTPError: 500 Internal Server Error",
log_output.output[0],
)
def test_http_urlerror(self):
"""Test that `HTTPBackend.purge` can handle `URLError`"""
url_error = URLError(reason="just for tests")
with self.assertLogs(level="ERROR") as log_output:
self._test_http_with_side_effect(urlopen_side_effect=url_error)
self.assertIn(
"Couldn't purge 'http://www.wagtail.org/home/events/christmas/' from HTTP cache. URLError: just for tests",
log_output.output[0],
)
@mock.patch("wagtail.contrib.frontend_cache.backends.http.urlopen")
def _test_http_with_side_effect(self, urlopen_mock, urlopen_side_effect):
# given a backends configuration with one HTTP backend
backends = get_backends(
backend_settings={
"varnish": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.HTTPBackend",
"LOCATION": "http://localhost:8000",
},
}
)
self.assertEqual(set(backends.keys()), {"varnish"})
self.assertIsInstance(backends["varnish"], HTTPBackend)
# and mocked urlopen that may or may not raise network-related exception
urlopen_mock.side_effect = urlopen_side_effect
# when making a purge request
backends.get("varnish").purge("http://www.wagtail.org/home/events/christmas/")
# then no exception is raised
# and mocked urlopen is called with a proper purge request
self.assertEqual(urlopen_mock.call_count, 1)
(purge_request,), _call_kwargs = urlopen_mock.call_args
self.assertEqual(
purge_request.full_url, "http://localhost:8000/home/events/christmas/"
)
def test_cloudfront_validate_distribution_id(self):
with self.assertRaises(ImproperlyConfigured):
get_backends(
backend_settings={
"cloudfront": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.CloudfrontBackend",
},
}
)
@mock.patch(
"wagtail.contrib.frontend_cache.backends.cloudfront.CloudfrontBackend._create_invalidation"
)
def test_cloudfront_distribution_id_mapping(self, _create_invalidation):
backends = get_backends(
backend_settings={
"cloudfront": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.CloudfrontBackend",
"DISTRIBUTION_ID": {
"www.wagtail.org": "frontend",
},
},
}
)
with self.assertWarnsMessage(
RemovedInWagtail70Warning,
"Using a `DISTRIBUTION_ID` mapping is deprecated - use `HOSTNAMES` in combination with multiple backends instead.",
):
backends.get("cloudfront").purge(
"http://www.wagtail.org/home/events/christmas/"
)
with self.assertWarnsMessage(
RemovedInWagtail70Warning,
"Using a `DISTRIBUTION_ID` mapping is deprecated - use `HOSTNAMES` in combination with multiple backends instead.",
):
backends.get("cloudfront").purge("http://torchbox.com/blog/")
_create_invalidation.assert_called_once_with(
"frontend", ["/home/events/christmas/"]
)
self.assertTrue(
backends.get("cloudfront").invalidates_hostname("www.wagtail.org")
)
self.assertFalse(
backends.get("cloudfront").invalidates_hostname("torchbox.com")
)
def test_multiple(self):
backends = get_backends(
backend_settings={
"varnish": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.HTTPBackend",
"LOCATION": "http://localhost:8000/",
},
"cloudflare": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.CloudflareBackend",
"EMAIL": "test@test.com",
"API_KEY": "this is the api key",
"ZONEID": "this is a zone id",
},
}
)
self.assertEqual(set(backends.keys()), {"varnish", "cloudflare"})
def test_filter(self):
backends = get_backends(
backend_settings={
"varnish": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.HTTPBackend",
"LOCATION": "http://localhost:8000/",
},
"cloudflare": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.CloudflareBackend",
"EMAIL": "test@test.com",
"API_KEY": "this is the api key",
"ZONEID": "this is a zone id",
},
},
backends=["cloudflare"],
)
self.assertEqual(set(backends.keys()), {"cloudflare"})
@override_settings(WAGTAILFRONTENDCACHE_LOCATION="http://localhost:8000")
def test_backwards_compatibility(self):
backends = get_backends()
self.assertEqual(set(backends.keys()), {"default"})
self.assertIsInstance(backends["default"], HTTPBackend)
self.assertEqual(backends["default"].cache_scheme, "http")
self.assertEqual(backends["default"].cache_netloc, "localhost:8000")
PURGED_URLS = []
class MockBackend(BaseBackend):
def purge(self, url):
PURGED_URLS.append(url)
class MockCloudflareBackend(CloudflareBackend):
def _purge_urls(self, urls):
if len(urls) > self.CHUNK_SIZE:
raise Exception("Cloudflare backend is not chunking requests as expected")
PURGED_URLS.extend(urls)
@override_settings(
WAGTAILFRONTENDCACHE={
"varnish": {
"BACKEND": "wagtail.contrib.frontend_cache.tests.MockBackend",
},
}
)
class TestCachePurgingFunctions(TestCase):
fixtures = ["test.json"]
def setUp(self):
# Reset PURGED_URLS to an empty list
PURGED_URLS[:] = []
def test_purge_url_from_cache(self):
purge_url_from_cache("http://localhost/foo")
self.assertEqual(PURGED_URLS, ["http://localhost/foo"])
def test_purge_urls_from_cache(self):
purge_urls_from_cache(["http://localhost/foo", "http://localhost/bar"])
self.assertEqual(PURGED_URLS, ["http://localhost/foo", "http://localhost/bar"])
def test_purge_page_from_cache(self):
page = EventIndex.objects.get(url_path="/home/events/")
purge_page_from_cache(page)
self.assertEqual(
PURGED_URLS, ["http://localhost/events/", "http://localhost/events/past/"]
)
def test_purge_pages_from_cache(self):
purge_pages_from_cache(EventIndex.objects.all())
self.assertEqual(
PURGED_URLS, ["http://localhost/events/", "http://localhost/events/past/"]
)
def test_purge_batch(self):
batch = PurgeBatch()
page = EventIndex.objects.get(url_path="/home/events/")
batch.add_page(page)
batch.add_url("http://localhost/foo")
batch.purge()
self.assertEqual(
PURGED_URLS,
[
"http://localhost/events/",
"http://localhost/events/past/",
"http://localhost/foo",
],
)
@override_settings(
WAGTAILFRONTENDCACHE={
"varnish": {
"BACKEND": "wagtail.contrib.frontend_cache.tests.MockBackend",
"HOSTNAMES": ["example.com"],
},
}
)
def test_invalidate_specific_location(self):
with self.assertLogs(level="INFO") as log_output:
purge_url_from_cache("http://localhost/foo")
self.assertEqual(PURGED_URLS, [])
self.assertIn(
"Unable to find purge backend for localhost",
log_output.output[0],
)
purge_url_from_cache("http://example.com/foo")
self.assertEqual(PURGED_URLS, ["http://example.com/foo"])
@override_settings(
WAGTAILFRONTENDCACHE={
"cloudflare": {
"BACKEND": "wagtail.contrib.frontend_cache.tests.MockCloudflareBackend",
"ZONEID": "zone",
"BEARER_TOKEN": "token",
},
}
)
class TestCloudflareCachePurgingFunctions(TestCase):
def setUp(self):
# Reset PURGED_URLS to an empty list
PURGED_URLS[:] = []
def test_cloudflare_purge_batch_chunked(self):
batch = PurgeBatch()
urls = [f"https://localhost/foo{i}" for i in range(1, 65)]
batch.add_urls(urls)
batch.purge()
self.assertCountEqual(PURGED_URLS, urls)
@override_settings(
WAGTAILFRONTENDCACHE={
"varnish": {
"BACKEND": "wagtail.contrib.frontend_cache.tests.MockBackend",
},
}
)
class TestCachePurgingSignals(TestCase):
fixtures = ["test.json"]
def setUp(self):
# Reset PURGED_URLS to an empty list
PURGED_URLS[:] = []
def test_purge_on_publish(self):
page = EventIndex.objects.get(url_path="/home/events/")
page.save_revision().publish()
self.assertEqual(
PURGED_URLS, ["http://localhost/events/", "http://localhost/events/past/"]
)
def test_purge_on_unpublish(self):
page = EventIndex.objects.get(url_path="/home/events/")
page.unpublish()
self.assertEqual(
PURGED_URLS, ["http://localhost/events/", "http://localhost/events/past/"]
)
def test_purge_with_unroutable_page(self):
root = Page.objects.get(url_path="/")
page = EventIndex(title="new top-level page")
root.add_child(instance=page)
page.save_revision().publish()
self.assertEqual(PURGED_URLS, [])
@override_settings(
ROOT_URLCONF="wagtail.test.urls_multilang",
LANGUAGE_CODE="en",
WAGTAILFRONTENDCACHE_LANGUAGES=["en", "fr", "pt-br"],
)
def test_purge_on_publish_in_multilang_env(self):
PURGED_URLS[:] = [] # reset PURGED_URLS to the empty list
page = EventIndex.objects.get(url_path="/home/events/")
page.save_revision().publish()
self.assertEqual(
PURGED_URLS,
[
"http://localhost/en/events/",
"http://localhost/en/events/past/",
"http://localhost/fr/events/",
"http://localhost/fr/events/past/",
"http://localhost/pt-br/events/",
"http://localhost/pt-br/events/past/",
],
)
@override_settings(
ROOT_URLCONF="wagtail.test.urls_multilang",
LANGUAGE_CODE="en",
WAGTAIL_I18N_ENABLED=True,
WAGTAIL_CONTENT_LANGUAGES=[("en", "English"), ("fr", "French")],
)
def test_purge_on_publish_with_i18n_enabled(self):
PURGED_URLS[:] = [] # reset PURGED_URLS to the empty list
page = EventIndex.objects.get(url_path="/home/events/")
page.save_revision().publish()
self.assertEqual(
PURGED_URLS,
[
"http://localhost/en/events/",
"http://localhost/en/events/past/",
"http://localhost/fr/events/",
"http://localhost/fr/events/past/",
],
)
@override_settings(
ROOT_URLCONF="wagtail.test.urls_multilang",
LANGUAGE_CODE="en",
WAGTAIL_CONTENT_LANGUAGES=[("en", "English"), ("fr", "French")],
)
def test_purge_on_publish_without_i18n_enabled(self):
# It should ignore WAGTAIL_CONTENT_LANGUAGES as WAGTAIL_I18N_ENABLED isn't set
PURGED_URLS[:] = [] # reset PURGED_URLS to the empty list
page = EventIndex.objects.get(url_path="/home/events/")
page.save_revision().publish()
self.assertEqual(
PURGED_URLS,
["http://localhost/en/events/", "http://localhost/en/events/past/"],
)
class TestPurgeBatchClass(TestCase):
# Tests the .add_*() methods on PurgeBatch. The .purge() method is tested
# by TestCachePurgingFunctions.test_purge_batch above
fixtures = ["test.json"]
def test_add_url(self):
batch = PurgeBatch()
batch.add_url("http://localhost/foo")
self.assertEqual(batch.urls, ["http://localhost/foo"])
def test_add_urls(self):
batch = PurgeBatch()
batch.add_urls(["http://localhost/foo", "http://localhost/bar"])
self.assertEqual(batch.urls, ["http://localhost/foo", "http://localhost/bar"])
def test_add_page(self):
page = EventIndex.objects.get(url_path="/home/events/")
batch = PurgeBatch()
batch.add_page(page)
self.assertEqual(
batch.urls, ["http://localhost/events/", "http://localhost/events/past/"]
)
def test_add_pages(self):
batch = PurgeBatch()
batch.add_pages(EventIndex.objects.all())
self.assertEqual(
batch.urls, ["http://localhost/events/", "http://localhost/events/past/"]
)
def test_multiple_calls(self):
page = EventIndex.objects.get(url_path="/home/events/")
batch = PurgeBatch()
batch.add_page(page)
batch.add_url("http://localhost/foo")
batch.purge()
self.assertEqual(
batch.urls,
[
"http://localhost/events/",
"http://localhost/events/past/",
"http://localhost/foo",
],
)
@mock.patch("wagtail.contrib.frontend_cache.backends.cloudflare.requests.delete")
def test_http_error_on_cloudflare_purge_batch(self, requests_delete_mock):
backend_settings = {
"cloudflare": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.CloudflareBackend",
"EMAIL": "test@test.com",
"API_KEY": "this is the api key",
"ZONEID": "this is a zone id",
},
}
class MockResponse:
def __init__(self, status_code=200):
self.status_code = status_code
http_error = requests.exceptions.HTTPError(
response=MockResponse(status_code=500)
)
requests_delete_mock.side_effect = http_error
page = EventIndex.objects.get(url_path="/home/events/")
batch = PurgeBatch()
batch.add_page(page)
with self.assertLogs(level="ERROR") as log_output:
batch.purge(backend_settings=backend_settings)
self.assertIn(
"Couldn't purge 'http://localhost/events/' from Cloudflare. HTTPError: 500",
log_output.output[0],
)

View File

@@ -0,0 +1,202 @@
import logging
import re
from collections import defaultdict
from urllib.parse import urlsplit, urlunsplit
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.utils.module_loading import import_string
from wagtail.coreutils import get_content_languages
logger = logging.getLogger("wagtail.frontendcache")
class InvalidFrontendCacheBackendError(ImproperlyConfigured):
pass
def get_backends(backend_settings=None, backends=None):
# Get backend settings from WAGTAILFRONTENDCACHE setting
if backend_settings is None:
backend_settings = getattr(settings, "WAGTAILFRONTENDCACHE", None)
# Fallback to using WAGTAILFRONTENDCACHE_LOCATION setting (backwards compatibility)
if backend_settings is None:
cache_location = getattr(settings, "WAGTAILFRONTENDCACHE_LOCATION", None)
if cache_location is not None:
backend_settings = {
"default": {
"BACKEND": "wagtail.contrib.frontend_cache.backends.HTTPBackend",
"LOCATION": cache_location,
},
}
# No settings found, return empty list
if backend_settings is None:
return {}
backend_objects = {}
for backend_name, _backend_config in backend_settings.items():
if backends is not None and backend_name not in backends:
continue
backend_config = _backend_config.copy()
backend = backend_config.pop("BACKEND")
# Try to import the backend
try:
backend_cls = import_string(backend)
except ImportError as e:
raise InvalidFrontendCacheBackendError(
f"Could not find backend '{backend}': {e}"
)
backend_objects[backend_name] = backend_cls(backend_config)
return backend_objects
def purge_url_from_cache(url, backend_settings=None, backends=None):
purge_urls_from_cache([url], backend_settings=backend_settings, backends=backends)
def purge_urls_from_cache(urls, backend_settings=None, backends=None):
# Convert each url to urls one for each managed language (WAGTAILFRONTENDCACHE_LANGUAGES setting).
# The managed languages are common to all the defined backends.
# This depends on settings.USE_I18N
# If WAGTAIL_I18N_ENABLED is True, this defaults to WAGTAIL_CONTENT_LANGUAGES
wagtail_i18n_enabled = getattr(settings, "WAGTAIL_I18N_ENABLED", False)
content_languages = get_content_languages() if wagtail_i18n_enabled else {}
languages = getattr(
settings, "WAGTAILFRONTENDCACHE_LANGUAGES", list(content_languages.keys())
)
if settings.USE_I18N and languages:
langs_regex = "^/(%s)/" % "|".join(languages)
new_urls = []
# Purge the given url for each managed language
for isocode in languages:
for url in urls:
up = urlsplit(url)
new_url = urlunsplit(
(
up.scheme,
up.netloc,
re.sub(langs_regex, "/%s/" % isocode, up.path),
up.query,
up.fragment,
)
)
# Check for best performance. True if re.sub found no match
# It happens when i18n_patterns was not used in urls.py to serve content for different languages from different URLs
if new_url in new_urls:
continue
new_urls.append(new_url)
urls = new_urls
urls_by_hostname = defaultdict(list)
for url in urls:
urls_by_hostname[urlsplit(url).netloc].append(url)
backends = get_backends(backend_settings, backends)
for hostname, urls in urls_by_hostname.items():
backends_for_hostname = {
backend_name: backend
for backend_name, backend in backends.items()
if backend.invalidates_hostname(hostname)
}
if not backends_for_hostname:
logger.info("Unable to find purge backend for %s", hostname)
continue
for backend_name, backend in backends_for_hostname.items():
for url in urls:
logger.info("[%s] Purging URL: %s", backend_name, url)
backend.purge_batch(urls)
def _get_page_cached_urls(page):
page_url = page.full_url
if page_url is None: # nothing to be done if the page has no routable URL
return []
return [page_url + path.lstrip("/") for path in page.specific.get_cached_paths()]
def purge_page_from_cache(page, backend_settings=None, backends=None):
purge_pages_from_cache([page], backend_settings=backend_settings, backends=backends)
def purge_pages_from_cache(pages, backend_settings=None, backends=None):
urls = []
for page in pages:
urls.extend(_get_page_cached_urls(page))
if urls:
purge_urls_from_cache(urls, backend_settings, backends)
class PurgeBatch:
"""Represents a list of URLs to be purged in a single request"""
def __init__(self, urls=None):
self.urls = []
if urls is not None:
self.add_urls(urls)
def add_url(self, url):
"""Adds a single URL"""
self.urls.append(url)
def add_urls(self, urls):
"""
Adds multiple URLs from an iterable
This is equivalent to running ``.add_url(url)`` on each URL
individually
"""
self.urls.extend(urls)
def add_page(self, page):
"""
Adds all URLs for the specified page
This combines the page's full URL with each path that is returned by
the page's `.get_cached_paths` method
"""
self.add_urls(_get_page_cached_urls(page))
def add_pages(self, pages):
"""
Adds multiple pages from a QuerySet or an iterable
This is equivalent to running ``.add_page(page)`` on each page
individually
"""
for page in pages:
self.add_page(page)
def purge(self, backend_settings=None, backends=None):
"""
Performs the purge of all the URLs in this batch
This method takes two optional keyword arguments: backend_settings and backends
- backend_settings can be used to override the WAGTAILFRONTENDCACHE setting for
just this call
- backends can be set to a list of backend names. When set, the invalidation request
will only be sent to these backends
"""
purge_urls_from_cache(self.urls, backend_settings, backends)