Initial commit

This commit is contained in:
2024-08-27 20:33:44 +02:00
commit 1f1832267d
14794 changed files with 1599592 additions and 0 deletions

View File

@@ -0,0 +1,43 @@
from django.apps import apps
from django.core.management.base import BaseCommand
from django.db import connection, models
from wagtail.models import (
BaseLogEntry,
BootstrapTranslatableMixin,
ReferenceIndex,
TranslatableMixin,
)
class Command(BaseCommand):
help = "Converts UUID columns from char type to the native UUID type used in MariaDB 10.7+ and Django 5.0+."
def convert_field(self, model, field_name, null=False):
if model._meta.get_field(field_name).model != model:
# Field is inherited from a parent model
return
if not model._meta.managed:
# The migration framework skips unmanaged models, so we should too
return
old_field = models.CharField(null=null, max_length=36)
old_field.set_attributes_from_name(field_name)
new_field = models.UUIDField(null=null)
new_field.set_attributes_from_name(field_name)
with connection.schema_editor() as schema_editor:
schema_editor.alter_field(model, old_field, new_field)
def handle(self, **options):
self.convert_field(ReferenceIndex, "content_path_hash")
for model in apps.get_models():
if issubclass(model, BaseLogEntry):
self.convert_field(model, "uuid", null=True)
elif issubclass(model, BootstrapTranslatableMixin):
self.convert_field(model, "translation_key", null=True)
elif issubclass(model, TranslatableMixin):
self.convert_field(model, "translation_key")

View File

@@ -0,0 +1,118 @@
from django.core.management.base import BaseCommand
from wagtail.models import PageLogEntry, Revision
def get_comparison(page, revision_a, revision_b):
comparison = (
page.get_edit_handler()
.get_bound_panel(instance=page, form=None, request=None)
.get_comparison()
)
comparison = [comp(revision_a, revision_b) for comp in comparison]
comparison = [comp for comp in comparison if comp.has_changed()]
return comparison
class Command(BaseCommand):
def handle(self, *args, **options):
current_page_id = None
missing_models_content_type_ids = set()
for revision in Revision.page_revisions.order_by(
"object_id", "created_at"
).iterator():
# This revision is for a page type that is no longer in the database. Bail out early.
if (
revision.content_object.content_type_id
in missing_models_content_type_ids
):
continue
if not revision.content_object.specific_class:
missing_models_content_type_ids.add(
revision.content_object.content_type_id
)
continue
is_new_page = revision.object_id != current_page_id
if is_new_page:
# reset previous revision when encountering a new page.
previous_revision = None
has_content_changes = False
current_page_id = revision.object_id
if not PageLogEntry.objects.filter(revision=revision).exists():
try:
current_revision_as_page = revision.as_object()
except Exception: # noqa: BLE001
# restoring old revisions may fail if e.g. they have an on_delete=PROTECT foreign key
# to a no-longer-existing model instance. We cannot compare changes between two
# non-restorable revisions, although we can at least infer that there was a content
# change at the point that it went from restorable to non-restorable or vice versa.
current_revision_as_page = None
published = revision.id == revision.content_object.live_revision_id
if previous_revision is not None:
try:
previous_revision_as_page = previous_revision.as_object()
except Exception: # noqa: BLE001
previous_revision_as_page = None
if (
previous_revision_as_page is None
and current_revision_as_page is None
):
# both revisions failed to restore - unable to determine presence of content changes
has_content_changes = False
elif (
previous_revision_as_page is None
or current_revision_as_page is None
):
# one or the other revision failed to restore, which indicates a content change
has_content_changes = True
else:
# Must use .specific so the comparison picks up all fields, not just base Page ones.
comparison = get_comparison(
revision.content_object.specific,
previous_revision_as_page,
current_revision_as_page,
)
has_content_changes = len(comparison) > 0
if (
current_revision_as_page is not None
and current_revision_as_page.live_revision_id
== previous_revision.id
):
# Log the previous revision publishing.
self.log_page_action("wagtail.publish", previous_revision, True)
if is_new_page or has_content_changes or published:
actions = []
if is_new_page:
actions.append("wagtail.create")
if is_new_page or has_content_changes:
actions.append("wagtail.edit")
if published:
actions.append("wagtail.publish")
for action in actions:
self.log_page_action(action, revision, has_content_changes)
previous_revision = revision
def log_page_action(self, action, revision, has_content_changes):
PageLogEntry.objects.log_action(
instance=revision.content_object.specific,
action=action,
data={},
revision=None if action == "wagtail.create" else revision,
user=revision.user,
timestamp=revision.created_at,
content_changed=has_content_changes,
)

View File

@@ -0,0 +1,165 @@
import functools
import operator
from django.core.management.base import BaseCommand
from django.db import models
from django.db.models import Q
from wagtail.models import Collection, Page
class Command(BaseCommand):
help = "Checks for data integrity errors on the page tree, and fixes them where possible."
stealth_options = ("delete_orphans",)
def add_arguments(self, parser):
parser.add_argument(
"--noinput",
action="store_false",
dest="interactive",
default=True,
help="If provided, any fixes requiring user interaction will be skipped.",
)
parser.add_argument(
"--full",
action="store_true",
dest="full",
default=False,
help="If provided, uses a more thorough but slower method that also fixes path ordering issues.",
)
def numberlist_to_string(self, numberlist):
# Converts a list of numbers into a string
# Doesn't put "L" after longs
return "[" + ", ".join(map(str, numberlist)) + "]"
def handle(self, **options):
any_page_problems_fixed = False
for page in Page.objects.all():
try:
page.specific
except page.specific_class.DoesNotExist:
self.stdout.write(
"Page %d (%s) is missing a subclass record; deleting."
% (page.id, page.title)
)
any_page_problems_fixed = True
page.delete()
self.handle_model(Page, "page", "pages", any_page_problems_fixed, options)
self.handle_model(Collection, "collection", "collections", False, options)
def handle_model(
self, model, model_name, model_name_plural, any_problems_fixed, options
):
fix_paths = options.get("full", False)
self.stdout.write("Checking %s tree for problems..." % model_name)
(bad_alpha, bad_path, orphans, bad_depth, bad_numchild) = model.find_problems()
if bad_depth:
self.stdout.write(
"Incorrect depth value found for %s: %s"
% (model_name_plural, self.numberlist_to_string(bad_depth))
)
if bad_numchild:
self.stdout.write(
"Incorrect numchild value found for %s: %s"
% (model_name_plural, self.numberlist_to_string(bad_numchild))
)
if orphans:
# The 'orphans' list as returned by treebeard only includes nodes that are
# missing an immediate parent; descendants of orphans are not included.
# Deleting only the *actual* orphans is a bit silly (since it'll just create
# more orphans), so generate a queryset that contains descendants as well.
orphan_paths = model.objects.filter(id__in=orphans).values_list(
"path", flat=True
)
filter_conditions = []
for path in orphan_paths:
filter_conditions.append(Q(path__startswith=path))
# combine filter_conditions into a single ORed condition
final_filter = functools.reduce(operator.or_, filter_conditions)
# build a queryset of all nodes to be removed; this must be a vanilla Django
# queryset rather than a treebeard MP_NodeQuerySet, so that we bypass treebeard's
# custom delete() logic that would trip up on the very same corruption that we're
# trying to fix here.
nodes_to_delete = models.query.QuerySet(model).filter(final_filter)
self.stdout.write("Orphaned %s found:" % model_name_plural)
for node in nodes_to_delete:
self.stdout.write("ID %d: %s" % (node.id, node))
self.stdout.write("")
if options.get("interactive", True):
yes_or_no = input("Delete these %s? [y/N] " % model_name_plural)
delete_orphans = yes_or_no.lower().startswith("y")
self.stdout.write("")
else:
# Running tests, check for the "delete_orphans" option
delete_orphans = options.get("delete_orphans", False)
if delete_orphans:
deletion_count = len(nodes_to_delete)
nodes_to_delete.delete()
self.stdout.write(
"%d orphaned %s deleted."
% (
deletion_count,
model_name_plural if deletion_count != 1 else model_name,
)
)
any_problems_fixed = True
# fix_paths will fix problems not identified by find_problems, so if that option has been
# passed, run it regardless (and set any_problems_fixed=True, since we don't have a way to
# test whether anything was actually fixed in that process)
if bad_depth or bad_numchild or fix_paths:
model.fix_tree(destructive=False, fix_paths=fix_paths)
any_problems_fixed = True
if any_problems_fixed:
# re-run find_problems to see if any new ones have surfaced
(
bad_alpha,
bad_path,
orphans,
bad_depth,
bad_numchild,
) = model.find_problems()
if any((bad_alpha, bad_path, orphans, bad_depth, bad_numchild)):
self.stdout.write("Remaining problems (cannot fix automatically):")
if bad_alpha:
self.stdout.write(
"Invalid characters found in path for %s: %s"
% (model_name_plural, self.numberlist_to_string(bad_alpha))
)
if bad_path:
self.stdout.write(
"Invalid path length found for %s: %s"
% (model_name_plural, self.numberlist_to_string(bad_path))
)
if orphans:
self.stdout.write(
"Orphaned %s found: %s"
% (model_name_plural, self.numberlist_to_string(orphans))
)
if bad_depth:
self.stdout.write(
"Incorrect depth value found for %s: %s"
% (model_name_plural, self.numberlist_to_string(bad_depth))
)
if bad_numchild:
self.stdout.write(
"Incorrect numchild value found for %s: %s"
% (model_name_plural, self.numberlist_to_string(bad_numchild))
)
elif any_problems_fixed:
self.stdout.write("All problems fixed.\n\n")
else:
self.stdout.write("No problems found.\n\n")

View File

@@ -0,0 +1,31 @@
from django.core.management.base import BaseCommand
from wagtail.models import Page
class Command(BaseCommand):
def add_arguments(self, parser):
# Positional arguments
parser.add_argument("from_id", type=int)
parser.add_argument("to_id", type=int)
def handle(self, *args, **options):
# Get pages
from_page = Page.objects.get(pk=options["from_id"])
to_page = Page.objects.get(pk=options["to_id"])
pages = from_page.get_children()
# Move the pages
self.stdout.write(
"Moving "
+ str(len(pages))
+ ' pages from "'
+ from_page.title
+ '" to "'
+ to_page.title
+ '"'
)
for page in pages:
page.move(to_page, pos="last-child")
self.stdout.write("Done")

View File

@@ -0,0 +1,117 @@
from django.apps import apps
from django.core.management.base import BaseCommand
from django.utils import dateparse, timezone
from wagtail.models import DraftStateMixin, Page, Revision
def revision_date_expired(r):
expiry_str = r.content.get("expire_at")
if not expiry_str:
return False
expire_at = dateparse.parse_datetime(expiry_str)
if expire_at < timezone.now():
return True
else:
return False
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument(
"--dryrun",
action="store_true",
dest="dryrun",
default=False,
help="Dry run -- don't change anything.",
)
def handle(self, *args, **options):
dryrun = False
if options["dryrun"]:
self.stdout.write("Will do a dry run.")
dryrun = True
models = [Page]
models += [
model
for model in apps.get_models()
if issubclass(model, DraftStateMixin) and not issubclass(model, Page)
]
# 1. get all expired objects with live = True
expired_objects = []
for model in models:
expired_objects += [
model.objects.filter(live=True, expire_at__lt=timezone.now()).order_by(
"expire_at"
)
]
if dryrun:
self.stdout.write("\n---------------------------------")
if any(expired_objects):
self.stdout.write("Expired objects to be deactivated:")
self.stdout.write("Expiry datetime\t\tModel\t\tSlug\t\tName")
self.stdout.write("---------------\t\t-----\t\t----\t\t----")
for queryset in expired_objects:
if queryset.model is Page:
for obj in queryset:
self.stdout.write(
"{}\t{}\t{}\t{}".format(
obj.expire_at.strftime("%Y-%m-%d %H:%M"),
obj.specific_class.__name__,
obj.slug,
obj.title,
)
)
else:
for obj in queryset:
self.stdout.write(
"{}\t{}\t{}\t\t{}".format(
obj.expire_at.strftime("%Y-%m-%d %H:%M"),
queryset.model.__name__,
"",
str(obj),
)
)
else:
self.stdout.write("No expired objects to be deactivated found.")
else:
# Unpublish the expired objects
for queryset in expired_objects:
# Cast to list to make sure the query is fully evaluated
# before unpublishing anything
for obj in list(queryset):
obj.unpublish(
set_expired=True, log_action="wagtail.unpublish.scheduled"
)
# 2. get all revisions that need to be published
revs_for_publishing = Revision.objects.filter(
approved_go_live_at__lt=timezone.now()
).order_by("approved_go_live_at")
if dryrun:
self.stdout.write("\n---------------------------------")
if revs_for_publishing:
self.stdout.write("Revisions to be published:")
self.stdout.write("Go live datetime\tModel\t\tSlug\t\tName")
self.stdout.write("----------------\t-----\t\t----\t\t----")
for rp in revs_for_publishing:
model = rp.content_type.model_class()
rev_data = rp.content
self.stdout.write(
"{}\t{}\t{}\t\t{}".format(
rp.approved_go_live_at.strftime("%Y-%m-%d %H:%M"),
model.__name__,
rev_data.get("slug", ""),
rev_data.get("title", rp.object_str),
)
)
else:
self.stdout.write("No objects to go live.")
else:
for rp in revs_for_publishing:
# just run publish for the revision -- since the approved go
# live datetime is before now it will make the object live
rp.publish(log_action="wagtail.publish.scheduled")

View File

@@ -0,0 +1,9 @@
from wagtail.management.commands.publish_scheduled import (
Command as PublishScheduledCommand,
)
class Command(PublishScheduledCommand):
"""
Alias for the publish_scheduled management command for backwards-compatibility.
"""

View File

@@ -0,0 +1,20 @@
from django.core.management.base import BaseCommand
from wagtail.embeds.models import Embed
class Command(BaseCommand):
help = "Deletes all of the Embed model objects"
def handle(self, *args, **options):
embeds = Embed.objects.all()
deleted_embeds_count = embeds.delete()[0]
if deleted_embeds_count:
self.stdout.write(
self.style.SUCCESS(
f"Successfully deleted {deleted_embeds_count} embeds"
)
)
else:
self.stdout.write("Successfully deleted 0 embeds")

View File

@@ -0,0 +1,93 @@
from django.conf import settings
from django.core.management.base import BaseCommand
from django.db.models import Q
from django.db.models.deletion import ProtectedError
from django.utils import timezone
from wagtail.models import Revision, WorkflowState
class Command(BaseCommand):
help = "Delete revisions which are not the latest revision, published or scheduled to be published, or in moderation"
def add_arguments(self, parser):
parser.add_argument(
"--days",
type=int,
help="Only delete revisions older than this number of days",
)
parser.add_argument(
"--pages",
action="store_true",
help="Only delete revisions of page models",
)
parser.add_argument(
"--non-pages",
action="store_true",
help="Only delete revisions of non-page models",
)
def handle(self, *args, **options):
days = options.get("days")
pages = options.get("pages")
non_pages = options.get("non_pages")
revisions_deleted, protected_error_count = purge_revisions(
days=days, pages=pages, non_pages=non_pages
)
if revisions_deleted:
self.stdout.write(
self.style.SUCCESS(
"Successfully deleted %s revisions" % revisions_deleted
)
)
self.stdout.write(
self.style.SUCCESS(
"Ignored %s revisions because one or more protected relations exist that prevent deletion."
% protected_error_count
)
)
else:
self.stdout.write("No revisions deleted")
def purge_revisions(days=None, pages=True, non_pages=True):
if pages == non_pages:
# If both are True or both are False, purge revisions of pages and non-pages
objects = Revision.objects.all()
elif pages:
objects = Revision.objects.page_revisions()
elif non_pages:
objects = Revision.objects.not_page_revisions()
purgeable_revisions = objects.exclude(
# and exclude revisions with an approved_go_live_at date
approved_go_live_at__isnull=False
)
if getattr(settings, "WAGTAIL_WORKFLOW_ENABLED", True):
purgeable_revisions = purgeable_revisions.exclude(
# and exclude revisions linked to an in progress or needs changes workflow state
Q(task_states__workflow_state__status=WorkflowState.STATUS_IN_PROGRESS)
| Q(task_states__workflow_state__status=WorkflowState.STATUS_NEEDS_CHANGES)
)
if days:
purgeable_until = timezone.now() - timezone.timedelta(days=days)
# only include revisions which were created before the cut off date
purgeable_revisions = purgeable_revisions.filter(created_at__lt=purgeable_until)
deleted_revisions_count = 0
protected_error_count = 0
for revision in purgeable_revisions.iterator():
# don't delete the latest revision
if not revision.is_latest_revision():
try:
revision.delete()
deleted_revisions_count += 1
except ProtectedError:
protected_error_count += 1
return deleted_revisions_count, protected_error_count

View File

@@ -0,0 +1,106 @@
from django.apps import apps
from django.core.management.base import BaseCommand
from django.db import transaction
from wagtail.models import ReferenceIndex
from wagtail.signal_handlers import disable_reference_index_auto_update
DEFAULT_CHUNK_SIZE = 1000
class Command(BaseCommand):
def write(self, *args, **kwargs):
"""
Helper function that writes based on verbosity parameter
"""
if self.verbosity != 0:
self.stdout.write(*args, **kwargs)
def add_arguments(self, parser):
parser.add_argument(
"--chunk_size",
action="store",
dest="chunk_size",
default=DEFAULT_CHUNK_SIZE,
type=int,
help="Set number of records to be fetched at once for inserting into the index",
)
def handle(self, **options):
self.verbosity = options["verbosity"]
chunk_size = options.get("chunk_size")
object_count = 0
self.write("Rebuilding reference index")
with transaction.atomic():
with disable_reference_index_auto_update():
# Use `_raw_delete` to avoid loading instances into memory
all_references = ReferenceIndex.objects.all()
all_references._raw_delete(using=all_references.db)
for model in apps.get_models():
if not ReferenceIndex.is_indexed(model):
continue
self.write(str(model))
# Add items (chunk_size at a time)
for chunk in self.print_iter_progress(
self.queryset_chunks(model.objects.all().order_by("pk"), chunk_size)
):
for instance in chunk:
ReferenceIndex.create_or_update_for_object(instance)
object_count += len(chunk)
self.print_newline()
self.write("Indexed %d objects" % object_count)
self.print_newline()
def print_newline(self):
self.write("")
def print_iter_progress(self, iterable):
"""
Print a progress meter while iterating over an iterable. Use it as part
of a ``for`` loop::
for item in self.print_iter_progress(big_long_list):
self.do_expensive_computation(item)
A ``.`` character is printed for every value in the iterable,
a space every 10 items, and a new line every 50 items.
"""
for i, value in enumerate(iterable, start=1):
yield value
self.write(".", ending="")
if i % 40 == 0:
self.print_newline()
self.write(" " * 35, ending="")
elif i % 10 == 0:
self.write(" ", ending="")
self.stdout.flush()
# Atomic so the count of models doesn't change as it is iterated
@transaction.atomic
def queryset_chunks(self, qs, chunk_size=DEFAULT_CHUNK_SIZE):
"""
Yield a queryset in chunks of at most ``chunk_size``. The chunk yielded
will be a list, not a queryset. Iterating over the chunks is done in a
transaction so that the order and count of items in the queryset
remains stable.
"""
i = 0
while True:
items = list(qs[i * chunk_size :][:chunk_size])
if not items:
break
yield items
i += 1

View File

@@ -0,0 +1,17 @@
from django.core.management.base import BaseCommand
from wagtail.models import Page
class Command(BaseCommand):
help = "Resets url_path fields on each page recursively"
def set_subtree(self, root, parent=None):
root.set_url_path(parent)
root.save(update_fields=["url_path"])
for child in root.get_children():
self.set_subtree(child, root)
def handle(self, *args, **options):
for node in Page.get_root_nodes():
self.set_subtree(node)

View File

@@ -0,0 +1,36 @@
from django.apps import apps
from django.contrib.contenttypes.models import ContentType
from django.core.management.base import BaseCommand
from wagtail.models import ReferenceIndex
def model_name(model):
return f"{model.__module__}.{model.__name__}"
class Command(BaseCommand):
def handle(self, **options):
self.stdout.write("Reference index entries:")
object_count = 0
for model in sorted(apps.get_models(), key=lambda m: model_name(m)):
if not ReferenceIndex.is_indexed(model):
continue
content_types = [
ContentType.objects.get_for_model(
model_or_object, for_concrete_model=False
)
for model_or_object in ([model] + model._meta.get_parent_list())
]
content_type = content_types[0]
base_content_type = content_types[-1]
count = ReferenceIndex.objects.filter(
content_type=content_type, base_content_type=base_content_type
).count()
self.stdout.write(f"{count:>6} {model_name(model)}")
object_count += count
self.stdout.write(f"Total entries: {object_count}")