Release reports refactor (#1996)

config/celery.py

@@ -98,7 +98,7 @@ def setup_periodic_tasks(sender, **kwargs):
     # Update data required for release report. Executes Saturday evenings.
     sender.add_periodic_task(
         crontab(day_of_week="sat", hour=20, minute=3),
-        app.signature("libraries.tasks.release_tasks", generate_report=True),
+        app.signature("libraries.tasks.release_tasks", generate_report=False),
     )
 
     # Update users' profile photos from GitHub. Executes daily at 3:30 AM.

config/settings.py

@@ -427,6 +427,7 @@
         ]
 
 
+ACCOUNT_DEFAULT_HTTP_PROTOCOL = "http"
 if not LOCAL_DEVELOPMENT:
     ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https"
     SECURE_PROXY_SSL_HEADER = (
@@ -615,7 +616,8 @@
 # Required by Wagtail
 DATA_UPLOAD_MAX_NUMBER_FIELDS = 10_000
 WAGTAIL_SITE_NAME = "Boost.org"
-WAGTAILADMIN_BASE_URL = env("WAGTAILADMIN_BASE_URL", default="https://www.boost.org/")
+WAGTAILADMIN_BASE_URL = env("WAGTAILADMIN_BASE_URL", default="https://www.boost.org")
+WAGTAILADMIN_NOTIFICATION_INCLUDE_SUPERUSERS = False
 WAGTAILDOCS_EXTENSIONS = [
     "csv",
     "docx",

docs/release_reports.md

@@ -6,9 +6,9 @@
    1. Ask Sam for a copy of the "subscribe" data.
    2. In the Django admin interface go to "Subscription datas" under "MAILING_LIST".
    3. At the top of the page click on the "IMPORT 'SUBSCRIBE' DATA" button.
-2. To update the mailing list counts, if you haven't already run the "DO IT ALL" button:
-   1. Go to "Versions" under "VERSIONS" in the admin interface
-   2. At the top of the page click on the "DO IT ALL" button.
+2. To update the mailing list counts, if you haven't already run the "GET RELEASE REPORT DATA" button:
+   1. Go to "Release Reports" under "VERSIONS" in the admin interface
+   2. At the top of the page click on the "GET RELEASE REPORT DATA" button.
 
 ## Report Creation
 

kube/boost/values-cppal-dev-gke.yaml

@@ -202,7 +202,7 @@ Env:
   - name: ENABLE_DB_CACHE
     value: "true"
   - name: WAGTAILADMIN_BASE_URL
-    value: https://www.cppal-dev.boost.org/
+    value: https://www.cppal-dev.boost.org
 
 # Volumes
 Volumes:

kube/boost/values-production-gke.yaml

@@ -202,7 +202,7 @@ Env:
   - name: ENABLE_DB_CACHE
     value: "true"
   - name: WAGTAILADMIN_BASE_URL
-    value: https://www.boost.org/
+    value: https://www.boost.org
 
 # Volumes
 Volumes:

kube/boost/values-stage-gke.yaml

@@ -202,7 +202,7 @@ Env:
   - name: ENABLE_DB_CACHE
     value: "true"
   - name: WAGTAILADMIN_BASE_URL
-    value: https://www.stage.boost.org/
+    value: https://www.stage.boost.org
 
 # Volumes
 Volumes:

libraries/admin.py

@@ -1,5 +1,7 @@
-from django.contrib import admin
-from django.core.files.storage import default_storage
+import structlog
+from django.conf import settings
+from django.contrib import admin, messages
+from django.core.exceptions import ValidationError
 from django.db import transaction
 from django.db.models import F, Count, OuterRef, Window
 from django.db.models.functions import RowNumber
@@ -42,6 +44,9 @@
 from .utils import generate_release_report_filename
 
 
+logger = structlog.get_logger()
+
+
 @admin.register(Commit)
 class CommitAdmin(admin.ModelAdmin):
     list_display = ["library_version", "sha", "author"]
@@ -183,13 +188,39 @@ def get_context_data(self, **kwargs):
         return context
 
     def generate_report(self):
+        uri = f"{settings.ACCOUNT_DEFAULT_HTTP_PROTOCOL}://{self.request.get_host()}"
         generate_release_report.delay(
-            user_id=self.request.user.id, params=self.request.GET
+            user_id=self.request.user.id,
+            params=self.request.GET,
+            base_uri=uri,
         )
 
+    def locked_publish_check(self):
+        form = self.get_form()
+        form.is_valid()
+        publish = form.cleaned_data["publish"]
+        report_configuration = form.cleaned_data["report_configuration"]
+        if publish and ReleaseReport.latest_published_locked(report_configuration):
+            msg = (
+                f"A release report already exists with locked status for "
+                f"{report_configuration.display_name}. Delete or unlock the most "
+                f"recent report."
+            )
+            raise ValueError(msg)
+
     def get(self, request, *args, **kwargs):
         form = self.get_form()
         if form.is_valid():
+            try:
+                self.locked_publish_check()
+            except ValueError as e:
+                messages.error(request, str(e))
+                return TemplateResponse(
+                    request,
+                    self.form_template,
+                    self.get_context_data(),
+                )
+
             if form.cleaned_data["no_cache"]:
                 params = request.GET.copy()
                 form.cache_clear()
@@ -458,28 +489,92 @@ class Meta:
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        if self.instance.pk and not self.instance.published:
-            file_name = generate_release_report_filename(
-                self.instance.report_configuration.get_slug()
+        if not self.is_publish_editable():
+            # we require users to intentionally manually delete existing reports
+            self.fields["published"].disabled = True
+            self.fields["published"].help_text = (
+                "⚠️ A published PDF already exists for this Report Configuration. See "
+                '"Publishing" notes at the top of this page.'
             )
-            published_filename = f"{ReleaseReport.upload_dir}{file_name}"
-            if default_storage.exists(published_filename):
-                # we require users to intentionally manually delete existing reports
-                self.fields["published"].disabled = True
-                self.fields["published"].help_text = (
-                    f"⚠️ A published '{file_name}' already exists. To prevent accidents "
-                    "you must manually delete that file before publishing this report."
+
+    def is_publish_editable(self) -> bool:
+        # in play here are currently published and previously published rows because of
+        # filename collision risk.
+        if self.instance.published:
+            return True
+
+        published_filename = generate_release_report_filename(
+            version_slug=self.instance.report_configuration.get_slug(),
+            published_format=True,
+        )
+        reports = ReleaseReport.objects.filter(
+            report_configuration=self.instance.report_configuration,
+            file=f"{ReleaseReport.upload_dir}{published_filename}",
+        )
+
+        if reports.count() == 0 or reports.latest("created_at") == self.instance:
+            return True
+
+        return False
+
+    def clean(self):
+        cleaned_data = super().clean()
+        if not self.is_publish_editable():
+            raise ValidationError("This file is not publishable.")
+        if cleaned_data.get("published"):
+            report_configuration = cleaned_data.get("report_configuration")
+            if ReleaseReport.latest_published_locked(
+                report_configuration, self.instance
+            ):
+                raise ValidationError(
+                    f"A release report already exists with locked status for "
+                    f"{report_configuration.display_name}. Delete or unlock the most "
+                    f"recent report."
                 )
 
+        return cleaned_data
+
 
 @admin.register(ReleaseReport)
 class ReleaseReportAdmin(admin.ModelAdmin):
     form = ReleaseReportAdminForm
-    list_display = ["__str__", "created_at", "published", "published_at"]
-    list_filter = ["published", ReportConfigurationFilter, StaffUserCreatedByFilter]
+    list_display = ["__str__", "created_at", "published", "published_at", "locked"]
+    list_filter = [
+        "published",
+        "locked",
+        ReportConfigurationFilter,
+        StaffUserCreatedByFilter,
+    ]
     search_fields = ["file"]
     readonly_fields = ["created_at", "created_by"]
     ordering = ["-created_at"]
+    change_list_template = "admin/releasereport_change_list.html"
+    change_form_template = "admin/releasereport_change_form.html"
+
+    def get_urls(self):
+        urls = super().get_urls()
+        my_urls = [
+            path(
+                "release_tasks/",
+                self.admin_site.admin_view(self.release_tasks),
+                name="release_tasks",
+            ),
+        ]
+        return my_urls + urls
+
+    def release_tasks(self, request):
+        from libraries.tasks import release_tasks
+
+        release_tasks.delay(
+            base_uri=f"{settings.ACCOUNT_DEFAULT_HTTP_PROTOCOL}://{request.get_host()}",
+            user_id=request.user.id,
+            generate_report=False,
+        )
+        self.message_user(
+            request,
+            "release_tasks has started, you will receive an email when the task finishes.",  # noqa: E501
+        )
+        return HttpResponseRedirect("../")
 
     def has_add_permission(self, request):
         return False
@@ -488,3 +583,25 @@ def save_model(self, request, obj, form, change):
         if not change:
             obj.created_by = request.user
         super().save_model(request, obj, form, change)
+
+    @staticmethod
+    def clear_other_report_files(release_report: ReleaseReport):
+        if release_report.file:
+            other_reports = ReleaseReport.objects.filter(
+                file=release_report.file.name
+            ).exclude(pk=release_report.pk)
+
+            if other_reports.exists():
+                release_report.file = None
+                release_report.save()
+
+    def delete_model(self, request, obj):
+        # check if another report uses the same file
+        self.clear_other_report_files(obj)
+        super().delete_model(request, obj)
+
+    def delete_queryset(self, request, queryset):
+        # clear file reference, prevents deletion of the file if it's linked elsewhere
+        for obj in queryset:
+            self.clear_other_report_files(obj)
+        super().delete_queryset(request, queryset)

libraries/constants.py

@@ -367,3 +367,5 @@
 VERSION_SLUG_PREFIX = "boost-"
 RELEASE_REPORT_SEARCH_TOP_COUNTRIES_LIMIT = 5
 DOCKER_CONTAINER_URL_WEB = "http://web:8000"
+
+RELEASE_REPORT_AUTHORS_PER_PAGE_THRESHOLD = 6