Add "Import new releases" emails (#1872)

core/management/actions.py

@@ -0,0 +1,87 @@
+from abc import ABCMeta, abstractmethod
+from dataclasses import dataclass
+from typing import Callable
+
+import djclick as click
+from django.core.mail import send_mail
+from django.core.management import call_command
+from django.utils import timezone
+
+from config import settings
+
+
+def progress_message(message: str):
+    click.secho(message, fg="green")
+    return f"{timezone.now()}: {message}"
+
+
+@dataclass
+class Action:
+    """
+    A distinct task to be completed.
+
+    Action can be a callable or a list of string arguments to pass to `call_command`
+    """
+
+    description: str
+    handler: Callable | list[str]
+
+    @property
+    def handler_name(self) -> str:
+        if isinstance(self.handler, Callable):
+            return f"function: {self.handler.__name__}"
+        return f"command: {self.handler[0]}"
+
+    def run(self):
+        if isinstance(self.handler, Callable):
+            self.handler()
+        else:
+            call_command(*self.handler)
+
+
+class ActionsManager(metaclass=ABCMeta):
+    progress_messages: list[str] = []
+
+    def __init__(self):
+        self.tasks: list[Action] = []
+        self.set_tasks()
+        self.validate_tasks()
+
+    @abstractmethod
+    def set_tasks(self):
+        """
+        Set self.tasks to a list of Action instances.
+        self.tasks = [Action(...), Action(...)]
+        """
+        raise NotImplementedError
+
+    def validate_tasks(self):
+        if not self.tasks:
+            raise ValueError("No tasks defined. You must set some with set_tasks()")
+        if not all(isinstance(task, Action) for task in self.tasks):
+            raise TypeError("All tasks must be instances of Action")
+
+    def add_progress_message(self, message: str):
+        message = progress_message(message)
+        self.progress_messages.append(message)
+
+    def run_tasks(self) -> dict[str:int]:
+        for task in self.tasks:
+            # "Task: " prefix for easy log parsing
+            self.add_progress_message(
+                f"Task start - {task.handler_name}, desc: {task.description.lower()}..."
+            )
+            task.run()
+            self.add_progress_message(
+                f"Task done - {task.handler_name}, desc: {task.description.lower()}"
+            )
+
+
+def send_notification(user, message, subject):
+    if user and user.email:
+        send_mail(
+            subject,
+            message,
+            settings.DEFAULT_FROM_EMAIL,
+            [user.email],
+        )

libraries/management/commands/release_tasks.py

@@ -1,19 +1,22 @@
 import traceback
 from contextlib import suppress
-from dataclasses import dataclass
 from datetime import timedelta
-from typing import Callable
 
 import djclick as click
 
-from django.core.mail import send_mail
 from django.utils import timezone
 from django.core.management import call_command
 from django.contrib.auth import get_user_model
 from django.conf import settings
 from slack_sdk.errors import SlackApiError
 
 from core.githubhelper import GithubAPIClient
+from core.management.actions import (
+    progress_message,
+    Action,
+    ActionsManager,
+    send_notification,
+)
 from libraries.forms import CreateReportForm
 from libraries.tasks import update_commits
 from reports.models import WebsiteStatReport
@@ -23,82 +26,39 @@
 User = get_user_model()
 
 
-def send_notification(user, message, subject="Task Started: release_tasks"):
-    if user.email:
-        send_mail(
-            subject,
-            message,
-            settings.DEFAULT_FROM_EMAIL,
-            [user.email],
-        )
-
-
-def progress_message(message: str):
-    click.secho(message, fg="green")
-    return f"{timezone.now()}: {message}"
-
-
-@dataclass
-class ReleaseTask:
-    """
-    A distinct task to be completed.
-
-    Action can be a callable or a list of string arguments to pass to `call_command`
-    """
-
-    description: str
-    action: Callable | list[str]
-
-    def run(self):
-        if isinstance(self.action, Callable):
-            self.action()
-        else:
-            call_command(*self.action)
-
-
-class ReleaseTasksManager:
+class ReleaseTasksManager(ActionsManager):
     latest_version: Version | None = None
-    progress_messages: list[str] = []
     handled_commits: dict[str, int] = {}
 
     def __init__(self, should_generate_report: bool = False):
         self.should_generate_report = should_generate_report
+        super().__init__()
+
+    def set_tasks(self):
         self.tasks = [
-            ReleaseTask("Importing versions", self.import_versions),
-            ReleaseTask(
+            Action("Importing versions", self.import_versions),
+            Action(
                 "Importing most recent beta version",
                 ["import_beta_release", "--delete-versions"],
             ),
-            ReleaseTask("Importing libraries", ["update_libraries"]),
-            ReleaseTask(
+            Action("Importing libraries", ["update_libraries"]),
+            Action(
                 "Saving library-version relationships", self.import_library_versions
             ),
-            ReleaseTask("Adding library maintainers", ["update_maintainers"]),
-            ReleaseTask("Adding library authors", ["update_authors"]),
-            ReleaseTask(
+            Action("Adding library maintainers", ["update_maintainers"]),
+            Action("Adding library authors", ["update_authors"]),
+            Action(
                 "Adding library version authors", ["update_library_version_authors"]
             ),
-            ReleaseTask("Importing git commits", self.handle_commits),
-            ReleaseTask("Syncing mailinglist statistics", ["sync_mailinglist_stats"]),
-            ReleaseTask("Updating github issues", ["update_issues"]),
-            ReleaseTask("Updating slack activity buckets", ["fetch_slack_activity"]),
-            ReleaseTask("Updating website statistics", self.update_website_statistics),
-            ReleaseTask("Importing mailing list counts", self.import_ml_counts),
-            ReleaseTask("Generating report", self.generate_report),
+            Action("Importing git commits", self.import_commits),
+            Action("Syncing mailinglist statistics", ["sync_mailinglist_stats"]),
+            Action("Updating github issues", ["update_issues"]),
+            Action("Updating slack activity buckets", ["fetch_slack_activity"]),
+            Action("Updating website statistics", self.update_website_statistics),
+            Action("Importing mailing list counts", self.import_ml_counts),
+            Action("Generating report", self.generate_report),
         ]
 
-    def update_release_data(self) -> dict[str:int]:
-        for task in self.tasks:
-            # "Release Task: " prefix for easy log parsing
-            self.progress_messages.append(
-                progress_message(f"Release Task: {task.description}...")
-            )
-            task.run()
-            self.progress_messages.append(
-                progress_message(f"Release Task: Finished {task.description.lower()}")
-            )
-        return self.handled_commits
-
     def import_versions(self):
         call_command("import_versions")
         self.latest_version = Version.objects.most_recent()
@@ -107,7 +67,7 @@ def import_library_versions(self):
         latest_version_number = self.latest_version.name.lstrip("boost-")
         call_command("import_library_versions", min_release=latest_version_number)
 
-    def handle_commits(self):
+    def import_commits(self):
         self.handled_commits = update_commits(min_version=self.latest_version.name)
 
     def update_website_statistics(self):
@@ -125,9 +85,7 @@ def import_ml_counts(self):
 
     def generate_report(self):
         if not self.should_generate_report:
-            self.progress_messages.append(
-                progress_message("Skipped - report generation not requested")
-            )
+            self.add_progress_message("Skipped - report generation not requested")
             return
         form = CreateReportForm({"version": self.latest_version.id})
         form.cache_html()
@@ -136,11 +94,9 @@ def generate_report(self):
 @locked(1138692)
 def run_commands(progress: list[str], generate_report: bool = False):
     manager = ReleaseTasksManager(should_generate_report=generate_report)
-    handled_commits = manager.update_release_data()
-
+    manager.run_tasks()
     progress.extend(manager.progress_messages)
-
-    return handled_commits
+    return manager.handled_commits
 
 
 def bad_credentials() -> list[str]:
@@ -185,40 +141,38 @@ def command(user_id=None, generate_report=False):
     """A long running chain of tasks to import and update library data."""
     start = timezone.now()
 
-    user = None
-    if user_id:
-        user = User.objects.filter(id=user_id).first()
+    user = User.objects.filter(id=user_id).first() if user_id else None
 
     progress = ["___Progress Messages___"]
     if missing_creds := bad_credentials():
         progress.append(
             progress_message(f"Missing credentials {', '.join(missing_creds)}")
         )
-        if user:
-            send_notification(
-                user,
-                message="Your task `release_tasks` failed.",
-                subject="Task Failed: release_tasks",
-            )
+        send_notification(
+            user,
+            message="Your task `release_tasks` failed.",
+            subject="Task Failed: release_tasks",
+        )
         return
-    if user:
-        send_notification(user, f"Your task `release_tasks` was started at: {start}")
+
+    send_notification(
+        user,
+        f"Your task `release_tasks` was started at: {start}",
+        subject="Task Started: release_tasks",
+    )
 
     try:
         handled_commits = run_commands(progress, generate_report)
         end = timezone.now()
-        progress.append(progress_message(f"All done! Completed in {end - start}"))
     except Exception:
         error = traceback.format_exc()
         message = [
             f"ERROR: There was an error while running release_tasks.\n\n{error}",
             "\n".join(progress),
         ]
-        if user:
-            send_notification(
-                user,
-                "\n\n".join(message),
-            )
+        send_notification(
+            user, "\n\n".join(message), subject="Task Failed: release_tasks"
+        )
         raise
 
     zero_commit_libraries = [
@@ -236,9 +190,8 @@ def command(user_id=None, generate_report=False):
         for lib, _ in zero_commit_libraries:
             zero_commit_message.append(lib)
         message.append("\n".join(zero_commit_message))
-    if user:
-        send_notification(
-            user,
-            "\n\n".join(message),
-            subject="Task Complete: release_tasks",
-        )
+    send_notification(
+        user,
+        "\n\n".join(message),
+        subject="Task Complete: release_tasks",
+    )

libraries/tasks.py

@@ -264,6 +264,19 @@ def release_tasks(user_id=None, generate_report=False):
     call_command(*command)
 
 
+@app.task
+def import_new_versions_tasks(user_id=None):
+    """Call the import_new_versions management command.
+
+    If a user_id is given, that user will receive an email at the beginning
+    and at the end of the task.
+    """
+    command = ["import_new_versions"]
+    if user_id:
+        command.extend(["--user_id", user_id])
+    call_command(*command)
+
+
 @app.task
 def synchronize_commit_author_user_data():
     logger.info("Starting synchronize_commit_author_user_data")

versions/admin.py

@@ -4,14 +4,9 @@
 from django.http import HttpRequest, HttpResponseRedirect
 from django.urls import path
 
-from libraries.tasks import release_tasks
+from libraries.tasks import release_tasks, import_new_versions_tasks
 
 from . import models
-from .tasks import (
-    import_versions,
-    import_most_recent_beta_release,
-    import_development_versions,
-)
 
 
 class VersionFileInline(admin.StackedInline):
@@ -57,17 +52,9 @@ def release_tasks(self, request):
         return HttpResponseRedirect("../")
 
     def import_new_releases(self, request):
-        import_versions.delay(new_versions_only=True)
-        import_most_recent_beta_release.delay(delete_old=True)
-        # Import the master and develop branches
-        import_development_versions.delay()
-        self.message_user(
-            request,
-            """
-            New releases are being imported. If you don't see any new releases,
-            please refresh this page or check the logs.
-        """,
-        )
+        import_new_versions_tasks.delay(user_id=request.user.id)
+        msg = "New releases are being imported. You will receive an email when the task finishes."  # noqa: E501
+        self.message_user(request, msg)
         return HttpResponseRedirect("../")
 
 

versions/management/commands/import_development_versions.py

@@ -0,0 +1,16 @@
+import djclick as click
+
+from versions.tasks import import_development_versions
+
+
+@click.command()
+def command():
+    """
+    Import development versions from GitHub.
+
+    This command will import the master and develop branches as versions
+    from GitHub based on the BOOST_BRANCHES setting.
+    """
+    click.secho("Importing development versions...", fg="green")
+    import_development_versions()
+    click.secho("Finished importing development versions.", fg="green")