Add ee cleanup tests

* Adds cleanup tests to the live test.

awx/main/tasks/receptor.py

@@ -228,22 +228,24 @@ class RemoteJobError(RuntimeError):
     pass
 
 
-def run_until_complete(node, timing_data=None, **kwargs):
+def run_until_complete(node, timing_data=None, worktype='ansible-runner', ttl='20s', **kwargs):
     """
     Runs an ansible-runner work_type on remote node, waits until it completes, then returns stdout.
     """
+
     config_data = read_receptor_config()
     receptor_ctl = get_receptor_ctl(config_data)
 
     use_stream_tls = getattr(get_conn_type(node, receptor_ctl), 'name', None) == "STREAMTLS"
     kwargs.setdefault('tlsclient', get_tls_client(config_data, use_stream_tls))
-    kwargs.setdefault('ttl', '20s')
+    if ttl is not None:
+        kwargs['ttl'] = ttl
     kwargs.setdefault('payload', '')
     if work_signing_enabled(config_data):
         kwargs['signwork'] = True
 
     transmit_start = time.time()
-    result = receptor_ctl.submit_work(worktype='ansible-runner', node=node, **kwargs)
+    result = receptor_ctl.submit_work(worktype=worktype, node=node, **kwargs)
 
     unit_id = result['unitid']
     run_start = time.time()
@@ -371,7 +373,7 @@ def _convert_args_to_cli(vargs):
     return args
 
 
-def worker_cleanup(node_name, vargs, timeout=300.0):
+def worker_cleanup(node_name, vargs):
     args = _convert_args_to_cli(vargs)
 
     remote_command = ' '.join(args)

awx/main/tasks/system.py

@@ -25,6 +25,7 @@
 from django.utils.translation import gettext_noop
 from django.core.cache import cache
 from django.core.exceptions import ObjectDoesNotExist
+from django.db.models.query import QuerySet
 
 # Django-CRUM
 from crum import impersonate
@@ -379,7 +380,21 @@ def purge_old_stdout_files():
             logger.debug("Removing {}".format(os.path.join(settings.JOBOUTPUT_ROOT, f)))
 
 
-def _cleanup_images_and_files(**kwargs):
+class CleanupImagesAndFilesHelper:
+    @classmethod
+    def get_first_control_instance(cls) -> Instance | None:
+        return (
+            Instance.objects.filter(node_type__in=['hybrid', 'control'], node_state=Instance.States.READY, enabled=True, capacity__gt=0)
+            .order_by('-hostname')
+            .first()
+        )
+
+    @classmethod
+    def get_execution_instances(cls) -> QuerySet[Instance]:
+        return Instance.objects.filter(node_type='execution', node_state=Instance.States.READY, enabled=True, capacity__gt=0)
+
+
+def _cleanup_images_and_files(worktype='ansible-runner', ttl='20s', **kwargs):
     if settings.IS_K8S:
         return
     this_inst = Instance.objects.me()
@@ -394,13 +409,10 @@ def _cleanup_images_and_files(**kwargs):
             logger.info(f'Performed local cleanup with kwargs {kwargs}, output:\n{stdout}')
 
     # if we are the first instance alphabetically, then run cleanup on execution nodes
-    checker_instance = (
-        Instance.objects.filter(node_type__in=['hybrid', 'control'], node_state=Instance.States.READY, enabled=True, capacity__gt=0)
-        .order_by('-hostname')
-        .first()
-    )
+    checker_instance = CleanupImagesAndFilesHelper.get_first_control_instance()
+
     if checker_instance and this_inst.hostname == checker_instance.hostname:
-        for inst in Instance.objects.filter(node_type='execution', node_state=Instance.States.READY, enabled=True, capacity__gt=0):
+        for inst in CleanupImagesAndFilesHelper.get_execution_instances():
             runner_cleanup_kwargs = inst.get_cleanup_task_kwargs(**kwargs)
             if not runner_cleanup_kwargs:
                 continue

awx/main/tests/live/tests/conftest.py

@@ -1,7 +1,10 @@
+import subprocess
 import time
 
 import pytest
 
+from unittest import mock
+
 # These tests are invoked from the awx/main/tests/live/ subfolder
 # so any fixtures from higher-up conftest files must be explicitly included
 from awx.main.tests.functional.conftest import *  # noqa
@@ -59,3 +62,20 @@ def default_org():
 def demo_inv(default_org):
     inventory, _ = Inventory.objects.get_or_create(name='Demo Inventory', defaults={'organization': default_org})
     return inventory
+
+
+@pytest.fixture
+def podman_image_generator():
+    """
+    Generate a tagless podman image from awx base EE
+    """
+
+    def fn():
+        dockerfile = """
+        FROM quay.io/ansible/awx-ee:latest
+        RUN echo "Hello, Podman!" > /tmp/hello.txt
+        """
+        cmd = ['podman', 'build', '-f', '-']  # Create an image without a tag
+        subprocess.run(cmd, capture_output=True, input=dockerfile, text=True, check=True)
+
+    return fn

awx/main/tests/live/tests/test_cleanup_task.py

@@ -1,9 +1,19 @@
+from contextlib import nullcontext
 import os
+import json
+import pytest
 import tempfile
 import subprocess
+from unittest import mock
 
-from awx.main.tasks.receptor import _convert_args_to_cli
+from awx.main.tasks.receptor import _convert_args_to_cli, run_until_complete
 from awx.main.models import Instance, JobTemplate
+from awx.main.tasks.system import _cleanup_images_and_files
+
+
+def get_podman_images():
+    cmd = ['podman', 'images', '--format', 'json']
+    return json.loads((subprocess.run(cmd, capture_output=True, text=True, check=True)).stdout)
 
 
 def test_folder_cleanup_multiple_running_jobs_execution_node(request):
@@ -37,3 +47,34 @@ def delete_jobs():
     print('ansible-runner worker ' + remote_command)
 
     assert [os.path.exists(job_dir) for job_dir in job_dirs] == [True for i in range(3)]
+
+
+@pytest.mark.parametrize(
+    'worktype',
+    ('remote', 'local'),
+)
+def test_tagless_image(podman_image_generator, worktype):
+    """
+    Ensure podman images on Control and Hybrid nodes are deleted during cleanup.
+    """
+    podman_image_generator()
+
+    dangling_image = next((image for image in get_podman_images() if image.get('Dangling', False)), None)
+    assert dangling_image
+
+    with (
+        (
+            mock.patch('awx.main.tasks.receptor.run_until_complete', lambda *args, **kwargs: run_until_complete(*args, worktype='local', ttl=None, **kwargs))
+            if worktype == 'local'
+            else nullcontext()
+        ),
+        (
+            mock.patch('awx.main.tasks.system.CleanupImagesAndFilesHelper.get_execution_instances', lambda: [Instance.objects.me()])
+            if worktype == 'local'
+            else nullcontext()
+        ),
+    ):
+        _cleanup_images_and_files(image_prune=True)
+
+    for image in get_podman_images():
+        assert image['Id'] != dangling_image['Id']