feat(deployment)!: Migrate package orchestration to Docker Compose (resolves #1177).

components/clp-package-utils/clp_package_utils/general.py

@@ -1,5 +1,6 @@
 import enum
 import errno
+import json
 import os
 import pathlib
 import re
@@ -15,6 +16,9 @@
     CLP_DEFAULT_CREDENTIALS_FILE_PATH,
     CLP_SHARED_CONFIG_FILENAME,
     CLPConfig,
+    CONTAINER_AWS_CONFIG_DIRECTORY,
+    CONTAINER_CLP_HOME,
+    CONTAINER_INPUT_LOGS_ROOT_DIR,
     DB_COMPONENT_NAME,
     QueryEngine,
     QUEUE_COMPONENT_NAME,
@@ -42,12 +46,6 @@
 EXTRACT_IR_CMD = "i"
 EXTRACT_JSON_CMD = "j"
 
-# Paths
-CONTAINER_AWS_CONFIG_DIRECTORY = pathlib.Path("/") / ".aws"
-CONTAINER_CLP_HOME = pathlib.Path("/") / "opt" / "clp"
-CONTAINER_INPUT_LOGS_ROOT_DIR = pathlib.Path("/") / "mnt" / "logs"
-CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH = pathlib.Path("etc") / "clp-config.yml"
-
 DOCKER_MOUNT_TYPE_STRINGS = ["bind"]
 
 
@@ -132,59 +130,58 @@ def generate_container_name(job_type: str) -> str:
     return f"clp-{job_type}-{str(uuid.uuid4())[-4:]}"
 
 
-def check_dependencies():
+def is_docker_compose_running(project_name: str) -> bool:
+    """
+    Checks if a Docker Compose project is running.
+
+    :param project_name:
+    :return: True if at least one instance is running, else False.
+    :raises EnvironmentError: If Docker Compose is not installed or fails.
+    """
+    cmd = ["docker", "compose", "ls", "--format", "json", "--filter", f"name={project_name}"]
+    try:
+        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+        running_instances = json.loads(output)
+        return len(running_instances) >= 1
+    except subprocess.CalledProcessError:
+        raise EnvironmentError("docker-compose is not installed or not functioning properly.")
+
+
+def check_docker_dependencies(should_compose_run: bool, project_name: str):
+    """
+    Checks if Docker and Docker Compose are installed, and whether Docker Compose is running or not.
+
+    :param should_compose_run:
+    :param project_name: The Docker Compose project name to check.
+    :raises EnvironmentError: If any Docker dependency is not installed or Docker Compose state
+    does not match expectation.
+    """
     try:
         subprocess.run(
             "command -v docker",
             shell=True,
-            stdout=subprocess.PIPE,
+            stdout=subprocess.DEVNULL,
             stderr=subprocess.STDOUT,
             check=True,
         )
     except subprocess.CalledProcessError:
         raise EnvironmentError("docker is not installed or available on the path")
-    try:
-        subprocess.run(
-            ["docker", "ps"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True
-        )
-    except subprocess.CalledProcessError:
-        raise EnvironmentError("docker cannot run without superuser privileges (sudo).")
-
-
-def is_container_running(container_name):
-    # fmt: off
-    cmd = [
-        "docker", "ps",
-        # Only return container IDs
-        "--quiet",
-        "--filter", f"name={container_name}"
-    ]
-    # fmt: on
-    proc = subprocess.run(cmd, stdout=subprocess.PIPE)
-    if proc.stdout.decode("utf-8"):
-        return True
-
-    return False
 
+    is_running = is_docker_compose_running(project_name)
+    if should_compose_run and not is_running:
+        raise EnvironmentError("docker-compose is not running.")
+    if not should_compose_run and is_running:
+        raise EnvironmentError("docker-compose is already running.")
 
-def is_container_exited(container_name):
-    # fmt: off
-    cmd = [
-        "docker", "ps",
-        # Only return container IDs
-        "--quiet",
-        "--filter", f"name={container_name}",
-        "--filter", "status=exited"
-    ]
-    # fmt: on
-    proc = subprocess.run(cmd, stdout=subprocess.PIPE)
-    if proc.stdout.decode("utf-8"):
-        return True
-
-    return False
 
+def _validate_log_directory(logs_dir: pathlib.Path, component_name: str):
+    """
+    Validate that a log directory path of a component is valid.
 
-def validate_log_directory(logs_dir: pathlib.Path, component_name: str) -> None:
+    :param logs_dir:
+    :param component_name:
+    :raises ValueError: If the path is invalid or not a directory.
+    """
     try:
         validate_path_could_be_dir(logs_dir)
     except ValueError as ex:
@@ -309,6 +306,19 @@ def generate_container_config(
     return container_clp_config, docker_mounts
 
 
+def generate_docker_compose_container_config(clp_config: CLPConfig) -> CLPConfig:
+    """
+    Copies the given config and transforms mount paths and hosts for Docker Compose.
+
+    :param clp_config:
+    :return: The container config and the mounts.
+    """
+    container_clp_config = clp_config.model_copy(deep=True)
+    container_clp_config.transform_for_container()
+
+    return container_clp_config
+
+
 def generate_worker_config(clp_config: CLPConfig) -> WorkerConfig:
     worker_config = WorkerConfig()
     worker_config.package = clp_config.package.model_copy(deep=True)
@@ -431,11 +441,6 @@ def load_config_file(
     validate_path_for_container_mount(clp_config.data_directory)
     validate_path_for_container_mount(clp_config.logs_directory)
 
-    # Make data and logs directories node-specific
-    hostname = socket.gethostname()
-    clp_config.data_directory /= hostname
-    clp_config.logs_directory /= hostname
-
     return clp_config
 
 
@@ -488,35 +493,40 @@ def validate_and_load_redis_credentials_file(
     clp_config.redis.load_credentials_from_file(clp_config.credentials_file_path)
 
 
-def validate_db_config(clp_config: CLPConfig, data_dir: pathlib.Path, logs_dir: pathlib.Path):
+def validate_db_config(
+    clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path
+):
+    if not base_config.exists():
+        raise ValueError(
+            f"{DB_COMPONENT_NAME} base configuration at {str(base_config)} is missing."
+        )
     _validate_data_directory(data_dir, DB_COMPONENT_NAME)
-    validate_log_directory(logs_dir, DB_COMPONENT_NAME)
+    _validate_log_directory(logs_dir, DB_COMPONENT_NAME)
 
     validate_port(f"{DB_COMPONENT_NAME}.port", clp_config.database.host, clp_config.database.port)
 
 
 def validate_queue_config(clp_config: CLPConfig, logs_dir: pathlib.Path):
-    validate_log_directory(logs_dir, QUEUE_COMPONENT_NAME)
+    _validate_log_directory(logs_dir, QUEUE_COMPONENT_NAME)
 
     validate_port(f"{QUEUE_COMPONENT_NAME}.port", clp_config.queue.host, clp_config.queue.port)
 
 
 def validate_redis_config(
-    clp_config: CLPConfig, data_dir: pathlib.Path, logs_dir: pathlib.Path, base_config: pathlib.Path
+    clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path
 ):
-    _validate_data_directory(data_dir, REDIS_COMPONENT_NAME)
-    validate_log_directory(logs_dir, REDIS_COMPONENT_NAME)
-
     if not base_config.exists():
         raise ValueError(
             f"{REDIS_COMPONENT_NAME} base configuration at {str(base_config)} is missing."
         )
+    _validate_data_directory(data_dir, REDIS_COMPONENT_NAME)
+    _validate_log_directory(logs_dir, REDIS_COMPONENT_NAME)
 
     validate_port(f"{REDIS_COMPONENT_NAME}.port", clp_config.redis.host, clp_config.redis.port)
 
 
 def validate_reducer_config(clp_config: CLPConfig, logs_dir: pathlib.Path, num_workers: int):
-    validate_log_directory(logs_dir, REDUCER_COMPONENT_NAME)
+    _validate_log_directory(logs_dir, REDUCER_COMPONENT_NAME)
 
     for i in range(0, num_workers):
         validate_port(
@@ -527,10 +537,14 @@ def validate_reducer_config(clp_config: CLPConfig, logs_dir: pathlib.Path, num_w
 
 
 def validate_results_cache_config(
-    clp_config: CLPConfig, data_dir: pathlib.Path, logs_dir: pathlib.Path
+    clp_config: CLPConfig, base_config: pathlib.Path, data_dir: pathlib.Path, logs_dir: pathlib.Path
 ):
+    if not base_config.exists():
+        raise ValueError(
+            f"{RESULTS_CACHE_COMPONENT_NAME} base configuration at {str(base_config)} is missing."
+        )
     _validate_data_directory(data_dir, RESULTS_CACHE_COMPONENT_NAME)
-    validate_log_directory(logs_dir, RESULTS_CACHE_COMPONENT_NAME)
+    _validate_log_directory(logs_dir, RESULTS_CACHE_COMPONENT_NAME)
 
     validate_port(
         f"{RESULTS_CACHE_COMPONENT_NAME}.port",

components/clp-package-utils/clp_package_utils/scripts/archive_manager.py

@@ -9,13 +9,13 @@
 from clp_py_utils.clp_config import (
     CLP_DB_PASS_ENV_VAR_NAME,
     CLP_DB_USER_ENV_VAR_NAME,
+    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     CLP_DEFAULT_DATASET_NAME,
     StorageEngine,
     StorageType,
 )
 
 from clp_package_utils.general import (
-    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     CLPConfig,
     DockerMount,
     dump_container_config,

components/clp-package-utils/clp_package_utils/scripts/compress.py

@@ -10,13 +10,13 @@
 from clp_py_utils.clp_config import (
     CLP_DB_PASS_ENV_VAR_NAME,
     CLP_DB_USER_ENV_VAR_NAME,
+    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     CLP_DEFAULT_DATASET_NAME,
     StorageEngine,
 )
 from job_orchestration.scheduler.job_config import InputType
 
 from clp_package_utils.general import (
-    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     CONTAINER_INPUT_LOGS_ROOT_DIR,
     dump_container_config,
     generate_container_config,

components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py

@@ -10,13 +10,13 @@
     ARCHIVE_MANAGER_ACTION_NAME,
     CLP_DB_PASS_ENV_VAR_NAME,
     CLP_DB_USER_ENV_VAR_NAME,
+    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     StorageEngine,
     StorageType,
 )
 from clp_py_utils.s3_utils import generate_container_auth_options
 
 from clp_package_utils.general import (
-    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     dump_container_config,
     generate_container_config,
     generate_container_name,

components/clp-package-utils/clp_package_utils/scripts/decompress.py

@@ -9,14 +9,14 @@
 from clp_py_utils.clp_config import (
     CLP_DB_PASS_ENV_VAR_NAME,
     CLP_DB_USER_ENV_VAR_NAME,
+    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     CLP_DEFAULT_DATASET_NAME,
     CLPConfig,
     StorageEngine,
     StorageType,
 )
 
 from clp_package_utils.general import (
-    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     DockerMount,
     DockerMountType,
     dump_container_config,

components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py

@@ -7,15 +7,14 @@
 from pathlib import Path
 from typing import Any, List, Optional
 
-from clp_py_utils.clp_config import Database
+from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, Database
 from clp_py_utils.clp_metadata_db_utils import (
     delete_archives_from_metadata_db,
     get_archives_table_name,
 )
 from clp_py_utils.sql_adapter import SQL_Adapter
 
 from clp_package_utils.general import (
-    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     CLPConfig,
     get_clp_home,
     load_config_file,

components/clp-package-utils/clp_package_utils/scripts/native/compress.py

@@ -10,6 +10,7 @@
 import brotli
 import msgpack
 from clp_py_utils.clp_config import (
+    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     CLPConfig,
     COMPRESSION_JOBS_TABLE_NAME,
 )
@@ -29,7 +30,6 @@
 )
 
 from clp_package_utils.general import (
-    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     CONTAINER_INPUT_LOGS_ROOT_DIR,
     get_clp_home,
     load_config_file,

components/clp-package-utils/clp_package_utils/scripts/native/dataset_manager.py

@@ -6,7 +6,13 @@
 from pathlib import Path
 from typing import Dict, List
 
-from clp_py_utils.clp_config import ArchiveOutput, Database, S3Config, StorageType
+from clp_py_utils.clp_config import (
+    ArchiveOutput,
+    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
+    Database,
+    S3Config,
+    StorageType,
+)
 from clp_py_utils.clp_metadata_db_utils import (
     delete_dataset_from_metadata_db,
     get_datasets_table_name,
@@ -15,7 +21,6 @@
 from clp_py_utils.sql_adapter import SQL_Adapter
 
 from clp_package_utils.general import (
-    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     CLPConfig,
     get_clp_home,
     load_config_file,

components/clp-package-utils/clp_package_utils/scripts/native/decompress.py

@@ -12,6 +12,7 @@
 from clp_py_utils.clp_config import (
     CLP_DB_PASS_ENV_VAR_NAME,
     CLP_DB_USER_ENV_VAR_NAME,
+    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     CLPConfig,
     Database,
 )
@@ -25,7 +26,6 @@
 )
 
 from clp_package_utils.general import (
-    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     EXTRACT_FILE_CMD,
     EXTRACT_IR_CMD,
     EXTRACT_JSON_CMD,

components/clp-package-utils/clp_package_utils/scripts/native/search.py

@@ -12,6 +12,7 @@
 import psutil
 import pymongo
 from clp_py_utils.clp_config import (
+    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     Database,
     ResultsCache,
 )
@@ -20,7 +21,6 @@
 from job_orchestration.scheduler.job_config import AggregationConfig, SearchJobConfig
 
 from clp_package_utils.general import (
-    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     get_clp_home,
     load_config_file,
 )

components/clp-package-utils/clp_package_utils/scripts/search.py

@@ -8,13 +8,13 @@
 from clp_py_utils.clp_config import (
     CLP_DB_PASS_ENV_VAR_NAME,
     CLP_DB_USER_ENV_VAR_NAME,
+    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     CLP_DEFAULT_DATASET_NAME,
     StorageEngine,
     StorageType,
 )
 
 from clp_package_utils.general import (
-    CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     dump_container_config,
     generate_container_config,
     generate_container_name,