diff --git a/metrics_utility/automation_controller_billing/collector.py b/metrics_utility/automation_controller_billing/collector.py index 53537f59..43eca60b 100644 --- a/metrics_utility/automation_controller_billing/collector.py +++ b/metrics_utility/automation_controller_billing/collector.py @@ -30,17 +30,20 @@ def __init__(self, collection_type=base.Collector.SCHEDULED_COLLECTION, collecto collector_module = collectors self.ship_target = ship_target - self.billing_provider_params = billing_provider_params - super(Collector, self).__init__(collection_type=collection_type, collector_module=collector_module) + super(Collector, self).__init__( + collection_type=collection_type, + collector_module=collector_module, + billing_provider_params=billing_provider_params, + ) # TODO: extract advisory lock name in the superclass and log message, so we can change it here and then use # this method from superclass # TODO: extract to superclass ability to push extra params into config.json - def gather(self, dest=None, subset=None, since=None, until=None, billing_provider_params=None): + # FIXME: subset is only used for tests, mock registered collectors instead? + def gather(self, subset=None, since=None, until=None, billing_provider_params=None): """Entry point for gathering - :param dest: (default: /tmp/awx-analytics-*) - directory for temp files :param subset: (list) collector_module's function names if only subset is required (typically tests) :param since: (datetime) - low threshold of data changes (max. and default - 28 days ago) :param until: (datetime) - high threshold of data changes (defaults to now) @@ -57,7 +60,7 @@ def gather(self, dest=None, subset=None, since=None, until=None, billing_provide logger.log(self.log_level, 'Not gathering Automation Controller billing data, another task holds lock') return None - self._gather_initialize(dest, subset, since, until) + self._gather_initialize(subset, since, until) if not self._gather_config(): return None @@ -74,18 +77,6 @@ def gather(self, dest=None, subset=None, since=None, until=None, billing_provide return self.all_tar_paths() - def _gather_config(self): - if not super()._gather_config(): - return False - - # Extend the config collection to contain billing specific info: - config_collection = self.collections['config'] - data = json.loads(config_collection.data) - data['billing_provider_params'] = self.billing_provider_params - config_collection._save_gathering(data) - - return True - @staticmethod def db_connection(): return connection diff --git a/metrics_utility/automation_controller_billing/collectors.py b/metrics_utility/automation_controller_billing/collectors.py index 7b977528..f190e72f 100644 --- a/metrics_utility/automation_controller_billing/collectors.py +++ b/metrics_utility/automation_controller_billing/collectors.py @@ -1,31 +1,25 @@ -import json import os -import os.path -import platform - -from datetime import datetime, timezone -from importlib.metadata import version -from typing import Tuple - -import distro from django.db import connection from django.db.utils import ProgrammingError from django.utils.timezone import now, timedelta from django.utils.translation import gettext_lazy as _ -from metrics_utility.automation_controller_billing.helpers import ( - get_config_and_settings_from_db, - get_controller_version_from_db, - get_last_entries_from_db, -) +from metrics_utility.automation_controller_billing.helpers import get_last_entries_from_db from metrics_utility.base import register from metrics_utility.base.utils import get_max_gather_period_days, get_optional_collectors from metrics_utility.exceptions import MetricsException, MissingRequiredEnvVar -from metrics_utility.library import CsvFileSplitter -from metrics_utility.logger import logger, logger_info_level - -from .prometheus_client import PrometheusClient +from metrics_utility.library.collectors.controller import config as config_collector +from metrics_utility.library.collectors.controller import execution_environments as execution_environments_collector +from metrics_utility.library.collectors.controller import job_host_summary as job_host_summary_collector +from metrics_utility.library.collectors.controller import job_host_summary_service as job_host_summary_service_collector +from metrics_utility.library.collectors.controller import main_host as main_host_collector +from metrics_utility.library.collectors.controller import main_indirectmanagednodeaudit as main_indirectmanagednodeaudit_collector +from metrics_utility.library.collectors.controller import main_jobevent as main_jobevent_collector +from metrics_utility.library.collectors.controller import main_jobevent_service as main_jobevent_service_collector +from metrics_utility.library.collectors.controller import unified_jobs as unified_jobs_collector +from metrics_utility.library.collectors.others import total_workers_vcpu as total_workers_vcpu_collector +from metrics_utility.logger import logger try: @@ -54,7 +48,7 @@ def something(since): """ -def daily_slicing(key, last_gather, **kwargs): +def daily_slicing(key, last_gather=None, **kwargs): since, until = kwargs.get('since', None), kwargs.get('until', now()) if since is not None: last_entry = since @@ -81,7 +75,7 @@ def daily_slicing(key, last_gather, **kwargs): start = end -def limit_slicing(key, last_gather, **kwargs): +def limit_slicing(key, **kwargs): # For tables where we always need to do a table full scan, we want to load batches # TODO: skip today's collection if it already happened, so we don't load full inventory @@ -98,843 +92,131 @@ def limit_slicing(key, last_gather, **kwargs): yield (today, today) -def get_install_type(): - if os.getenv('container') == 'oci': - return 'openshift' - - if os.getenv('KUBERNETES_SERVICE_PORT'): - return 'k8s' - - return 'traditional' - - -@register('config', '1.0', description=_('General platform configuration.'), config=True) -def config(since, **kwargs): - license_info, settings_info = get_config_and_settings_from_db() - return { - 'platform': { - 'system': platform.system(), - 'dist': distro.linux_distribution(), - 'release': platform.release(), - 'type': get_install_type(), - }, - 'install_uuid': settings_info.get('install_uuid'), - 'instance_uuid': settings_info.get('system_uuid', '00000000-0000-0000-0000-000000000000'), - 'controller_url_base': settings_info.get('tower_url_base'), - 'controller_version': get_controller_version_from_db(), - 'license_type': license_info.get('license_type', 'UNLICENSED'), - 'license_date': license_info.get('license_date'), - 'subscription_name': license_info.get('subscription_name', ''), - 'sku': license_info.get('sku'), - 'support_level': license_info.get('support_level'), - 'usage': license_info.get('usage'), - 'product_name': license_info.get('product_name'), - 'valid_key': license_info.get('valid_key'), - 'satellite': license_info.get('satellite'), - 'pool_id': license_info.get('pool_id'), - 'subscription_id': license_info.get('subscription_id'), - 'account_number': license_info.get('account_number'), - 'current_instances': license_info.get('current_instances'), - 'automated_instances': license_info.get('automated_instances'), - 'automated_since': license_info.get('automated_since'), - 'trial': license_info.get('trial'), - 'grace_period_remaining': license_info.get('grace_period_remaining'), - 'compliant': license_info.get('compliant'), - 'date_warning': license_info.get('date_warning'), - 'date_expired': license_info.get('date_expired'), - 'subscription_usage_model': settings_info.get('subscription_usage_model', ''), # 1.5+ - 'free_instances': license_info.get('free_instances', 0), - 'total_licensed_instances': license_info.get('instance_count', 0), - 'license_expiry': license_info.get('time_remaining', 0), - 'pendo_tracking': settings_info.get('pendo_tracking_state', ''), - 'authentication_backends': settings_info.get('authentication_backends', ''), - 'logging_aggregators': settings_info.get('log_aggregator_loggers', ''), - 'external_logger_enabled': settings_info.get('log_aggregator_enabled', False), - 'external_logger_type': settings_info.get('log_aggregator_type', None), - 'metrics_utility_version': version('metrics-utility'), # version from setup.cfg - 'billing_provider_params': {}, # Is being overwritten in collector.gather by set ENV VARS - } - - -def _copy_table(table, query, path, prepend_query=None): - file_path = os.path.join(path, table + '_table.csv') - file = CsvFileSplitter(filespec=file_path) - - with connection.cursor() as cursor: - if prepend_query: - cursor.execute(prepend_query) - - if hasattr(cursor, 'copy_expert') and callable(cursor.copy_expert): - _copy_table_aap_2_4_and_below(cursor, query, file) - else: - _copy_table_aap_2_5_and_above(cursor, query, file) - - return file.file_list(keep_empty=True) - - -def _copy_table_aap_2_4_and_below(cursor, query, file): - # Automation Controller 4.4 and below use psycopg2 with .copy_expert() method - cursor.copy_expert(query, file) - - -def _copy_table_aap_2_5_and_above(cursor, query, file): - # Automation Controller 4.5 and above use psycopg3 with .copy() method - with cursor.copy(query) as copy: - while data := copy.read(): - byte_data = bytes(data) - file.write(byte_data.decode()) - - -def yaml_and_json_parsing_functions(): - query = """ - -- Define function for parsing field out of yaml encoded as text - CREATE OR REPLACE FUNCTION metrics_utility_parse_yaml_field( - str text, - field text - ) - RETURNS text AS - $$ - DECLARE - line_re text; - field_re text; - BEGIN - field_re := ' *[:=] *(.+?) *$'; - line_re := '(?n)^' || field || field_re; - RETURN trim(both '"' from substring(str from line_re) ); - END; - $$ - LANGUAGE plpgsql; - - -- Define function to check if field is a valid json - CREATE OR REPLACE FUNCTION metrics_utility_is_valid_json(p_json text) - returns boolean - AS - $$ - BEGIN - RETURN (p_json::json is not null); - EXCEPTION - WHEN others THEN - RETURN false; - END; - $$ - LANGUAGE plpgsql; - """ - return query +def until_slicing(key, **kwargs): + # exactly like limit_slicing, but uses the `until` timestamp, or now + # TODO: just replace limit_slicing? + until = kwargs.get('until', now()) + # Store the snapshot at the last day being collected (until - 1 day) + # since 'until' is the exclusive upper bound + last_day = until - timedelta(days=1) + yield (last_day, last_day) -@register('job_host_summary', '1.2', format='csv', description=_('Data for billing'), fnc_slicing=daily_slicing) -def job_host_summary_table(since, full_path, until, **kwargs): - disable_job_host_summary_str = os.getenv('METRICS_UTILITY_DISABLE_JOB_HOST_SUMMARY_COLLECTOR', 'false') - disable_job_host_summary = False - if disable_job_host_summary_str and (disable_job_host_summary_str.lower() == 'true'): - disable_job_host_summary = True +@register('config', '1.1', description=_('General platform configuration.'), config=True) +def config(billing_provider_params, **kwargs): + return config_collector(db=connection, billing_provider_params=billing_provider_params).gather() + - if disable_job_host_summary: +@register('job_host_summary', '1.2', format='csv', description=_('Data for billing'), fnc_slicing=daily_slicing) +def job_host_summary_table(since, until, output_dir=None, **kwargs): + disable = os.getenv('METRICS_UTILITY_DISABLE_JOB_HOST_SUMMARY_COLLECTOR', 'false').lower() + if disable == 'true': return None - # TODO: controler needs to have an index on main_jobhostsummary.modified - prepend_query = """ - -- Define function for parsing field out of yaml encoded as text - CREATE OR REPLACE FUNCTION metrics_utility_parse_yaml_field( - str text, - field text - ) - RETURNS text AS - $$ - DECLARE - line_re text; - field_re text; - BEGIN - field_re := ' *[:=] *(.+?) *$'; - line_re := '(?n)^' || field || field_re; - RETURN trim(both '"' from substring(str from line_re) ); - END; - $$ - LANGUAGE plpgsql; - - -- Define function to check if field is a valid json - CREATE OR REPLACE FUNCTION metrics_utility_is_valid_json(p_json text) - returns boolean - AS - $$ - BEGIN - RETURN (p_json::json is not null); - EXCEPTION - WHEN others THEN - RETURN false; - END; - $$ - LANGUAGE plpgsql; - """ - query = f""" - WITH - filtered_hosts AS ( - SELECT DISTINCT main_jobhostsummary.host_id - FROM main_jobhostsummary - WHERE (main_jobhostsummary.modified >= '{since.isoformat()}' - AND main_jobhostsummary.modified < '{until.isoformat()}') - ), - hosts_variables as ( - SELECT - filtered_hosts.host_id, - CASE - WHEN (metrics_utility_is_valid_json(main_host.variables)) - THEN main_host.variables::jsonb->>'ansible_host' - ELSE metrics_utility_parse_yaml_field(main_host.variables, 'ansible_host' ) - END AS ansible_host_variable, - CASE - WHEN (metrics_utility_is_valid_json(main_host.variables)) - THEN main_host.variables::jsonb->>'ansible_connection' - ELSE metrics_utility_parse_yaml_field(main_host.variables, 'ansible_connection' ) - END AS ansible_connection_variable - - FROM filtered_hosts - LEFT JOIN main_host ON main_host.id = filtered_hosts.host_id) - - SELECT main_jobhostsummary.id, - main_jobhostsummary.created, - main_jobhostsummary.modified, - main_jobhostsummary.host_name, - main_jobhostsummary.host_id as host_remote_id, - hosts_variables.ansible_host_variable, - hosts_variables.ansible_connection_variable, - main_jobhostsummary.changed, - main_jobhostsummary.dark, - main_jobhostsummary.failures, - main_jobhostsummary.ok, - main_jobhostsummary.processed, - main_jobhostsummary.skipped, - main_jobhostsummary.failed, - main_jobhostsummary.ignored, - main_jobhostsummary.rescued, - main_unifiedjob.created AS job_created, - main_jobhostsummary.job_id AS job_remote_id, - main_unifiedjob.unified_job_template_id AS job_template_remote_id, - main_unifiedjob.name AS job_template_name, - main_inventory.id AS inventory_remote_id, - main_inventory.name AS inventory_name, - main_organization.id AS organization_remote_id, - main_organization.name AS organization_name, - main_unifiedjobtemplate_project.id AS project_remote_id, - main_unifiedjobtemplate_project.name AS project_name - FROM main_jobhostsummary - -- connect to main_job, that has connections into inventory and project - LEFT JOIN main_job ON main_jobhostsummary.job_id = main_job.unifiedjob_ptr_id - -- get project name from project_options - LEFT JOIN main_unifiedjobtemplate AS main_unifiedjobtemplate_project ON main_unifiedjobtemplate_project.id = main_job.project_id - -- get inventory name from main_inventory - LEFT JOIN main_inventory ON main_inventory.id = main_job.inventory_id - -- get job name from main_unifiedjob - LEFT JOIN main_unifiedjob ON main_unifiedjob.id = main_jobhostsummary.job_id - -- get organization name from main_organization - LEFT JOIN main_organization ON main_organization.id = main_unifiedjob.organization_id - -- get variables from precomputed hosts_variables - LEFT JOIN hosts_variables ON hosts_variables.host_id = main_jobhostsummary.host_id - WHERE (main_jobhostsummary.modified >= '{since.isoformat()}' - AND main_jobhostsummary.modified < '{until.isoformat()}') - ORDER BY main_jobhostsummary.modified ASC - """ - - return _copy_table(table='main_jobhostsummary', query=f'COPY ({query}) TO STDOUT WITH CSV HEADER', path=full_path, prepend_query=prepend_query) + return job_host_summary_collector(db=connection, since=since, until=until, output_dir=output_dir).gather() @register('main_jobevent', '1.0', format='csv', description=_('Content usage'), fnc_slicing=daily_slicing) -def main_jobevent_table(since, full_path, until, **kwargs): +def main_jobevent_table(since, until, output_dir=None, **kwargs): if 'main_jobevent' not in get_optional_collectors(): return None - tbl = 'main_jobevent' - event_data = rf"replace({tbl}.event_data, '\u', '\u005cu')::jsonb" - - query = f""" - WITH job_scope AS ( - SELECT main_jobhostsummary.id AS main_jobhostsummary_id, - main_jobhostsummary.created AS main_jobhostsummary_created, - main_jobhostsummary.modified AS main_jobhostsummary_modified, - main_unifiedjob.created AS job_created, - main_jobhostsummary.job_id AS job_id, - main_jobhostsummary.host_name - FROM main_jobhostsummary - JOIN main_unifiedjob ON main_unifiedjob.id = main_jobhostsummary.job_id - WHERE (main_jobhostsummary.modified >= '{since.isoformat()}' AND main_jobhostsummary.modified < '{until.isoformat()}') - ) - SELECT - job_scope.main_jobhostsummary_id, - job_scope.main_jobhostsummary_created, - {tbl}.id, - {tbl}.created, - {tbl}.modified, - {tbl}.job_created as job_created, - {tbl}.event, - ({event_data}->>'task_action')::TEXT AS task_action, - ({event_data}->>'resolved_action')::TEXT AS resolved_action, - ({event_data}->>'resolved_role')::TEXT AS resolved_role, - ({event_data}->>'duration')::TEXT AS duration, - {tbl}.failed, - {tbl}.changed, - {tbl}.playbook, - {tbl}.play, - {tbl}.task, - {tbl}.role, - {tbl}.job_id as job_remote_id, - {tbl}.host_id as host_remote_id, - {tbl}.host_name - - FROM {tbl} - JOIN job_scope ON job_scope.job_created = {tbl}.job_created AND job_scope.job_id={tbl}.job_id AND job_scope.host_name={tbl}.host_name - WHERE {tbl}.event IN ('runner_on_ok', - 'runner_on_failed', - 'runner_on_unreachable', - 'runner_on_skipped', - 'runner_retry', - 'runner_on_async_ok', - 'runner_item_on_ok', - 'runner_item_on_failed', - 'runner_item_on_skipped') - """ - return _copy_table(table=tbl, query=f'COPY ({query}) TO STDOUT WITH CSV HEADER', path=full_path) + return main_jobevent_collector(db=connection, since=since, until=until, output_dir=output_dir).gather() @register('main_indirectmanagednodeaudit', '1.0', format='csv', description=_('Data for billing'), fnc_slicing=daily_slicing) -def main_indirectmanagednodeaudit_table(since, full_path, until, **kwargs): +def main_indirectmanagednodeaudit_table(since, until, output_dir=None, **kwargs): if 'main_indirectmanagednodeaudit' not in get_optional_collectors(): return None try: - query = f""" - ( - SELECT - main_indirectmanagednodeaudit.id, - main_indirectmanagednodeaudit.created as created, - main_indirectmanagednodeaudit.name as host_name, - main_indirectmanagednodeaudit.host_id AS host_remote_id, - main_indirectmanagednodeaudit.canonical_facts, - main_indirectmanagednodeaudit.facts, - main_indirectmanagednodeaudit.events, - main_indirectmanagednodeaudit.count as task_runs, - main_unifiedjob.created AS job_created, - main_indirectmanagednodeaudit.job_id AS job_remote_id, - main_unifiedjob.unified_job_template_id AS job_template_remote_id, - main_unifiedjob.name AS job_template_name, - main_inventory.id AS inventory_remote_id, - main_inventory.name AS inventory_name, - main_organization.id AS organization_remote_id, - main_organization.name AS organization_name, - main_unifiedjobtemplate_project.id AS project_remote_id, - main_unifiedjobtemplate_project.name AS project_name - FROM main_indirectmanagednodeaudit - LEFT JOIN main_job - ON main_job.unifiedjob_ptr_id = main_indirectmanagednodeaudit.job_id - LEFT JOIN main_unifiedjob - ON main_unifiedjob.id = main_indirectmanagednodeaudit.job_id - LEFT JOIN main_inventory - ON main_inventory.id = main_indirectmanagednodeaudit.inventory_id - LEFT JOIN main_organization - ON main_organization.id = main_unifiedjob.organization_id - LEFT JOIN main_unifiedjobtemplate AS main_unifiedjobtemplate_project - ON main_unifiedjobtemplate_project.id = main_job.project_id - WHERE (main_indirectmanagednodeaudit.created >= '{since.isoformat()}' - AND main_indirectmanagednodeaudit.created < '{until.isoformat()}') - ORDER BY main_indirectmanagednodeaudit.created ASC - ) - """ - - return _copy_table( - table='main_indirectmanagednodeaudit', - query=f'COPY ({query}) TO STDOUT WITH CSV HEADER', - path=full_path, - ) + return main_indirectmanagednodeaudit_collector(db=connection, since=since, until=until, output_dir=output_dir).gather() except (ProgrammingError, UndefinedTable) as e: logger.warning( - 'main_indirectmanagednodeaudit table missing in the database schema: %s.' - ' Falling back to behavior without indirect managed node audit data.', + 'main_indirectmanagednodeaudit table missing in the database schema: %s. ' + 'Falling back to behavior without indirect managed node audit data.', e, ) return None @register('main_host', '1.0', format='csv', description=_('Inventory data'), fnc_slicing=limit_slicing) -def main_host_table(since, full_path, until, **kwargs): +def main_host_table(output_dir=None, **kwargs): if 'main_host' not in get_optional_collectors(): return None - query = """ - ( - SELECT main_host.name as host_name, - main_host.id AS host_id, - main_inventory.id AS inventory_remote_id, - main_inventory.name AS inventory_name, - main_organization.id AS organization_remote_id, - main_organization.name AS organization_name, - main_unifiedjob.created AS last_automation, - - CASE - WHEN (metrics_utility_is_valid_json(main_host.variables)) - THEN main_host.variables::jsonb->>'ansible_host' - ELSE metrics_utility_parse_yaml_field(main_host.variables, 'ansible_host' ) - END AS ansible_host_variable, - - jsonb_build_object( - 'ansible_product_serial', main_host.ansible_facts->>'ansible_product_serial'::TEXT, - 'ansible_machine_id', main_host.ansible_facts->>'ansible_machine_id'::TEXT, - 'ansible_host', - CASE - WHEN (metrics_utility_is_valid_json(main_host.variables)) - THEN main_host.variables::jsonb->>'ansible_host' - ELSE metrics_utility_parse_yaml_field(main_host.variables, 'ansible_host' ) - END, - 'host_name', main_host.name, - 'ansible_port', - CASE - WHEN ( - CASE - WHEN (metrics_utility_is_valid_json(main_host.variables)) - THEN main_host.variables::jsonb->>'ansible_port' - ELSE metrics_utility_parse_yaml_field(main_host.variables, 'ansible_port' ) - END - ) ~ '^[0-9]+$' THEN - ( - CASE - WHEN (metrics_utility_is_valid_json(main_host.variables)) - THEN main_host.variables::jsonb->>'ansible_port' - ELSE metrics_utility_parse_yaml_field(main_host.variables, 'ansible_port' ) - END - )::INTEGER - ELSE NULL - END - ) AS canonical_facts, - - jsonb_build_object( - 'ansible_connection_variable', - CASE - WHEN (metrics_utility_is_valid_json(main_host.variables)) - THEN main_host.variables::jsonb->>'ansible_connection' - ELSE metrics_utility_parse_yaml_field(main_host.variables, 'ansible_connection' ) - END, - 'ansible_virtualization_type', - main_host.ansible_facts->>'ansible_virtualization_type'::TEXT, - 'ansible_virtualization_role', - main_host.ansible_facts->>'ansible_virtualization_role'::TEXT, - 'ansible_system_vendor', - main_host.ansible_facts->>'ansible_system_vendor'::TEXT, - 'ansible_product_name', - main_host.ansible_facts->>'ansible_product_name'::TEXT, - 'ansible_architecture', - main_host.ansible_facts->>'ansible_architecture'::TEXT, - 'ansible_processor', - main_host.ansible_facts->>'ansible_processor'::TEXT, - 'ansible_form_factor', - main_host.ansible_facts->>'ansible_form_factor'::TEXT, - 'ansible_bios_vendor', - main_host.ansible_facts->>'ansible_bios_vendor'::TEXT, - 'ansible_bios_version', - main_host.ansible_facts->>'ansible_bios_version'::TEXT, - 'ansible_board_serial', - main_host.ansible_facts->>'ansible_board_serial'::TEXT - ) AS facts - - FROM main_host - LEFT JOIN main_inventory - ON main_inventory.id = main_host.inventory_id - LEFT JOIN main_organization - ON main_organization.id = main_inventory.organization_id - LEFT JOIN main_unifiedjob - ON main_unifiedjob.id = main_host.last_job_id - WHERE enabled='t' - ORDER BY main_host.id ASC - ) - """ - - return _copy_table( - table='main_host', query=f'COPY ({query}) TO STDOUT WITH CSV HEADER', path=full_path, prepend_query=yaml_and_json_parsing_functions() - ) + return main_host_collector(db=connection, output_dir=output_dir).gather() @register('total_workers_vcpu', '1.0', format='json', description=_('Total workers vCPU'), fnc_slicing=limit_slicing) -def total_workers_vcpu(since, full_path, until, **kwargs): +def total_workers_vcpu(**kwargs): if 'total_workers_vcpu' not in get_optional_collectors(): return None + # If METRICS_UTILITY_USAGE_BASED_METERING_ENABLED is not set or set to false then it returns 1 + usage_based_billing_enabled_str = os.getenv('METRICS_UTILITY_USAGE_BASED_METERING_ENABLED') + metering_enabled = bool(usage_based_billing_enabled_str and (usage_based_billing_enabled_str.lower() == 'true')) + cluster_name = os.getenv('METRICS_UTILITY_CLUSTER_NAME') if not cluster_name: logger.error('environment variable METRICS_UTILITY_CLUSTER_NAME is not set') raise MissingRequiredEnvVar('environment variable METRICS_UTILITY_CLUSTER_NAME is not set') - now = datetime.now(timezone.utc) - current_ts = now.timestamp() - prev_hour_start, prev_hour_end = get_hour_boundaries(current_ts) - - info = { - 'cluster_name': cluster_name, - 'collection_timestamp': datetime.fromtimestamp(current_ts).isoformat(), - 'start_timestamp': datetime.fromtimestamp(prev_hour_start).isoformat(), - 'end_timestamp': datetime.fromtimestamp(prev_hour_end).isoformat(), - } - # If METRICS_UTILITY_USAGE_BASED_METERING_ENABLED is not set or set to false then it returns 1 - usage_based_billing_enabled_str = os.getenv('METRICS_UTILITY_USAGE_BASED_METERING_ENABLED') - usage_based_billing_enabled = False - if usage_based_billing_enabled_str and (usage_based_billing_enabled_str.lower() == 'true'): - usage_based_billing_enabled = True - info['usage_based_billing_enabled'] = usage_based_billing_enabled - if not usage_based_billing_enabled: - info['total_workers_vcpu'] = 1 - # This message must always appear in the log regardless of the log level. - logger_info_level.info(json.dumps(info, indent=2)) - return {'timestamp': info['end_timestamp'], 'cluster_name': info['cluster_name'], 'total_workers_vcpu': info['total_workers_vcpu']} - - url = os.getenv('METRICS_UTILITY_PROMETHEUS_URL') - if not url: - prometheus_default_url = 'https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091' - logger.info( - f'environment variable METRICS_UTILITY_PROMETHEUS_URL is not set, \ - default {prometheus_default_url} will be assigned' - ) - url = prometheus_default_url - - try: - prom = PrometheusClient(url=url) - except Exception as e: - raise MetricsException(f'Can not create a prometheus api client ERROR: {e}') - - try: - total_workers_vcpu, promql_query = get_total_workers_cpu(prom, prev_hour_start) - timeline = get_cpu_timeline(prom, prev_hour_start, prev_hour_end) - except MetricsException as e: - raise MetricsException(f'Unexpected error when retrieving nodes: {e}') - - info['promql_query'] = promql_query - info['timeline'] = timeline - - logger.debug(f'total_workers_vcpu: {total_workers_vcpu}') - - # This can happen when the prev_hour_start doesn't have data, it could be when the cluster just started or - # if for some reasons prometheus loss some data. - if total_workers_vcpu is None: - logger.warning('No data availble yet, the cluster is probably running for less than an hour') - raise MetricsException('No data availble yet, the cluster is probably running for less than an hour') - - info['total_workers_vcpu'] = int(total_workers_vcpu) - - # This message must always appear in the log regardless of the log level. - logger_info_level.info(json.dumps(info, indent=2)) + prometheus_url = os.getenv('METRICS_UTILITY_PROMETHEUS_URL') + if not prometheus_url: + prometheus_url = 'https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091' + logger.info(f'environment variable METRICS_UTILITY_PROMETHEUS_URL is not set, default {prometheus_url} will be assigned') - return {'timestamp': info['end_timestamp'], 'cluster_name': info['cluster_name'], 'total_workers_vcpu': info['total_workers_vcpu']} + token_path = '/var/run/secrets/kubernetes.io/serviceaccount/token' + if not os.path.exists(token_path): + raise MetricsException(f'Service account token not found at {token_path}') + ca_cert_path = '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' + if not os.path.exists(ca_cert_path): + raise MetricsException(f'CA_CERT not found at {ca_cert_path}') -def get_hour_boundaries(current_timestamp: float) -> Tuple[float, float, float]: - current_hour_start = (current_timestamp // 3600) * 3600 - previous_hour_start = current_hour_start - 3600 - previous_hour_end = current_hour_start - 1 - return previous_hour_start, previous_hour_end - - -def get_total_workers_cpu(prom: PrometheusClient, base_timestamp: float) -> Tuple[float, str]: - promql_query = f'max_over_time(sum(machine_cpu_cores)[59m59s:5m] @ {base_timestamp})' - - try: - total_workers_vcpu = prom.get_current_value(promql_query) - except Exception as e: - raise MetricsException(f'Unexpected error when retrieving nodes: {e}') - - return total_workers_vcpu, promql_query + token = None + with open(token_path, 'r') as f: + token = f.read().strip() + if not token: + raise MetricsException(f'Unable to retrieve the token for the current service account from {token_path}') - -def get_cpu_timeline(prom: PrometheusClient, previous_hour_start, previous_hour_end: float) -> list: - """ - Get array of timestamp/CPU pairs for the hour leading up to previous_hour_end - Returns: - List of dicts with 'timestamp' (ISO format) and 'cpu_sum' keys - """ - # Use instant query - query_range will handle the time range - query = 'sum(machine_cpu_cores)' - - try: - response = prom.query_range(query=query, start_time=previous_hour_start, end_time=previous_hour_end, step='5m') - - result = [] - if response and 'data' in response and 'result' in response['data']: - for series in response['data']['result']: - if 'values' in series: - for timestamp_val, cpu_val in series['values']: - result.append( - {'timestamp': datetime.fromtimestamp(float(timestamp_val), timezone.utc).isoformat(), 'cpu_sum': float(cpu_val)} - ) - - # Sort by timestamp - result.sort(key=lambda x: x['timestamp']) - return result - - except Exception as e: - raise MetricsException(f'Error querying CPU timeline: {e}') + return total_workers_vcpu_collector( + ca_cert_path=ca_cert_path, + cluster_name=cluster_name, + metering_enabled=metering_enabled, + prometheus_url=prometheus_url, + token=token, + ).gather() @register('unified_jobs', '1.4', format='csv', description=_('Data on jobs run'), fnc_slicing=daily_slicing) -def unified_jobs_table(since, full_path, until, **kwargs): +def unified_jobs_table(since, until, output_dir=None, **kwargs): if 'unified_jobs' not in get_optional_collectors(): return None - unified_job_query = """COPY (SELECT main_unifiedjob.id, - main_unifiedjob.polymorphic_ctype_id, - django_content_type.model, - main_unifiedjob.organization_id, - main_organization.name as organization_name, - main_executionenvironment.image as execution_environment_image, - main_job.inventory_id, - main_inventory.name as inventory_name, - main_unifiedjob.created, - main_unifiedjob.name, - main_unifiedjob.unified_job_template_id, - main_unifiedjob.launch_type, - main_unifiedjob.schedule_id, - main_unifiedjob.execution_node, - main_unifiedjob.controller_node, - main_unifiedjob.cancel_flag, - main_unifiedjob.status, - main_unifiedjob.failed, - main_unifiedjob.started, - main_unifiedjob.finished, - main_unifiedjob.elapsed, - main_unifiedjob.job_explanation, - main_unifiedjob.instance_group_id, - main_unifiedjob.installed_collections, - main_unifiedjob.ansible_version, - main_job.forks, - main_unifiedjobtemplate.name as job_template_name - FROM main_unifiedjob - LEFT JOIN main_unifiedjobtemplate ON main_unifiedjobtemplate.id = main_unifiedjob.unified_job_template_id - LEFT JOIN django_content_type ON main_unifiedjob.polymorphic_ctype_id = django_content_type.id - LEFT JOIN main_job ON main_unifiedjob.id = main_job.unifiedjob_ptr_id - LEFT JOIN main_inventory ON main_job.inventory_id = main_inventory.id - LEFT JOIN main_organization ON main_organization.id = main_unifiedjob.organization_id - LEFT JOIN main_executionenvironment ON main_executionenvironment.id = main_unifiedjob.execution_environment_id - WHERE ((main_unifiedjob.created >= '{0}' AND main_unifiedjob.created < '{1}') - OR (main_unifiedjob.finished >= '{0}' AND main_unifiedjob.finished < '{1}')) - AND main_unifiedjob.launch_type != 'sync' - ORDER BY main_unifiedjob.id ASC) TO STDOUT WITH CSV HEADER - """.format(since.isoformat(), until.isoformat()) - - return _copy_table(table='unified_jobs', query=unified_job_query, path=full_path) + return unified_jobs_collector(db=connection, since=since, until=until, output_dir=output_dir).gather() @register('job_host_summary_service', '1.4', format='csv', description=_('Data for billing'), fnc_slicing=daily_slicing) -def job_host_summary_service_table(since, full_path, until, **kwargs): +def job_host_summary_service_table(since, until, output_dir=None, **kwargs): if 'job_host_summary_service' not in get_optional_collectors(): return None - prepend_query = """ - -- Define function for parsing field out of yaml encoded as text - CREATE OR REPLACE FUNCTION metrics_utility_parse_yaml_field( - str text, - field text - ) - RETURNS text AS - $$ - DECLARE - line_re text; - field_re text; - BEGIN - field_re := ' *[:=] *(.+?) *$'; - line_re := '(?n)^' || field || field_re; - RETURN trim(both '"' from substring(str from line_re) ); - END; - $$ - LANGUAGE plpgsql; - - -- Define function to check if field is a valid json - CREATE OR REPLACE FUNCTION metrics_utility_is_valid_json(p_json text) - returns boolean - AS - $$ - BEGIN - RETURN (p_json::json is not null); - EXCEPTION - WHEN others THEN - RETURN false; - END; - $$ - LANGUAGE plpgsql; - """ - - query = f""" - WITH - -- First: restrict to jobs that FINISHED in the window (uses index on main_unifiedjob.finished if present) - filtered_jobs AS ( - SELECT mu.id - FROM main_unifiedjob mu - WHERE mu.finished >= '{since.isoformat()}' - AND mu.finished < '{until.isoformat()}' - AND mu.finished IS NOT NULL - ), - -- - -- Then: only host summaries that belong to those jobs (uses index on main_jobhostsummary.job_id) - filtered_hosts AS ( - SELECT DISTINCT mjs.host_id - FROM main_jobhostsummary mjs - JOIN filtered_jobs fj ON fj.id = mjs.job_id - ), - -- - hosts_variables AS ( - SELECT - fh.host_id, - CASE - WHEN metrics_utility_is_valid_json(h.variables) - THEN h.variables::jsonb->>'ansible_host' - ELSE metrics_utility_parse_yaml_field(h.variables, 'ansible_host') - END AS ansible_host_variable, - CASE - WHEN metrics_utility_is_valid_json(h.variables) - THEN h.variables::jsonb->>'ansible_connection' - ELSE metrics_utility_parse_yaml_field(h.variables, 'ansible_connection') - END AS ansible_connection_variable - FROM filtered_hosts fh - LEFT JOIN main_host h ON h.id = fh.host_id - ) - - SELECT - mjs.id, - mjs.created, - mjs.modified, - mjs.host_name, - mjs.host_id AS host_remote_id, - hv.ansible_host_variable, - hv.ansible_connection_variable, - mjs.changed, - mjs.dark, - mjs.failures, - mjs.ok, - mjs.processed, - mjs.skipped, - mjs.failed, - mjs.ignored, - mjs.rescued, - mu.created AS job_created, - mjs.job_id AS job_remote_id, - mu.unified_job_template_id AS job_template_remote_id, - mu.name AS job_template_name, - mi.id AS inventory_remote_id, - mi.name AS inventory_name, - mo.id AS organization_remote_id, - mo.name AS organization_name, - mup.id AS project_remote_id, - mup.name AS project_name - FROM filtered_jobs fj - JOIN main_jobhostsummary mjs ON mjs.job_id = fj.id - LEFT JOIN main_job mj ON mjs.job_id = mj.unifiedjob_ptr_id - LEFT JOIN main_unifiedjob mu ON mu.id = mjs.job_id - LEFT JOIN main_unifiedjobtemplate AS mup ON mup.id = mj.project_id - LEFT JOIN main_inventory mi ON mi.id = mj.inventory_id - LEFT JOIN main_organization mo ON mo.id = mu.organization_id - LEFT JOIN hosts_variables hv ON hv.host_id = mjs.host_id - ORDER BY mu.finished ASC - """ - - return _copy_table(table='main_jobhostsummary', query=f'COPY ({query}) TO STDOUT WITH CSV HEADER', path=full_path, prepend_query=prepend_query) + return job_host_summary_service_collector(db=connection, since=since, until=until, output_dir=output_dir).gather() @register('main_jobevent_service', '1.4', format='csv', description=_('Content usage'), fnc_slicing=daily_slicing) -def main_jobevent_service_table(since, full_path, until, **kwargs): +def main_jobevent_service_table(since, until, output_dir=None, **kwargs): if 'main_jobevent_service' not in get_optional_collectors(): return None - # Use the table alias 'e' here (you alias main_jobevent as e in the FROM) - event_data = r"replace(e.event_data, '\u', '\u005cu')::jsonb" - - # 1) Load finished jobs in the window - jobs_query = """ - SELECT uj.id AS job_id, - uj.created AS job_created - FROM main_unifiedjob uj - WHERE uj.finished >= %(since)s - AND uj.finished < %(until)s - """ - jobs = [] - - # do raw sql for django.db connection - with connection.cursor() as cursor: - cursor.execute(jobs_query, {'since': since, 'until': until}) - jobs = cursor.fetchall() - - # 2) Build a literal WHERE clause that preserves (job_id, job_created) pairing - if jobs: - # (e.job_id, e.job_created) IN (VALUES (id1, 'ts1'::timestamptz), ...) - pairs_sql = ',\n'.join(f"({jid}, '{jcreated.isoformat()}'::timestamptz)" for jid, jcreated in jobs) - where_clause = f'(e.job_id, e.job_created) IN (VALUES {pairs_sql})' - else: - # No jobs in the window → no events - where_clause = 'FALSE' - - # 3) Final event query - query = f""" - SELECT - e.id, - e.created, - e.modified, - e.job_created, - uj.finished as job_finished, - e.uuid, - e.parent_uuid, - e.event, - - -- JSON extracted fields - ({event_data}->>'task_action') AS task_action, - ({event_data}->>'resolved_action') AS resolved_action, - ({event_data}->>'resolved_role') AS resolved_role, - ({event_data}->>'duration') AS duration, - ({event_data}->>'start')::timestamptz AS start, - ({event_data}->>'end')::timestamptz AS end, - ({event_data}->>'task_uuid') AS task_uuid, - COALESCE( ({event_data}->>'ignore_errors')::boolean, false ) AS ignore_errors, - e.failed, - e.changed, - e.playbook, - e.play, - e.task, - e.role, - e.job_id AS job_remote_id, - e.job_id, - e.host_id AS host_remote_id, - e.host_id, - e.host_name, - - -- Warnings and deprecations (json arrays) - {event_data}->'res'->'warnings' AS warnings, - {event_data}->'res'->'deprecations' AS deprecations, - - CASE WHEN e.event = 'playbook_on_stats' - THEN {event_data} - 'artifact_data' - END AS playbook_on_stats, - - uj.failed as job_failed, - uj.started as job_started - - FROM main_jobevent e - LEFT JOIN main_unifiedjob uj ON uj.id = e.job_id - WHERE {where_clause} - """ - - return _copy_table(table='main_jobevent', query=f'COPY ({query}) TO STDOUT WITH CSV HEADER', path=full_path) - - -@register('execution_environments', '1.4', format='csv', description=_('Execution environments'), fnc_slicing=daily_slicing) -def execution_environments_table(since, full_path, until, **kwargs): + return main_jobevent_service_collector(db=connection, since=since, until=until, output_dir=output_dir).gather() + + +@register('execution_environments', '1.4', format='csv', description=_('Execution environments'), fnc_slicing=until_slicing) +def execution_environments_table(output_dir=None, **kwargs): if 'execution_environments' not in get_optional_collectors(): return None - sql = """ - SELECT - id, - created, - modified, - description, - image, - managed, - created_by_id, - credential_id, - modified_by_id, - organization_id, - name, - pull - FROM public.main_executionenvironment - """ - - return _copy_table(table='main_executionenvironment', query=f'COPY ({sql}) TO STDOUT WITH CSV HEADER', path=full_path) + return execution_environments_collector(db=connection, output_dir=output_dir).gather() diff --git a/metrics_utility/automation_controller_billing/helpers.py b/metrics_utility/automation_controller_billing/helpers.py index 66f7fbbb..3d2e95b4 100644 --- a/metrics_utility/automation_controller_billing/helpers.py +++ b/metrics_utility/automation_controller_billing/helpers.py @@ -1,7 +1,7 @@ import json from itertools import chain -from typing import Any, Dict, Tuple +from typing import Dict import pandas as pd @@ -36,60 +36,6 @@ def get_last_entries_from_db() -> Dict: return {} -def get_config_and_settings_from_db() -> Tuple[Dict[str, Any], Dict[str, Any]]: - """Get license information directly from the database.""" - license_info = {} - settings_info = {} - try: - with connection.cursor() as cursor: - cursor.execute(""" - SELECT key, value - FROM conf_setting - WHERE key IN ('LICENSE', 'INSTALL_UUID', 'TOWER_URL_BASE', - 'SUBSCRIPTION_USAGE_MODEL','PENDO_TRACKING_STATE','AUTHENTICATION_BACKENDS', - 'LOG_AGGREGATOR_LOGGERS', 'SYSTEM_UUID', 'LOG_AGGREGATOR_ENABLED', - 'LOG_AGGREGATOR_TYPE') - """) - rows = cursor.fetchall() - for row in rows: - key, value = row - if key == 'LICENSE': - license_info = json.loads(value) # The LICENSE key has a value which is an object. - # We want all the items in the object put on their own - # dict. - else: - settings_info[key.lower()] = json.loads(value) - - except Exception as e: - logger.error(f'Error getting license information from database: {e}') - return license_info, settings_info - - -def _fetch_one(db, sql): - with db.cursor() as cursor: - cursor.execute(sql) - result = cursor.fetchone() - if result and result[0]: - return result[0] - return None - - -def get_controller_version_from_db() -> str: - """Get AWX/Controller version from the main_instance DB table.""" - return _fetch_one( - connection, - """ - SELECT version - FROM main_instance - WHERE enabled = true - AND version IS NOT NULL - AND version != '' - ORDER BY last_seen DESC - LIMIT 1 - """, - ) - - def datetime_hook(d): new_d = {} for key, value in d.items(): diff --git a/metrics_utility/automation_controller_billing/kubernetes_client.py b/metrics_utility/automation_controller_billing/kubernetes_client.py deleted file mode 100644 index e2034877..00000000 --- a/metrics_utility/automation_controller_billing/kubernetes_client.py +++ /dev/null @@ -1,58 +0,0 @@ -import os - -from metrics_utility.exceptions import MetricsException -from metrics_utility.logger import logger - - -TOKEN_PATH = '/var/run/secrets/kubernetes.io/serviceaccount/token' -CA_CERT_PATH = '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' - - -class KubernetesClient: - """ - Simplified Kubernetes client for service account token operations. - - This class assumes running in a Kubernetes pod with standard service account - files mounted at /var/run/secrets/kubernetes.io/serviceaccount/ - """ - - def __init__(self): - """Initialize the client and validate service account files are available.""" - self._validate_service_account_files() - - def _validate_service_account_files(self): - """Validate that required service account files exist.""" - - if not os.path.exists(TOKEN_PATH): - raise MetricsException('Service account token not found at /var/run/secrets/kubernetes.io/serviceaccount/token') - - logger.info('Service account files validated') - - def get_current_token(self) -> str: - """ - Get the current pod's service account token from the mounted file. - - Returns: - Current service account token - - Raises: - MetricsException: If token cannot be read - """ - - try: - with open(TOKEN_PATH, 'r') as f: - token = f.read().strip() - logger.info("Retrieved current pod's mounted token") - logger.info(f' Token Length: {len(token)} characters') - return token - except Exception as e: - raise MetricsException(f'Error reading token: {e}') - - def get_ca_cert_path(self) -> str: - """ - Get the current pod's service account ca_cert from the mounted file. - - Returns: - Current service account ca_cert - """ - return CA_CERT_PATH diff --git a/metrics_utility/automation_controller_billing/prometheus_client.py b/metrics_utility/automation_controller_billing/prometheus_client.py deleted file mode 100644 index 035ff5f0..00000000 --- a/metrics_utility/automation_controller_billing/prometheus_client.py +++ /dev/null @@ -1,142 +0,0 @@ -import os - -from typing import Optional - -import requests - -from metrics_utility.exceptions import MetricsException -from metrics_utility.logger import logger - -from .kubernetes_client import KubernetesClient - - -class PrometheusClient: - """ - Prometheus client with Kubernetes service account authentication support. - - This class handles: - - Service account token retrieval from Kubernetes - - Prometheus connection management - - Query execution with proper error handling - """ - - def __init__(self, url: str, timeout: int = 30): - """ - Initialize Prometheus client. - - Args: - url: Prometheus server URL - timeout: Request timeout in seconds (default: 30) - """ - self.url = url.rstrip('/') # Remove trailing slash - self.timeout = timeout - self.token = None - self.ca_cert_path = None - self.session = requests.Session() - - kube_client = KubernetesClient() - self.token = kube_client.get_current_token() - if not self.token: - raise MetricsException('Unable to retrieve the token for the current service account') - - self.ca_cert_path = kube_client.get_ca_cert_path() - - # Setup session - self._setup_session() - - def _setup_session(self): - """Setup HTTP session with authentication headers and CA certificate""" - if self.token: - logger.info('Creating authenticated Prometheus client') - logger.info(f' URL: {self.url}') - - self.session.headers.update({'Authorization': f'Bearer {self.token}', 'Content-Type': 'application/x-www-form-urlencoded'}) - else: - logger.info('Creating unauthenticated Prometheus client') - logger.info(f' URL: {self.url}') - - # Use service CA certificate for SSL verification - if os.path.exists(self.ca_cert_path): - self.session.verify = self.ca_cert_path - logger.info(f'Using service CA certificate: {self.ca_cert_path}') - else: - raise MetricsException(f'CA_CERT not found at {self.ca_cert_path}') - - def query(self, query: str, time_param: Optional[float] = None) -> Optional[list]: - """ - Execute instant PromQL query. - - Args: - query: PromQL query string - time_param: Optional timestamp for the query - - Returns: - Query results as list, or raise MetricsException if failed - """ - try: - url = f'{self.url}/api/v1/query' - params = {'query': query} - - if time_param: - params['time'] = time_param - - response = self.session.get(url, params=params, timeout=self.timeout) - - logger.debug(f'response: {response}') - if response.status_code == 200: - data = response.json() - logger.debug(f'data: {data}') - if data.get('status') == 'success': - return data.get('data', {}).get('result', []) - else: - raise MetricsException(f'Prometheus API error: {data.get("error", "Unknown error")}') - else: - raise MetricsException(f'HTTP error {response.status_code}: {response.text}') - - except Exception as e: - raise MetricsException(f'Query failed: {e}') - - def get_current_value(self, query: str) -> Optional[float]: - """ - Get current value from an instant query. - - Args: - query: PromQL query string - - Returns: - Current value as float, or None if result is empty - """ - result = self.query(query) - if result and len(result) > 0: - return float(result[0]['value'][1]) - return None - - def query_range(self, query: str, start_time: float, end_time: float, step: str = '5m') -> Optional[dict]: - """ - Execute a range query against Prometheus. - Args: - query: PromQL instant query (not range query) - start_time: Start time (Unix timestamp) - end_time: End time (Unix timestamp) - step: Query resolution step (e.g., '1m', '5m') - """ - params = {'query': query, 'start': start_time, 'end': end_time, 'step': step} - - try: - url = f'{self.url}/api/v1/query_range' - logger.debug(f'Range query URL: {url}') - logger.debug(f'Range query params: {params}') - - response = self.session.get(url, params=params, timeout=self.timeout) - response.raise_for_status() - - data = response.json() - if data.get('status') == 'success': - return data - else: - logger.error(f'Prometheus range query failed: {data.get("error", "Unknown error")}') - return None - - except Exception as e: - logger.error(f'Range query failed: {e}') - raise MetricsException(e) diff --git a/metrics_utility/base/collection.py b/metrics_utility/base/collection.py index de6d182e..66a9f527 100644 --- a/metrics_utility/base/collection.py +++ b/metrics_utility/base/collection.py @@ -29,10 +29,8 @@ def __init__(self, collector, fnc_collecting): self.data_type = fnc_collecting.__insights_analytics_type__ self.filename = f'{self.key}.{self.data_type}' - # either since/until or full sync(if enabled) self.since = None # set by Collector._create_collections() self.until = None # set by Collector._create_collections() - self.full_sync_enabled = self._is_full_sync_enabled(fnc_collecting.__insights_analytics_full_sync_interval_days__) self.gathering_started_at = None self.gathering_finished_at = None @@ -51,7 +49,7 @@ def cleanup(self): def data_size(self): pass - def gather(self, max_data_size): + def gather(self): self.gathering_started_at = now() try: @@ -61,8 +59,8 @@ def gather(self, max_data_size): result = self.fnc_collecting( since=self.since, until=self.until, - max_data_size=max_data_size, - full_path=self.collector.gather_dir, + output_dir=self.collector.gather_dir, + billing_provider_params=self.collector.billing_provider_params, # FIXME: used only by config collector ) self._save_gathering(result) @@ -84,13 +82,8 @@ def slices(self): # These slicer functions may return a generator. The `since` parameter is # allowed to be None, and will fall back to LAST_ENTRIES[key] or to # LAST_GATHER (truncated appropriately to match the 28-day limit). - # - # Or it can force full table sync if interval is given if self.fnc_slicing: - if self.full_sync_enabled: - slices = self.fnc_slicing(self.key, last_gather, full_sync_enabled=True) - else: - slices = self.fnc_slicing(self.key, last_gather, since=since, until=until) + slices = self.fnc_slicing(self.key, last_gather=last_gather, since=since, until=until) else: slices = [(self._gather_since(), self._gather_until())] @@ -115,9 +108,6 @@ def update_last_gathered_entries(self, updates_dict): if self.gathering_successful: self._update_last_gathered_key(updates_dict, self.key, self.until) - - if self.full_sync_enabled: - self._update_last_gathered_key(updates_dict, f'{self.key}_full', self.gathering_finished_at) else: # collections are ordered by time slices. # in case of error all collections with newer timestamp are ignored @@ -134,13 +124,6 @@ def _update_last_gathered_key(updates_dict, key, timestamp): else: updates_dict['keys'][key] = max(previous, timestamp) - def _is_full_sync_enabled(self, interval_days): - if not interval_days: - return False - - last_full_sync = self.collector.last_gathered_entry_for(f'{self.key}_full') - return not last_full_sync or last_full_sync < now() - timedelta(days=interval_days) - def _gather_since(self): """Start of gathering based on settings excluding slices""" diff --git a/metrics_utility/base/collection_data_status.py b/metrics_utility/base/collection_data_status.py index e2e04677..910638f2 100644 --- a/metrics_utility/base/collection_data_status.py +++ b/metrics_utility/base/collection_data_status.py @@ -17,8 +17,8 @@ def __init__(self, collector, package): format='csv', description='Data collection status', ) - def data_collection_status(self, full_path, **kwargs): - file_path = os.path.join(full_path, self.filename) + def data_collection_status(self, output_dir=None, **kwargs): + file_path = os.path.join(output_dir, self.filename) with open(file_path, 'w', newline='') as csvfile: fieldnames = [ 'collection_start_timestamp', diff --git a/metrics_utility/base/collector.py b/metrics_utility/base/collector.py index c30c9858..3599b87a 100644 --- a/metrics_utility/base/collector.py +++ b/metrics_utility/base/collector.py @@ -3,14 +3,13 @@ import inspect import logging import os -import pathlib import shutil -import tempfile from abc import abstractmethod from django.utils.timezone import now, timedelta +from metrics_utility.library.collectors.util import init_tmp_dir from metrics_utility.logger import logger from .collection import Collection @@ -43,8 +42,10 @@ class Collector: DRY_RUN = 'dry-run' SCHEDULED_COLLECTION = 'scheduled' - def __init__(self, collection_type=DRY_RUN, collector_module=None): + def __init__(self, collection_type=DRY_RUN, collector_module=None, billing_provider_params=None): self.collector_module = collector_module + self.billing_provider_params = billing_provider_params + self.collections = {} self.packages = {} @@ -54,6 +55,7 @@ def __init__(self, collection_type=DRY_RUN, collector_module=None): self.tmp_dir = None self.gather_dir = None + self.gather_since = None self.gather_until = None self.last_gather = None @@ -98,10 +100,9 @@ def db_connection(): """ pass - def gather(self, dest=None, subset=None, since=None, until=None): + def gather(self, subset=None, since=None, until=None): """Entry point for gathering - :param dest: (default: /tmp/awx-analytics-*) - directory for temp files :param subset: (list) collector_module's function names if only subset is required (typically tests) :param since: (datetime) - low threshold of data changes (max. and default - 28 days ago) :param until: (datetime) - high threshold of data changes (defaults to now) @@ -112,7 +113,7 @@ def gather(self, dest=None, subset=None, since=None, until=None): logger.log(self.log_level, 'Not gathering analytics, another task holds lock') return None - self._gather_initialize(dest, subset, since, until) + self._gather_initialize(subset, since, until) if not self._gather_config(): return None @@ -225,8 +226,9 @@ def _find_available_package(self, group, key, requested_size=None): return available_package - def _gather_initialize(self, tmp_root_dir, collectors_subset, since, until): - self._init_tmp_dir(tmp_root_dir) + def _gather_initialize(self, collectors_subset, since, until): + self.gather_dir = init_tmp_dir() + self.tmp_dir = self.gather_dir.parent self.last_gathered_entries = self._load_last_gathered_entries() @@ -244,13 +246,13 @@ def _gather_config(self): logger.log(self.log_level, "'config' collector data is missing") return False else: - self.collections['config'].gather(self._package_class().max_data_size()) + self.collections['config'].gather() return True def _gather_json_collections(self): """JSON collections are simpler, they're just gathered and added to the Package""" for collection in self.collections[Collection.COLLECTION_TYPE_JSON]: - collection.gather(self._package_class().max_data_size()) + collection.gather() if collection.is_empty() or not collection.gathering_successful: continue @@ -288,7 +290,7 @@ def _gather_csv_collections(self): last_key = collection.key - collection.gather(self._package_class().max_data_size()) + collection.gather() if collection.is_empty() or not collection.gathering_successful: continue @@ -366,11 +368,6 @@ def _gather_cleanup(self): if self.ship: self.delete_tarballs() - def _init_tmp_dir(self, tmp_root_dir=None): - self.tmp_dir = pathlib.Path(tmp_root_dir or tempfile.mkdtemp(prefix='awx_analytics-')) - self.gather_dir = self.tmp_dir.joinpath('stage') - self.gather_dir.mkdir(mode=0o700) - @abstractmethod def _load_last_gathered_entries(self): """Loads persisted timestamps named by keys from collector_module diff --git a/metrics_utility/base/decorators.py b/metrics_utility/base/decorators.py index 925e5f0d..b79906b2 100644 --- a/metrics_utility/base/decorators.py +++ b/metrics_utility/base/decorators.py @@ -6,7 +6,6 @@ def register( config=False, fnc_slicing=None, shipping_group='default', - full_sync_interval_days=None, ): """ A decorator used to register a function as a metric collector. @@ -30,7 +29,6 @@ def decorate(f): f.__insights_analytics_config__ = config # True | False (default) f.__insights_analytics_fnc_slicing__ = fnc_slicing f.__insights_analytics_shipping_group__ = shipping_group - f.__insights_analytics_full_sync_interval_days__ = full_sync_interval_days return f diff --git a/metrics_utility/base/package.py b/metrics_utility/base/package.py index 85bb6aaf..f976de87 100644 --- a/metrics_utility/base/package.py +++ b/metrics_utility/base/package.py @@ -305,7 +305,7 @@ def _get_x_rh_identity(self): def _data_collection_status_to_tar(self, tar): try: - self.data_collection_status.gather(None) + self.data_collection_status.gather() self.data_collection_status.add_to_tar(tar) self.manifest.add_collection(self.data_collection_status) except Exception as e: @@ -313,7 +313,7 @@ def _data_collection_status_to_tar(self, tar): def _manifest_to_tar(self, tar): try: - self.manifest.gather(None) + self.manifest.gather() self.manifest.add_to_tar(tar) self.add_collection(self.manifest) except Exception as e: diff --git a/metrics_utility/logger.py b/metrics_utility/logger.py index 18e669fe..64620daf 100644 --- a/metrics_utility/logger.py +++ b/metrics_utility/logger.py @@ -3,19 +3,12 @@ import warnings -first = sys.argv[0] - -if first.endswith('manage.py'): +if sys.argv[0].endswith('manage.py'): warnings.simplefilter(action='ignore', category=FutureWarning) -# FIXME: warning logging.basicConfig(format='%(message)s', level=logging.INFO) logger = logging.getLogger(__name__) -# This logger will log all message info and up -logger_info_level = logging.getLogger(__name__) -logger_info_level.setLevel(logging.INFO) - -def debug(): +def logger_debug(): logger.setLevel(logging.DEBUG) diff --git a/metrics_utility/management/commands/build_report.py b/metrics_utility/management/commands/build_report.py index 969bb2cf..3466282e 100644 --- a/metrics_utility/management/commands/build_report.py +++ b/metrics_utility/management/commands/build_report.py @@ -11,7 +11,7 @@ from metrics_utility.automation_controller_billing.report.factory import Factory as ReportFactory from metrics_utility.automation_controller_billing.report_saver.factory import Factory as ReportSaverFactory from metrics_utility.exceptions import BadRequiredEnvVar, BadShipTarget, MissingRequiredEnvVar -from metrics_utility.logger import debug, logger +from metrics_utility.logger import logger, logger_debug from metrics_utility.management.validation import ( date_format_text, handle_directory_ship_target, @@ -120,7 +120,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): if options.get('verbose'): - debug() + logger_debug() handle_env_validation('build') diff --git a/metrics_utility/management/commands/gather_automation_controller_billing_data.py b/metrics_utility/management/commands/gather_automation_controller_billing_data.py index 0cf6632b..f1b8505d 100644 --- a/metrics_utility/management/commands/gather_automation_controller_billing_data.py +++ b/metrics_utility/management/commands/gather_automation_controller_billing_data.py @@ -9,7 +9,7 @@ BadShipTarget, NoAnalyticsCollected, ) -from metrics_utility.logger import debug, logger +from metrics_utility.logger import logger, logger_debug from metrics_utility.management.validation import ( date_format_text, handle_crc_ship_target, @@ -91,7 +91,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): if options.get('verbose'): - debug() + logger_debug() handle_env_validation('gather') opt_since = options.get('since') diff --git a/metrics_utility/test/base/functional/collector_module.py b/metrics_utility/test/base/functional/collector_module.py index f139194f..67b24ef2 100644 --- a/metrics_utility/test/base/functional/collector_module.py +++ b/metrics_utility/test/base/functional/collector_module.py @@ -8,8 +8,8 @@ def config(since, **kwargs): @register('big_table', '1.0', format='csv', description='Testing CSV data - file splitting') -def big_table(full_path, max_data_size, **kwargs): - return simple_csv(full_path, 'big_table', 10, max_data_size) +def big_table(**kwargs): + return simple_csv('big_table', 10) @register( @@ -18,23 +18,23 @@ def big_table(full_path, max_data_size, **kwargs): format='csv', description='Testing CSV data - file splitting 2', ) -def big_table_2(full_path, max_data_size, **kwargs): - return simple_csv(full_path, 'big_table', 3, 800) +def big_table_2(**kwargs): + return simple_csv('big_table', 3, 800) @register('csv_collection_1', '1.0', format='csv', description='CSV 1') -def csv_collection_1(full_path, max_data_size, **kwargs): - return simple_csv(full_path, 'csv_collection_1', 1, max_data_size=100) +def csv_collection_1(**kwargs): + return simple_csv('csv_collection_1', 1) @register('csv_collection_2', '1.0', format='csv', description='CSV 2') -def csv_collection_2(full_path, max_data_size, **kwargs): - return simple_csv(full_path, 'csv_collection_2', 1, max_data_size=200) +def csv_collection_2(**kwargs): + return simple_csv('csv_collection_2', 1) @register('csv_collection_3', '1.0', format='csv', description='CSV 3') -def csv_collection_3(full_path, max_data_size, **kwargs): - return simple_csv(full_path, 'csv_collection_3', 1, max_data_size=300) +def csv_collection_3(**kwargs): + return simple_csv('csv_collection_3', 1) @register( @@ -44,8 +44,8 @@ def csv_collection_3(full_path, max_data_size, **kwargs): description='CSV with slicing 1', fnc_slicing=trivial_slicing, ) -def csv_slicing_1(full_path, max_data_size, **kwargs): - return simple_csv(full_path, 'csv_slicing_1', 1, max_data_size=100) +def csv_slicing_1(**kwargs): + return simple_csv('csv_slicing_1', 1) @register( @@ -55,8 +55,8 @@ def csv_slicing_1(full_path, max_data_size, **kwargs): description='CSV with slicing 2', fnc_slicing=trivial_slicing, ) -def csv_slicing_2(full_path, max_data_size, **kwargs): - return simple_csv(full_path, 'csv_slicing_2', 2, max_data_size=100) +def csv_slicing_2(**kwargs): + return simple_csv('csv_slicing_2', 2) @register('json_collection_1', '1.0', format='json', description='JSON 1') diff --git a/metrics_utility/test/base/functional/collector_module3.py b/metrics_utility/test/base/functional/collector_module3.py index 25354319..e5e43fbe 100644 --- a/metrics_utility/test/base/functional/collector_module3.py +++ b/metrics_utility/test/base/functional/collector_module3.py @@ -13,8 +13,8 @@ def simple_json1(**kwargs): @register('csv_no_slicing_1-2x', '1.0', format='csv', description='CSV No slicing 1') -def csv_no_slicing1(full_path, **kwargs): - return simple_csv(full_path, 'csv_no_slicing_1-2x', 2, 100) +def csv_no_slicing1(**kwargs): + return simple_csv('csv_no_slicing_1-2x', 2) @register( @@ -24,8 +24,8 @@ def csv_no_slicing1(full_path, **kwargs): description='CSV With Slicing 1a', fnc_slicing=trivial_slicing, ) -def csv_with_slicing1a(full_path, **kwargs): - return simple_csv(full_path, 'csv_with_slicing_1-5x', 2, 100) +def csv_with_slicing1a(**kwargs): + return simple_csv('csv_with_slicing_1-5x', 2) @register( @@ -35,13 +35,13 @@ def csv_with_slicing1a(full_path, **kwargs): description='CSV With Slicing 1b', fnc_slicing=trivial_slicing, ) -def csv_with_slicing1b(full_path, **kwargs): - return simple_csv(full_path, 'csv_with_slicing_1-5x', 3, 100) +def csv_with_slicing1b(**kwargs): + return simple_csv('csv_with_slicing_1-5x', 3) @register('csv_no_slicing_2-1x', '1.0', format='csv', description='CSV No slicing 2') -def csv_no_slicing2(full_path, **kwargs): - return simple_csv(full_path, 'csv_no_slicing_2-1x', 1, 100) +def csv_no_slicing2(**kwargs): + return simple_csv('csv_no_slicing_2-1x', 1) @register( @@ -51,13 +51,13 @@ def csv_no_slicing2(full_path, **kwargs): description='CSV With Slicing 2a', fnc_slicing=trivial_slicing, ) -def csv_with_slicing2a(full_path, **kwargs): - return simple_csv(full_path, 'csv_with_slicing_2-3x', 2, 100) +def csv_with_slicing2a(**kwargs): + return simple_csv('csv_with_slicing_2-3x', 2) @register('csv_no_slicing_3-10x', '1.0', format='csv', description='CSV No slicing 3') -def csv_no_slicing3(full_path, **kwargs): - return simple_csv(full_path, 'csv_no_slicing_3-10x', 10, 100) +def csv_no_slicing3(**kwargs): + return simple_csv('csv_no_slicing_3-10x', 10) @register( @@ -67,13 +67,13 @@ def csv_no_slicing3(full_path, **kwargs): description='CSV With Slicing 3', fnc_slicing=trivial_slicing, ) -def csv_with_slicing3(full_path, **kwargs): - return simple_csv(full_path, 'csv_with_slicing_3-2x', 2, 100) +def csv_with_slicing3(**kwargs): + return simple_csv('csv_with_slicing_3-2x', 2) @register('csv_no_slicing_4-12x', '1.0', format='csv', description='CSV No slicing 3') -def csv_no_slicing4(full_path, **kwargs): - return simple_csv(full_path, 'csv_no_slicing_4-12x', 12, 100) +def csv_no_slicing4(**kwargs): + return simple_csv('csv_no_slicing_4-12x', 12) @register( @@ -83,8 +83,8 @@ def csv_no_slicing4(full_path, **kwargs): description='CSV With Slicing 2b', fnc_slicing=trivial_slicing, ) -def csv_with_slicing2b(full_path, **kwargs): - return simple_csv(full_path, 'csv_with_slicing_2-3x', 1, 100) +def csv_with_slicing2b(**kwargs): + return simple_csv('csv_with_slicing_2-3x', 1) @register( @@ -94,5 +94,5 @@ def csv_with_slicing2b(full_path, **kwargs): description='CSV With Slicing 4', fnc_slicing=trivial_slicing, ) -def csv_with_slicing4(full_path, **kwargs): - return simple_csv(full_path, 'csv_with_slicing_4-3x', 3, 100) +def csv_with_slicing4(**kwargs): + return simple_csv('csv_with_slicing_4-3x', 3) diff --git a/metrics_utility/test/base/functional/collector_module4_slicing.py b/metrics_utility/test/base/functional/collector_module4_slicing.py index 63c2c949..72870a39 100644 --- a/metrics_utility/test/base/functional/collector_module4_slicing.py +++ b/metrics_utility/test/base/functional/collector_module4_slicing.py @@ -1,6 +1,5 @@ from base.functional.helpers import ( TIMESTAMP_CSV_LINE_LENGTH, - full_sync_slicing, one_day_slicing, timestamp_csv, ) @@ -19,9 +18,9 @@ def config(since, **kwargs): description='CSVs splitted by date', fnc_slicing=one_day_slicing, ) -def csv_one_day_slicing_1(since, full_path, until, **kwargs): +def csv_one_day_slicing_1(**kwargs): + since, until = kwargs.get('since', None), kwargs.get('until', None) return timestamp_csv( - full_path, 'csv_one_day_slicing_1', 1, 2 * TIMESTAMP_CSV_LINE_LENGTH, @@ -37,31 +36,12 @@ def csv_one_day_slicing_1(since, full_path, until, **kwargs): description='CSVs splitted by size and date', fnc_slicing=one_day_slicing, ) -def csv_one_day_slicing_2(since, full_path, until, **kwargs): +def csv_one_day_slicing_2(**kwargs): + since, until = kwargs.get('since', None), kwargs.get('until', None) return timestamp_csv( - full_path, 'csv_one_day_slicing_2', 2, 2 * TIMESTAMP_CSV_LINE_LENGTH, since=since, until=until, ) - - -@register( - 'csv_full_sync_slicing_1', - '1.0', - format='csv', - description='CSVs splitted by date', - fnc_slicing=full_sync_slicing, - full_sync_interval_days=5, -) -def csv_full_sync_slicing_1(since, full_path, until, **kwargs): - return timestamp_csv( - full_path, - 'csv_full_sync_slicing_1', - 1, - 2 * TIMESTAMP_CSV_LINE_LENGTH, - since=since, - until=until, - ) diff --git a/metrics_utility/test/base/functional/helpers.py b/metrics_utility/test/base/functional/helpers.py index a11ebbc2..fc09adfd 100644 --- a/metrics_utility/test/base/functional/helpers.py +++ b/metrics_utility/test/base/functional/helpers.py @@ -1,18 +1,21 @@ import os -from django.utils.timezone import now, timedelta +from django.utils.timezone import timedelta from metrics_utility.library import CsvFileSplitter +from metrics_utility.library.collectors.util import init_tmp_dir TIMESTAMP_CSV_LINE_LENGTH = 40 -def trivial_slicing(key, last_gather, since, until, **kwargs): +def trivial_slicing(key, **kwargs): + since, until = kwargs.get('since', None), kwargs.get('until', None) return [(since, until)] -def one_day_slicing(key, last_gather, since, until, **kwargs): +def one_day_slicing(key, **kwargs): + since, until = kwargs.get('since', None), kwargs.get('until', None) since = since.replace(hour=0, minute=0, second=0, microsecond=0) until = until.replace(hour=0, minute=0, second=0, microsecond=0) start, end = since, None @@ -22,27 +25,8 @@ def one_day_slicing(key, last_gather, since, until, **kwargs): start = end -def full_sync_slicing(key, last_gather, full_sync_enabled=False, since=None, **kwargs): - """ - If full_sync_enabled is: - - True: Yields 10 time slices in 1-day intervals - - False: Yields slices since 'since' in 1-day intervals - """ - current_time = now().replace(hour=0, minute=0, second=0, microsecond=0) - if full_sync_enabled: - start = current_time - timedelta(days=10) - start = start.replace(hour=0, minute=0, second=0, microsecond=0) - else: - start = since.replace(hour=0, minute=0, second=0, microsecond=0) - - while start < current_time: - end = start + timedelta(days=1) - yield (start, end) - start = end - - -def csv_generator(full_path, file_name, files_cnt, max_data_size, header, line): - file_path = get_file_path(full_path, file_name) +def csv_generator(file_name, files_cnt, max_data_size, header, line): + file_path = get_file_path(file_name) file = CsvFileSplitter(filespec=file_path, max_file_size=max_data_size) # create required number of files (decrease by headers - it's CSV) @@ -53,14 +37,14 @@ def csv_generator(full_path, file_name, files_cnt, max_data_size, header, line): return file.file_list() -def simple_csv(full_path, file_name, files_cnt, max_data_size): +def simple_csv(file_name, files_cnt, max_data_size=100): """CSVs with line length 10 bytes""" header = 'Col1,Col2\n' # 10 chars line = '1234,6789\n' # 10 chars - return csv_generator(full_path, file_name, files_cnt, max_data_size, header, line) + return csv_generator(file_name, files_cnt, max_data_size, header, line) -def timestamp_csv(full_path, file_name, files_cnt, max_data_size, since, until): +def timestamp_csv(file_name, files_cnt, max_data_size, since, until): """CSVs with line length 40 bytes""" header = 'since______________,until______________\n' # 40 chars line = [ @@ -69,10 +53,11 @@ def timestamp_csv(full_path, file_name, files_cnt, max_data_size, since, until): ] # 19 chars line = f'{",".join(line)}\n' # +2 = 40 chars - return csv_generator(full_path, file_name, files_cnt, max_data_size, header, line) + return csv_generator(file_name, files_cnt, max_data_size, header, line) -def get_file_path(path, table): +def get_file_path(table): + path = init_tmp_dir() return os.path.join(path, table + '_table.csv') diff --git a/metrics_utility/test/base/functional/test_gathering.py b/metrics_utility/test/base/functional/test_gathering.py index cabdc145..21e8d5f9 100644 --- a/metrics_utility/test/base/functional/test_gathering.py +++ b/metrics_utility/test/base/functional/test_gathering.py @@ -68,11 +68,6 @@ def test_small_csvs(collector): assert './csv_collection_2.csv' in files.keys() assert './csv_collection_3.csv' in files.keys() - # length defined by @registered function - assert len(files['./csv_collection_1.csv'].read()) == 100 - assert len(files['./csv_collection_2.csv'].read()) == 200 - assert len(files['./csv_collection_3.csv'].read()) == 300 - collector._gather_cleanup() @@ -120,7 +115,6 @@ def test_one_csv_collection_splitted_by_size(collector): assert_common_files(files) assert len(files.keys()) == 1 + _common_files_count() assert './big_table.csv' in files.keys() - assert len(files['./big_table.csv'].read()) == 1000 collector._gather_cleanup() @@ -140,13 +134,13 @@ def test_multiple_collections_multiple_tarballs(mocker, collector): assert_common_files(files) if i == 0: - assert len(files.keys()) == 2 + _common_files_count() + assert len(files.keys()) == 3 + _common_files_count() assert './big_table_2.csv' in files.keys() assert './csv_collection_1.csv' in files.keys() + assert './csv_collection_2.csv' in files.keys() elif i == 1: - assert len(files.keys()) == 2 + _common_files_count() + assert len(files.keys()) == 1 + _common_files_count() assert './big_table_2.csv' in files.keys() - assert './csv_collection_2.csv' in files.keys() elif i == 2: assert len(files.keys()) == 1 + _common_files_count() assert './big_table_2.csv' in files.keys() diff --git a/metrics_utility/test/base/functional/test_slicing.py b/metrics_utility/test/base/functional/test_slicing.py index f37e9e6f..21b7f90b 100644 --- a/metrics_utility/test/base/functional/test_slicing.py +++ b/metrics_utility/test/base/functional/test_slicing.py @@ -94,29 +94,3 @@ def test_slices_by_date_and_size(collector): tgz_files = collector.gather(subset=['config', 'csv_one_day_slicing_2'], since=since, until=until) assert len(tgz_files) == days_to_collect * 2 - - -@pytest.mark.parametrize('last_sync_days_ago', [4, 6]) -def test_slices_by_full_sync(mocker, collector, last_sync_days_ago): - """ - In the collector method `csv_full_sync_slicing_1()` there is 5 days interval for full sync - if `last_sync_days_ago` is: - - 4 days ago, it uses `since` (7 days ago) - - 6 days ago, it uses slicing interval (10 days ago) (in `full_sync_slicing()`) - - """ - last_gathered_entries = { - 'csv_full_sync_slicing_1': (now() - timedelta(days=7)).replace(hour=0, minute=0, second=0, microsecond=0), - 'csv_full_sync_slicing_1_full': now() - timedelta(days=last_sync_days_ago), - } - mocker.patch.object(collector, '_load_last_gathered_entries', return_value=last_gathered_entries) - - tgz_files = collector.gather( - subset=['config', 'csv_full_sync_slicing_1'], - since=last_gathered_entries['csv_full_sync_slicing_1'], - ) - - if last_sync_days_ago == 4: - assert len(tgz_files) == 7 - else: - assert len(tgz_files) == 10 diff --git a/metrics_utility/test/gather/test_jobhostsummary_gather.py b/metrics_utility/test/gather/test_jobhostsummary_gather.py index 777c9592..ec2a6f65 100644 --- a/metrics_utility/test/gather/test_jobhostsummary_gather.py +++ b/metrics_utility/test/gather/test_jobhostsummary_gather.py @@ -345,10 +345,10 @@ def main_host_collection(cleanup_glob, collectors='main_jobevent,main_host', tra # Mock the Collection.gather method to capture success/failure status original_collection_gather = Collection.gather - def mock_collection_gather(self, path): + def mock_collection_gather(self): """Mock collection gather to capture statuses.""" # Call the original method - result = original_collection_gather(self, path) + result = original_collection_gather(self) # Capture the status collection_name = getattr(self, 'filename', 'unknown') diff --git a/metrics_utility/test/gather/test_total_workers_vcpu.py b/metrics_utility/test/gather/test_total_workers_vcpu.py index 4bb4e5fc..e247ba94 100644 --- a/metrics_utility/test/gather/test_total_workers_vcpu.py +++ b/metrics_utility/test/gather/test_total_workers_vcpu.py @@ -1,12 +1,13 @@ import json from datetime import datetime, timezone -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, mock_open, patch import pytest -from metrics_utility.automation_controller_billing.collectors import get_hour_boundaries, total_workers_vcpu -from metrics_utility.exceptions import MetricsException, MissingRequiredEnvVar +from metrics_utility.automation_controller_billing.collectors import total_workers_vcpu +from metrics_utility.exceptions import MissingRequiredEnvVar +from metrics_utility.library.collectors.others.total_workers_vcpu import get_hour_boundaries from metrics_utility.test.util import temporary_env @@ -17,7 +18,7 @@ def test_returns_none_when_not_in_optional_collectors(self): """Test that the function returns None when total_workers_vcpu is not in optional collectors.""" with patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') as mock_get: mock_get.return_value = [] - result = total_workers_vcpu(None, None, None) + result = total_workers_vcpu() assert result is None def test_raises_missing_required_env_var_when_cluster_name_not_set(self): @@ -29,7 +30,7 @@ def test_raises_missing_required_env_var_when_cluster_name_not_set(self): mock_get.return_value = ['total_workers_vcpu'] with temporary_env({'METRICS_UTILITY_CLUSTER_NAME': None}): with pytest.raises(MissingRequiredEnvVar) as exc_info: - total_workers_vcpu(None, None, None) + total_workers_vcpu() assert 'environment variable METRICS_UTILITY_CLUSTER_NAME is not set' in str(exc_info.value) mock_logger.error.assert_called_once_with('environment variable METRICS_UTILITY_CLUSTER_NAME is not set') @@ -38,19 +39,21 @@ def test_returns_hardcoded_value_when_usage_based_billing_disabled(self): """Test that the function returns hardcoded value when METRICS_UTILITY_USAGE_BASED_METERING_ENABLED is not set or false (default behavior).""" with ( patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') as mock_get, - patch('metrics_utility.automation_controller_billing.collectors.logger_info_level') as mock_logger_info, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.logger') as mock_logger, + patch('metrics_utility.automation_controller_billing.collectors.os.path.exists', return_value=True), + patch('builtins.open', mock_open(read_data='fake-token')), ): mock_get.return_value = ['total_workers_vcpu'] # Test when not set (default behavior) with temporary_env({'METRICS_UTILITY_CLUSTER_NAME': 'test-cluster'}): - result = total_workers_vcpu(None, None, None) + result = total_workers_vcpu() assert result['cluster_name'] == 'test-cluster' assert result['total_workers_vcpu'] == 1 assert 'timestamp' in result # Verify the logged JSON contains usage_based_billing_enabled = False and all required fields - logged_json = json.loads(mock_logger_info.info.call_args[0][0]) + logged_json = json.loads(mock_logger.info.call_args[0][0]) assert not logged_json['usage_based_billing_enabled'] assert logged_json['total_workers_vcpu'] == 1 assert 'cluster_name' in logged_json @@ -60,19 +63,21 @@ def test_returns_hardcoded_value_when_usage_based_billing_disabled(self): # Test when explicitly set to false with temporary_env({'METRICS_UTILITY_CLUSTER_NAME': 'test-cluster', 'METRICS_UTILITY_USAGE_BASED_METERING_ENABLED': 'false'}): - result = total_workers_vcpu(None, None, None) + result = total_workers_vcpu() assert result['cluster_name'] == 'test-cluster' assert result['total_workers_vcpu'] == 1 # Verify the logged JSON contains usage_based_billing_enabled = False - logged_json = json.loads(mock_logger_info.info.call_args[0][0]) + logged_json = json.loads(mock_logger.info.call_args[0][0]) assert not logged_json['usage_based_billing_enabled'] def test_usage_based_billing_enabled_case_insensitive(self): """Test that METRICS_UTILITY_USAGE_BASED_METERING_ENABLED is case insensitive.""" with ( patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') as mock_get, - patch('metrics_utility.automation_controller_billing.collectors.PrometheusClient') as mock_prom_client_class, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.PrometheusClient') as mock_prom_client_class, + patch('metrics_utility.automation_controller_billing.collectors.os.path.exists', return_value=True), + patch('builtins.open', mock_open(read_data='fake-token')), ): mock_get.return_value = ['total_workers_vcpu'] @@ -89,7 +94,7 @@ def test_usage_based_billing_enabled_case_insensitive(self): 'METRICS_UTILITY_PROMETHEUS_URL': 'https://prometheus.example.com:9090', } ): - result = total_workers_vcpu(None, None, None) + result = total_workers_vcpu() assert result['cluster_name'] == 'test-cluster' assert result['total_workers_vcpu'] == 8 @@ -97,11 +102,13 @@ def test_uses_default_prometheus_url_when_not_set(self): """Test that the function uses default Prometheus URL when METRICS_UTILITY_PROMETHEUS_URL is not set.""" with ( patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') as mock_get, - patch('metrics_utility.automation_controller_billing.collectors.PrometheusClient') as mock_prom_client_class, - patch('metrics_utility.automation_controller_billing.collectors.get_total_workers_cpu') as mock_get_cpu, - patch('metrics_utility.automation_controller_billing.collectors.get_cpu_timeline') as mock_get_timeline, - patch('metrics_utility.automation_controller_billing.collectors.logger') as mock_logger, - patch('metrics_utility.automation_controller_billing.collectors.logger_info_level') as mock_logger_info, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.PrometheusClient') as mock_prom_client_class, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.get_total_workers_cpu') as mock_get_cpu, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.get_cpu_timeline') as mock_get_timeline, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.logger') as mock_logger, + patch('metrics_utility.automation_controller_billing.collectors.logger') as mock_collectors_logger, + patch('metrics_utility.automation_controller_billing.collectors.os.path.exists', return_value=True), + patch('builtins.open', mock_open(read_data='fake-token')), ): mock_get.return_value = ['total_workers_vcpu'] @@ -123,16 +130,20 @@ def test_uses_default_prometheus_url_when_not_set(self): 'METRICS_UTILITY_PROMETHEUS_URL': None, } ): - result = total_workers_vcpu(None, None, None) + result = total_workers_vcpu() # Verify it uses the default URL - mock_prom_client_class.assert_called_once_with(url='https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091') + mock_prom_client_class.assert_called_once_with( + url='https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091', + ca_cert_path='/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt', + token='fake-token', + ) expected_message = ( - 'environment variable METRICS_UTILITY_PROMETHEUS_URL is not set,' - ' default https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 will be assigned' + 'environment variable METRICS_UTILITY_PROMETHEUS_URL is not set, ' + 'default https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 will be assigned' ) # Check that the expected message was called (not necessarily the last call) - mock_logger.info.assert_any_call(expected_message) + mock_collectors_logger.info.assert_any_call(expected_message) # Also verify the total_workers_vcpu value was logged mock_logger.debug.assert_called_with('total_workers_vcpu: 16.0') @@ -145,7 +156,7 @@ def test_uses_default_prometheus_url_when_not_set(self): assert result['total_workers_vcpu'] == 16 # Verify that the logged info contains all expected fields - logged_json = json.loads(mock_logger_info.info.call_args[0][0]) + logged_json = json.loads(mock_logger.info.call_args[0][0]) assert 'cluster_name' in logged_json assert 'collection_timestamp' in logged_json assert 'start_timestamp' in logged_json @@ -157,11 +168,13 @@ def test_uses_default_prometheus_url_when_not_set(self): assert logged_json['usage_based_billing_enabled'] is True assert 'max_over_time(sum(machine_cpu_cores)[59m59s:5m]' in logged_json['promql_query'] - def test_prometheus_client_creation_failure_raises_metrics_exception(self): - """Test that PrometheusClient creation failure raises MetricsException.""" + def test_prometheus_client_creation_failure_raises_exception(self): + """Test that PrometheusClient creation failure raises an exception.""" with ( patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') as mock_get, - patch('metrics_utility.automation_controller_billing.collectors.PrometheusClient') as mock_prom_client_class, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.PrometheusClient') as mock_prom_client_class, + patch('metrics_utility.automation_controller_billing.collectors.os.path.exists', return_value=True), + patch('builtins.open', mock_open(read_data='fake-token')), ): mock_get.return_value = ['total_workers_vcpu'] mock_prom_client_class.side_effect = Exception('Failed to create Prometheus client') @@ -173,16 +186,18 @@ def test_prometheus_client_creation_failure_raises_metrics_exception(self): 'METRICS_UTILITY_PROMETHEUS_URL': 'https://prometheus.example.com:9090', } ): - with pytest.raises(MetricsException) as exc_info: - total_workers_vcpu(None, None, None) + with pytest.raises(Exception) as exc_info: + total_workers_vcpu() - assert 'Can not create a prometheus api client ERROR:' in str(exc_info.value) + assert 'Failed to create Prometheus client' in str(exc_info.value) - def test_prometheus_query_failure_raises_metrics_exception(self): - """Test that Prometheus query failure raises MetricsException.""" + def test_prometheus_query_failure_raises_exception(self): + """Test that Prometheus query failure raises an exception.""" with ( patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') as mock_get, - patch('metrics_utility.automation_controller_billing.collectors.PrometheusClient') as mock_prom_client_class, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.PrometheusClient') as mock_prom_client_class, + patch('metrics_utility.automation_controller_billing.collectors.os.path.exists', return_value=True), + patch('builtins.open', mock_open(read_data='fake-token')), ): mock_get.return_value = ['total_workers_vcpu'] @@ -198,17 +213,19 @@ def test_prometheus_query_failure_raises_metrics_exception(self): 'METRICS_UTILITY_PROMETHEUS_URL': 'https://prometheus.example.com:9090', } ): - with pytest.raises(MetricsException) as exc_info: - total_workers_vcpu(None, None, None) + with pytest.raises(Exception) as exc_info: + total_workers_vcpu() - assert 'Unexpected error when retrieving nodes:' in str(exc_info.value) + assert 'Prometheus query failed' in str(exc_info.value) - def test_prometheus_query_returns_none_raises_metrics_exception(self): - """Test that the function raises MetricsException when Prometheus query returns None (no data available).""" + def test_prometheus_query_returns_none_when_no_data_available(self): + """Test that the function returns None when Prometheus query returns None (no data available).""" with ( patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') as mock_get, - patch('metrics_utility.automation_controller_billing.collectors.PrometheusClient') as mock_prom_client_class, - patch('metrics_utility.automation_controller_billing.collectors.logger') as mock_logger, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.PrometheusClient') as mock_prom_client_class, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.logger') as mock_logger, + patch('metrics_utility.automation_controller_billing.collectors.os.path.exists', return_value=True), + patch('builtins.open', mock_open(read_data='fake-token')), ): mock_get.return_value = ['total_workers_vcpu'] @@ -224,22 +241,23 @@ def test_prometheus_query_returns_none_raises_metrics_exception(self): 'METRICS_UTILITY_PROMETHEUS_URL': 'https://prometheus.example.com:9090', } ): - with pytest.raises(MetricsException) as exc_info: - total_workers_vcpu(None, None, None) + result = total_workers_vcpu() - # Verify the exception message - assert 'No data availble yet, the cluster is probably running for less than an hour' in str(exc_info.value) + # Verify the function returns None + assert result is None # Verify the warning message was logged mock_logger.debug.assert_called_with('total_workers_vcpu: None') - mock_logger.warning.assert_called_with('No data availble yet, the cluster is probably running for less than an hour') + mock_logger.warning.assert_called_with('No data available yet, the cluster is probably running for less than an hour') def test_successful_prometheus_query_with_vcpu_calculation(self): """Test successful Prometheus query with vCPU calculation.""" with ( patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') as mock_get, - patch('metrics_utility.automation_controller_billing.collectors.PrometheusClient') as mock_prom_client_class, - patch('metrics_utility.automation_controller_billing.collectors.logger_info_level') as mock_logger_info, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.PrometheusClient') as mock_prom_client_class, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.logger') as mock_logger, + patch('metrics_utility.automation_controller_billing.collectors.os.path.exists', return_value=True), + patch('builtins.open', mock_open(read_data='fake-token')), ): mock_get.return_value = ['total_workers_vcpu'] @@ -255,14 +273,18 @@ def test_successful_prometheus_query_with_vcpu_calculation(self): 'METRICS_UTILITY_PROMETHEUS_URL': 'https://prometheus.example.com:9090', } ): - result = total_workers_vcpu(None, None, None) + result = total_workers_vcpu() assert result['cluster_name'] == 'my-cluster' assert result['total_workers_vcpu'] == 24 # Should be converted to int assert 'timestamp' in result # Verify PrometheusClient was created with correct parameters - mock_prom_client_class.assert_called_once_with(url='https://prometheus.example.com:9090') + mock_prom_client_class.assert_called_once_with( + url='https://prometheus.example.com:9090', + ca_cert_path='/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt', + token='fake-token', + ) # Verify the query was called with correct PromQL mock_prom_client.get_current_value.assert_called_once() @@ -271,8 +293,8 @@ def test_successful_prometheus_query_with_vcpu_calculation(self): assert '@' in query_call # Should contain timestamp # Verify logging - mock_logger_info.info.assert_called_once() - logged_json = json.loads(mock_logger_info.info.call_args[0][0]) + mock_logger.info.assert_called_once() + logged_json = json.loads(mock_logger.info.call_args[0][0]) assert logged_json['cluster_name'] == 'my-cluster' assert logged_json['total_workers_vcpu'] == 24 assert logged_json['usage_based_billing_enabled'] @@ -281,7 +303,9 @@ def test_prometheus_client_initialized_correctly(self): """Test that PrometheusClient is initialized correctly.""" with ( patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') as mock_get, - patch('metrics_utility.automation_controller_billing.collectors.PrometheusClient') as mock_prom_client_class, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.PrometheusClient') as mock_prom_client_class, + patch('metrics_utility.automation_controller_billing.collectors.os.path.exists', return_value=True), + patch('builtins.open', mock_open(read_data='fake-token')), ): mock_get.return_value = ['total_workers_vcpu'] @@ -297,17 +321,23 @@ def test_prometheus_client_initialized_correctly(self): 'METRICS_UTILITY_PROMETHEUS_URL': 'https://prometheus.example.com:9090', } ): - total_workers_vcpu(None, None, None) + total_workers_vcpu() # Verify PrometheusClient was created correctly - mock_prom_client_class.assert_called_once_with(url='https://prometheus.example.com:9090') + mock_prom_client_class.assert_called_once_with( + url='https://prometheus.example.com:9090', + ca_cert_path='/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt', + token='fake-token', + ) def test_prometheus_query_uses_correct_promql_with_hour_boundaries(self): """Test that the Prometheus query uses correct PromQL with hour boundaries.""" with ( patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') as mock_get, - patch('metrics_utility.automation_controller_billing.collectors.PrometheusClient') as mock_prom_client_class, - patch('metrics_utility.automation_controller_billing.collectors.datetime') as mock_datetime, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.PrometheusClient') as mock_prom_client_class, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.datetime') as mock_datetime, + patch('metrics_utility.automation_controller_billing.collectors.os.path.exists', return_value=True), + patch('builtins.open', mock_open(read_data='fake-token')), ): mock_get.return_value = ['total_workers_vcpu'] @@ -329,7 +359,7 @@ def test_prometheus_query_uses_correct_promql_with_hour_boundaries(self): 'METRICS_UTILITY_PROMETHEUS_URL': 'https://prometheus.example.com:9090', } ): - total_workers_vcpu(None, None, None) + total_workers_vcpu() # Calculate expected timestamp current_ts = mock_now.timestamp() @@ -345,7 +375,9 @@ def test_vcpu_value_converted_to_int(self): """Test that vCPU values from Prometheus (floats) are properly converted to integers.""" with ( patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') as mock_get, - patch('metrics_utility.automation_controller_billing.collectors.PrometheusClient') as mock_prom_client_class, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.PrometheusClient') as mock_prom_client_class, + patch('metrics_utility.automation_controller_billing.collectors.os.path.exists', return_value=True), + patch('builtins.open', mock_open(read_data='fake-token')), ): mock_get.return_value = ['total_workers_vcpu'] @@ -361,12 +393,12 @@ def test_vcpu_value_converted_to_int(self): 'METRICS_UTILITY_PROMETHEUS_URL': 'https://prometheus.example.com:9090', } ): - result = total_workers_vcpu(None, None, None) + result = total_workers_vcpu() assert result['total_workers_vcpu'] == 15 # Should be truncated to int assert isinstance(result['total_workers_vcpu'], int) - @patch('metrics_utility.automation_controller_billing.collectors.datetime') + @patch('metrics_utility.library.collectors.others.total_workers_vcpu.datetime') def test_timestamp_in_output_with_hour_boundaries(self, mock_datetime): """Test that the function includes proper timestamp based on hour boundaries.""" # Mock the datetime.now() call @@ -377,8 +409,10 @@ def test_timestamp_in_output_with_hour_boundaries(self, mock_datetime): with ( patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') as mock_get, - patch('metrics_utility.automation_controller_billing.collectors.PrometheusClient') as mock_prom_client_class, - patch('metrics_utility.automation_controller_billing.collectors.logger_info_level') as mock_logger_info, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.PrometheusClient') as mock_prom_client_class, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.logger') as mock_logger, + patch('metrics_utility.automation_controller_billing.collectors.os.path.exists', return_value=True), + patch('builtins.open', mock_open(read_data='fake-token')), ): mock_get.return_value = ['total_workers_vcpu'] @@ -394,7 +428,7 @@ def test_timestamp_in_output_with_hour_boundaries(self, mock_datetime): 'METRICS_UTILITY_PROMETHEUS_URL': 'https://prometheus.example.com:9090', } ): - result = total_workers_vcpu(None, None, None) + result = total_workers_vcpu() # Calculate expected timestamp current_ts = mock_now.timestamp() @@ -406,8 +440,8 @@ def test_timestamp_in_output_with_hour_boundaries(self, mock_datetime): assert result['timestamp'] == expected_timestamp # Check logged JSON - mock_logger_info.info.assert_called_once() - logged_json = json.loads(mock_logger_info.info.call_args[0][0]) + mock_logger.info.assert_called_once() + logged_json = json.loads(mock_logger.info.call_args[0][0]) assert logged_json['end_timestamp'] == expected_timestamp assert logged_json['cluster_name'] == 'test-cluster' assert logged_json['total_workers_vcpu'] == 8 @@ -417,17 +451,19 @@ def test_usage_based_billing_disabled_unset_returns_hardcoded_value(self): """Test that when METRICS_UTILITY_USAGE_BASED_METERING_ENABLED is unset, it returns hardcoded value.""" with ( patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') as mock_get, - patch('metrics_utility.automation_controller_billing.collectors.logger_info_level') as mock_logger_info, + patch('metrics_utility.library.collectors.others.total_workers_vcpu.logger') as mock_logger, + patch('metrics_utility.automation_controller_billing.collectors.os.path.exists', return_value=True), + patch('builtins.open', mock_open(read_data='fake-token')), ): mock_get.return_value = ['total_workers_vcpu'] with temporary_env({'METRICS_UTILITY_CLUSTER_NAME': 'test-cluster', 'METRICS_UTILITY_USAGE_BASED_METERING_ENABLED': None}): - result = total_workers_vcpu(None, None, None) + result = total_workers_vcpu() assert result['cluster_name'] == 'test-cluster' assert result['total_workers_vcpu'] == 1 # Verify the logged JSON contains usage_based_billing_enabled = False - logged_json = json.loads(mock_logger_info.info.call_args[0][0]) + logged_json = json.loads(mock_logger.info.call_args[0][0]) assert not logged_json['usage_based_billing_enabled'] diff --git a/metrics_utility/test/test_anonymized_rollups/test_from_gather_to_json.py b/metrics_utility/test/test_anonymized_rollups/test_from_gather_to_json.py index 86d93e0e..a228c0ab 100644 --- a/metrics_utility/test/test_anonymized_rollups/test_from_gather_to_json.py +++ b/metrics_utility/test/test_anonymized_rollups/test_from_gather_to_json.py @@ -37,7 +37,7 @@ def test_from_gather_to_json(cleanup_glob): print(json_data) # save as json inside rollups/2025/06/13/anonymized.json - json_path = f'./out/rollups/{2025}/06/13/anonymized.json' + json_path = './out/rollups/2025/06/13/anonymized.json' # create the dir os.makedirs(os.path.dirname(json_path), exist_ok=True) diff --git a/metrics_utility/test/test_automation_controller_billing_helpers.py b/metrics_utility/test/test_automation_controller_billing_helpers.py index daf20246..17736e40 100644 --- a/metrics_utility/test/test_automation_controller_billing_helpers.py +++ b/metrics_utility/test/test_automation_controller_billing_helpers.py @@ -5,62 +5,10 @@ from metrics_utility.automation_controller_billing.helpers import ( datetime_hook, - get_config_and_settings_from_db, get_last_entries_from_db, ) -class TestGetLicenseInfoFromDb: - """Test cases for get_config_and_settings_from_db function""" - - @patch('metrics_utility.automation_controller_billing.helpers.connection') - def test_successful_license_retrieval(self, mock_connection): - """Test successful license information retrieval""" - # Setup - mock_cursor = MagicMock() - mock_connection.cursor.return_value.__enter__.return_value = mock_cursor - mock_cursor.fetchall.return_value = [ - ('LICENSE', '{"license_type": "enterprise", "product_name": "AWX"}'), - ('SUBSCRIPTION_NAME', '"Red Hat Ansible Automation Platform"'), - ('INSTALL_UUID', '"12345-67890"'), - ] - - # Execute - license_info, settings_info = get_config_and_settings_from_db() - - # Assert license info (only LICENSE field data) - expected_license = { - 'license_type': 'enterprise', - 'product_name': 'AWX', - } - assert license_info == expected_license - - # Assert settings info (other fields) - expected_settings = { - 'subscription_name': 'Red Hat Ansible Automation Platform', - 'install_uuid': '12345-67890', - } - assert settings_info == expected_settings - - mock_cursor.execute.assert_called() - - @patch('metrics_utility.automation_controller_billing.helpers.connection') - def test_empty_database_result(self, mock_connection): - """Test when database returns no license information""" - # Setup - mock_cursor = MagicMock() - mock_connection.cursor.return_value.__enter__.return_value = mock_cursor - mock_cursor.fetchall.return_value = [] - mock_cursor.fetchone.return_value = None # No version found - - # Execute - license_info, settings_info = get_config_and_settings_from_db() - - # Assert both should be empty - assert license_info == {} - assert settings_info == {} - - class TestGetLastEntriesFromDb: """Test cases for get_last_entries_from_db function""" @@ -155,31 +103,18 @@ class TestIntegration: @patch('metrics_utility.automation_controller_billing.helpers.connection') def test_functions_work_with_real_data(self, mock_connection): - """Test that all helper functions work with realistic data""" + """Test that helper functions work with realistic data""" # Setup realistic database responses mock_cursor = MagicMock() mock_connection.cursor.return_value.__enter__.return_value = mock_cursor - # Setup data for all function calls in sequence: - # 1. get_config_and_settings_from_db() - fetchall() - # 3. get_last_entries_from_db() - fetchone() - mock_cursor.fetchall.return_value = [ - ('LICENSE', '{"license_type": "enterprise"}'), - ('SUBSCRIPTION_NAME', '"Red Hat AAP"'), - ('ABC', '"1.2.3"'), - ] + # Setup data for get_last_entries_from_db() - fetchone() test_json = '"{\\"config\\": \\"2024-01-01T00:00:00Z\\", \\"jobs\\": \\"2024-01-02T00:00:00Z\\"}"' # Last entries result mock_cursor.fetchone.return_value = (test_json,) - # Execute all functions - license_info, settings_info = get_config_and_settings_from_db() + # Execute function entries = get_last_entries_from_db() - # Assert all return expected realistic data - assert license_info == { - 'license_type': 'enterprise', - } - assert settings_info.get('abc') == '1.2.3' # datetime_hook parses datetime strings to datetime objects expected_entries = { 'config': datetime(2024, 1, 1, 0, 0, tzinfo=timezone.utc), diff --git a/metrics_utility/test/test_collectors.py b/metrics_utility/test/test_collectors.py index 6fc7ded6..821a294f 100644 --- a/metrics_utility/test/test_collectors.py +++ b/metrics_utility/test/test_collectors.py @@ -2,15 +2,13 @@ from django.db.utils import ProgrammingError -from metrics_utility.automation_controller_billing.collectors import ( - main_indirectmanagednodeaudit_table, -) +from metrics_utility.automation_controller_billing.collectors import main_indirectmanagednodeaudit_table class TestMainIndirectManagedNodeAuditTable: """Test cases for the main_indirectmanagednodeaudit_table function""" - @patch('metrics_utility.automation_controller_billing.collectors._copy_table') + @patch('metrics_utility.library.collectors.controller.main_indirectmanagednodeaudit.copy_table') @patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') def test_main_indirectmanagednodeaudit_table_success(self, mock_get_optional_collectors, mock_copy_table): """Test successful execution when table exists""" @@ -24,15 +22,14 @@ def test_main_indirectmanagednodeaudit_table_success(self, mock_get_optional_col until.isoformat.return_value = '2024-01-02T00:00:00' # Execute - result = main_indirectmanagednodeaudit_table(since=since, full_path='/test/path', until=until) + result = main_indirectmanagednodeaudit_table(since=since, until=until) # Assert assert result == ['test_file.csv'] mock_copy_table.assert_called_once() call_args = mock_copy_table.call_args assert call_args[1]['table'] == 'main_indirectmanagednodeaudit' - assert 'COPY' in call_args[1]['query'] - assert call_args[1]['path'] == '/test/path' + assert 'SELECT' in call_args[1]['query'] @patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') def test_main_indirectmanagednodeaudit_table_not_in_optional_collectors(self, mock_get_optional_collectors): @@ -41,13 +38,13 @@ def test_main_indirectmanagednodeaudit_table_not_in_optional_collectors(self, mo mock_get_optional_collectors.return_value = {'other_collector'} # Execute - result = main_indirectmanagednodeaudit_table(since=Mock(), full_path='/test/path', until=Mock()) + result = main_indirectmanagednodeaudit_table(since=Mock(), until=Mock()) # Assert assert result is None @patch('metrics_utility.automation_controller_billing.collectors.logger') - @patch('metrics_utility.automation_controller_billing.collectors._copy_table') + @patch('metrics_utility.library.collectors.controller.main_indirectmanagednodeaudit.copy_table') @patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') def test_main_indirectmanagednodeaudit_table_programming_error(self, mock_get_optional_collectors, mock_copy_table, mock_logger): """Test graceful handling when table doesn't exist (ProgrammingError)""" @@ -62,7 +59,7 @@ def test_main_indirectmanagednodeaudit_table_programming_error(self, mock_get_op until.isoformat.return_value = '2024-01-02T00:00:00' # Execute - result = main_indirectmanagednodeaudit_table(since=since, full_path='/test/path', until=until) + result = main_indirectmanagednodeaudit_table(since=since, until=until) # Assert assert result is None @@ -72,7 +69,7 @@ def test_main_indirectmanagednodeaudit_table_programming_error(self, mock_get_op assert 'Falling back to behavior without indirect managed node audit data.' in warning_call[0][0] assert warning_call[0][1] is mock_copy_table.side_effect - @patch('metrics_utility.automation_controller_billing.collectors._copy_table') + @patch('metrics_utility.library.collectors.controller.main_indirectmanagednodeaudit.copy_table') @patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') def test_main_indirectmanagednodeaudit_table_query_format(self, mock_get_optional_collectors, mock_copy_table): """Test that the SQL query contains expected elements""" @@ -86,7 +83,7 @@ def test_main_indirectmanagednodeaudit_table_query_format(self, mock_get_optiona until.isoformat.return_value = '2024-01-02T00:00:00' # Execute - main_indirectmanagednodeaudit_table(since=since, full_path='/test/path', until=until) + main_indirectmanagednodeaudit_table(since=since, until=until) # Assert mock_copy_table.assert_called_once() @@ -105,7 +102,7 @@ def test_main_indirectmanagednodeaudit_table_query_format(self, mock_get_optiona assert '2024-01-02T00:00:00' in query @patch('metrics_utility.automation_controller_billing.collectors.logger') - @patch('metrics_utility.automation_controller_billing.collectors._copy_table') + @patch('metrics_utility.library.collectors.controller.main_indirectmanagednodeaudit.copy_table') @patch('metrics_utility.automation_controller_billing.collectors.get_optional_collectors') def test_main_indirectmanagednodeaudit_table_logs_specific_error(self, mock_get_optional_collectors, mock_copy_table, mock_logger): """Test that the specific error message is logged correctly""" @@ -120,7 +117,7 @@ def test_main_indirectmanagednodeaudit_table_logs_specific_error(self, mock_get_ until.isoformat.return_value = '2024-01-02T00:00:00' # Execute - result = main_indirectmanagednodeaudit_table(since=since, full_path='/test/path', until=until) + result = main_indirectmanagednodeaudit_table(since=since, until=until) # Assert assert result is None diff --git a/metrics_utility/test/test_kubernetes_client.py b/metrics_utility/test/test_kubernetes_client.py deleted file mode 100644 index b716e211..00000000 --- a/metrics_utility/test/test_kubernetes_client.py +++ /dev/null @@ -1,101 +0,0 @@ -""" -Unit tests for simplified KubernetesClient class. - -This module contains comprehensive tests for the simplified KubernetesClient class, -which assumes running in a Kubernetes pod with mounted service account files. -""" - -from unittest.mock import mock_open, patch - -import pytest - -from metrics_utility.automation_controller_billing.kubernetes_client import CA_CERT_PATH, KubernetesClient -from metrics_utility.exceptions import MetricsException - - -class TestKubernetesClient: - """Test suite for the simplified KubernetesClient class.""" - - def test_init_success(self): - """Test successful initialization when service account token file exists.""" - with ( - patch('os.path.exists') as mock_exists, - ): - # Setup mocks - token file exists - mock_exists.side_effect = lambda path: path == '/var/run/secrets/kubernetes.io/serviceaccount/token' - - # Create client - if this succeeds without exception, the test passes - KubernetesClient() - - def test_init_failure_no_token(self): - """Test initialization failure when token file doesn't exist.""" - with ( - patch('os.path.exists') as mock_exists, - ): - # Setup mocks - token file doesn't exist - mock_exists.return_value = False - - # Test that MetricsException is raised - with pytest.raises(MetricsException, match='Service account token not found'): - KubernetesClient() - - def test_init_failure_no_files(self): - """Test initialization failure when no service account files exist.""" - with ( - patch('os.path.exists', return_value=False), - ): - # Test that MetricsException is raised - with pytest.raises(MetricsException, match='Service account token not found'): - KubernetesClient() - - def test_get_current_token_success(self): - """Test successful token retrieval.""" - with ( - patch('os.path.exists', return_value=True), - patch('builtins.open', mock_open(read_data='test-token-12345\n')), - ): - # Create client and test - client_instance = KubernetesClient() - token = client_instance.get_current_token() - - # Assertions - assert token == 'test-token-12345' - - def test_get_current_token_failure(self): - """Test token retrieval failure when file cannot be read.""" - with ( - patch('os.path.exists', return_value=True), - patch('builtins.open', side_effect=IOError('Permission denied')), - ): - # Create client and test - client_instance = KubernetesClient() - - with pytest.raises(MetricsException, match='Error reading token'): - client_instance.get_current_token() - - def test_multiple_token_operations(self): - """Test multiple token operations on the same client instance.""" - with ( - patch('os.path.exists', return_value=True), - patch('builtins.open', mock_open(read_data='multi-op-token\n')), - ): - # Create client and perform multiple operations - client_instance = KubernetesClient() - - token1 = client_instance.get_current_token() - token2 = client_instance.get_current_token() - - # Assertions - both calls should return the same token - assert token1 == 'multi-op-token' - assert token2 == 'multi-op-token' - - def test_get_ca_cert_path(self): - """Test that get_ca_cert_path returns the correct CA certificate path.""" - with patch('os.path.exists', return_value=True): - # Create client and test - client_instance = KubernetesClient() - ca_cert_path = client_instance.get_ca_cert_path() - - # Assertions - assert ca_cert_path == CA_CERT_PATH - assert ca_cert_path == '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' diff --git a/metrics_utility/test/test_prometheus_client.py b/metrics_utility/test/test_prometheus_client.py index cfa8937a..15314067 100644 --- a/metrics_utility/test/test_prometheus_client.py +++ b/metrics_utility/test/test_prometheus_client.py @@ -2,7 +2,7 @@ Unit tests for PrometheusClient class. This module contains comprehensive tests for the PrometheusClient class, -including mocking of HTTP requests and Kubernetes client interactions. +including mocking of HTTP requests. """ import json @@ -12,124 +12,78 @@ import pytest import requests -from metrics_utility.automation_controller_billing.prometheus_client import PrometheusClient -from metrics_utility.exceptions import MetricsException +from metrics_utility.library.collectors.others.prometheus_client import PrometheusClient class TestPrometheusClient: """Test cases for PrometheusClient class.""" - def _setup_kubernetes_client_mock(self, mock_k8s_client, token='test-token'): - """Helper method to set up KubernetesClient mock with common return values.""" - mock_k8s_client.get_current_token.return_value = token - mock_k8s_client.get_ca_cert_path.return_value = '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' - return mock_k8s_client - - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_init_success_with_token(self, mock_k8s_client_class): + def test_init_success_with_token(self): """Test successful initialization with valid token.""" - # Setup mock - mock_k8s_client = MagicMock() - self._setup_kubernetes_client_mock(mock_k8s_client, 'test-token-12345') - mock_k8s_client_class.return_value = mock_k8s_client - - with patch('os.path.exists', return_value=True): - # Create client - client = PrometheusClient(url='https://prometheus.example.com:9090') - - # Assertions - assert client.url == 'https://prometheus.example.com:9090' - assert client.token == 'test-token-12345' - assert client.timeout == 30 # default - assert client.session is not None - mock_k8s_client.get_current_token.assert_called_once() - - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_init_success_with_mounted_token(self, mock_k8s_client_class): - """Test successful initialization requesting mounted token.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'mounted-token-67890' - mock_k8s_client.get_ca_cert_path.return_value = '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' - mock_k8s_client_class.return_value = mock_k8s_client - - with patch('os.path.exists', return_value=True): - # Create client - client = PrometheusClient(url='https://prometheus.example.com:9090', timeout=60) - - # Assertions - assert client.url == 'https://prometheus.example.com:9090' - assert client.token == 'mounted-token-67890' - assert client.timeout == 60 - assert client.session is not None - mock_k8s_client.get_current_token.assert_called_once() - - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_init_failure_no_token(self, mock_k8s_client_class): - """Test initialization failure when no token can be obtained.""" - # Setup mock to return None - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = None - mock_k8s_client.get_ca_cert_path.return_value = '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' - mock_k8s_client_class.return_value = mock_k8s_client - - # Test that MetricsException is raised - with pytest.raises(MetricsException, match='Unable to retrieve the token for the current service account'): - PrometheusClient(url='https://prometheus.example.com:9090') - - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_init_url_trailing_slash_removal(self, mock_k8s_client_class): + # Create client + client = PrometheusClient( + url='https://prometheus.example.com:9090', + token='test-token-12345', + ca_cert_path='/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt', + ) + + # Assertions + assert client.url == 'https://prometheus.example.com:9090' + assert client.timeout == 30 # default + assert client.session is not None + assert client.session.headers['Authorization'] == 'Bearer test-token-12345' + + def test_init_success_with_custom_timeout(self): + """Test successful initialization with custom timeout.""" + # Create client + client = PrometheusClient( + url='https://prometheus.example.com:9090', + timeout=60, + token='mounted-token-67890', + ca_cert_path='/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt', + ) + + # Assertions + assert client.url == 'https://prometheus.example.com:9090' + assert client.timeout == 60 + assert client.session is not None + assert client.session.headers['Authorization'] == 'Bearer mounted-token-67890' + + def test_init_without_token(self): + """Test initialization without authentication token.""" + # Create client without token + client = PrometheusClient(url='https://prometheus.example.com:9090') + + # Assertions - should work fine without token (unauthenticated) + assert client.url == 'https://prometheus.example.com:9090' + assert client.timeout == 30 + assert client.session is not None + assert 'Authorization' not in client.session.headers + + def test_init_url_trailing_slash_removal(self): """Test that trailing slash is removed from URL.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client.get_ca_cert_path.return_value = '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' - mock_k8s_client_class.return_value = mock_k8s_client + # Create client with trailing slash + client = PrometheusClient(url='https://prometheus.example.com:9090/', token='test-token') - with patch('os.path.exists', return_value=True): - # Create client with trailing slash - client = PrometheusClient(url='https://prometheus.example.com:9090/') - - # Assertions - assert client.url == 'https://prometheus.example.com:9090' + # Assertions + assert client.url == 'https://prometheus.example.com:9090' - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_setup_session_with_token(self, mock_k8s_client_class): + def test_setup_session_with_token(self): """Test session setup with authentication token.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token-12345' - mock_k8s_client.get_ca_cert_path.return_value = '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' - mock_k8s_client_class.return_value = mock_k8s_client - - with ( - patch('urllib3.disable_warnings') as mock_disable_warnings, - patch('os.path.exists') as mock_exists, - ): - # Test when CA certificate exists - mock_exists.return_value = True - client = PrometheusClient(url='https://prometheus.example.com:9090') - - # Assertions - assert client.session.headers['Authorization'] == 'Bearer test-token-12345' - assert client.session.headers['Content-Type'] == 'application/x-www-form-urlencoded' - assert client.session.verify == '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' - assert client.ca_cert_path == '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' - mock_disable_warnings.assert_not_called() - - # Test when CA certificate doesn't exist - should raise exception - mock_exists.return_value = False - with pytest.raises(MetricsException, match='CA_CERT not found at'): - PrometheusClient(url='https://prometheus.example.com:9090') - - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_query_success(self, mock_k8s_client_class): + # Create client with token and CA cert + client = PrometheusClient( + url='https://prometheus.example.com:9090', + token='test-token-12345', + ca_cert_path='/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt', + ) + + # Assertions + assert client.session.headers['Authorization'] == 'Bearer test-token-12345' + assert client.session.headers['Content-Type'] == 'application/x-www-form-urlencoded' + assert client.session.verify == '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' + + def test_query_success(self): """Test successful query execution.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - # Mock response data mock_response_data = {'status': 'success', 'data': {'result': [{'metric': {'__name__': 'test_metric'}, 'value': [1640995200, '42.0']}]}} @@ -140,21 +94,15 @@ def test_query_success(self, mock_k8s_client_class): mock_get.return_value = mock_response # Create client and execute query - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') result = client.query('test_metric') # Assertions assert result == mock_response_data['data']['result'] mock_get.assert_called_once_with('https://prometheus.example.com:9090/api/v1/query', params={'query': 'test_metric'}, timeout=30) - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_query_with_time_param(self, mock_k8s_client_class): + def test_query_with_time_param(self): """Test query execution with time parameter.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - # Mock response data mock_response_data = {'status': 'success', 'data': {'result': []}} @@ -165,7 +113,7 @@ def test_query_with_time_param(self, mock_k8s_client_class): mock_get.return_value = mock_response # Create client and execute query with time - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') result = client.query('test_metric', time_param=1640995200.0) # Assertions @@ -174,14 +122,8 @@ def test_query_with_time_param(self, mock_k8s_client_class): 'https://prometheus.example.com:9090/api/v1/query', params={'query': 'test_metric', 'time': 1640995200.0}, timeout=30 ) - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_query_prometheus_api_error(self, mock_k8s_client_class): + def test_query_prometheus_api_error(self): """Test query failure with Prometheus API error.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - # Mock error response mock_response_data = {'status': 'error', 'error': 'invalid query: parse error at position 5'} @@ -192,19 +134,13 @@ def test_query_prometheus_api_error(self, mock_k8s_client_class): mock_get.return_value = mock_response # Create client and test error - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') - with pytest.raises(MetricsException, match='Prometheus API error: invalid query: parse error at position 5'): + with pytest.raises(Exception, match='Prometheus API error: invalid query: parse error at position 5'): client.query('invalid_query') - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_query_http_error(self, mock_k8s_client_class): + def test_query_http_error(self): """Test query failure with HTTP error.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - with patch.object(requests.Session, 'get') as mock_get: mock_response = MagicMock() mock_response.status_code = 404 @@ -212,53 +148,35 @@ def test_query_http_error(self, mock_k8s_client_class): mock_get.return_value = mock_response # Create client and test error - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') - with pytest.raises(MetricsException, match='HTTP error 404: Not Found'): + with pytest.raises(Exception, match='HTTP error 404: Not Found'): client.query('test_metric') - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_query_connection_error(self, mock_k8s_client_class): + def test_query_connection_error(self): """Test query failure with connection error.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - with patch.object(requests.Session, 'get') as mock_get: mock_get.side_effect = requests.ConnectionError('Connection failed') # Create client and test error - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') - with pytest.raises(MetricsException, match='Query failed: Connection failed'): + with pytest.raises(requests.ConnectionError, match='Connection failed'): client.query('test_metric') - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_query_timeout_error(self, mock_k8s_client_class): + def test_query_timeout_error(self): """Test query failure with timeout error.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - with patch.object(requests.Session, 'get') as mock_get: mock_get.side_effect = requests.Timeout('Request timed out') # Create client and test error - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') - with pytest.raises(MetricsException, match='Query failed: Request timed out'): + with pytest.raises(requests.Timeout, match='Request timed out'): client.query('test_metric') - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_query_json_decode_error(self, mock_k8s_client_class): + def test_query_json_decode_error(self): """Test query failure with JSON decode error.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - with patch.object(requests.Session, 'get') as mock_get: mock_response = MagicMock() mock_response.status_code = 200 @@ -266,19 +184,13 @@ def test_query_json_decode_error(self, mock_k8s_client_class): mock_get.return_value = mock_response # Create client and test error - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') - with pytest.raises(MetricsException, match='Query failed:'): + with pytest.raises(json.JSONDecodeError): client.query('test_metric') - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_get_current_value_success(self, mock_k8s_client_class): + def test_get_current_value_success(self): """Test successful get_current_value execution.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - # Mock response data - using whole number float as vCPU counts are always whole numbers mock_response_data = {'status': 'success', 'data': {'result': [{'metric': {'__name__': 'test_metric'}, 'value': [1640995200, '42']}]}} @@ -289,21 +201,15 @@ def test_get_current_value_success(self, mock_k8s_client_class): mock_get.return_value = mock_response # Create client and execute query - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') value = client.get_current_value('test_metric') # Assertions assert value == 42 mock_get.assert_called_once_with('https://prometheus.example.com:9090/api/v1/query', params={'query': 'test_metric'}, timeout=30) - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_get_current_value_empty_result(self, mock_k8s_client_class): + def test_get_current_value_empty_result(self): """Test get_current_value with empty result.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - # Mock empty response data mock_response_data = {'status': 'success', 'data': {'result': []}} @@ -314,37 +220,25 @@ def test_get_current_value_empty_result(self, mock_k8s_client_class): mock_get.return_value = mock_response # Create client and execute query - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') value = client.get_current_value('test_metric') # Assertions assert value is None - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_get_current_value_query_failure(self, mock_k8s_client_class): + def test_get_current_value_query_failure(self): """Test get_current_value when underlying query fails.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - with patch.object(requests.Session, 'get') as mock_get: mock_get.side_effect = requests.ConnectionError('Connection failed') # Create client and test error propagation - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') - with pytest.raises(MetricsException, match='Query failed: Connection failed'): + with pytest.raises(requests.ConnectionError, match='Connection failed'): client.get_current_value('test_metric') - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_get_current_value_invalid_value_format(self, mock_k8s_client_class): + def test_get_current_value_invalid_value_format(self): """Test get_current_value with invalid value format.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - # Mock response with invalid value format mock_response_data = { 'status': 'success', @@ -358,57 +252,36 @@ def test_get_current_value_invalid_value_format(self, mock_k8s_client_class): mock_get.return_value = mock_response # Create client and test error - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') with pytest.raises(ValueError): client.get_current_value('test_metric') - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_session_configuration(self, mock_k8s_client_class): + def test_session_configuration(self): """Test that session is configured correctly.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client.get_ca_cert_path.return_value = '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' - mock_k8s_client_class.return_value = mock_k8s_client - - with ( - patch('urllib3.disable_warnings') as mock_disable_warnings, - patch('os.path.exists', return_value=True), - ): - # Create client - client = PrometheusClient(url='https://prometheus.example.com:9090') - - # Check session configuration - assert isinstance(client.session, requests.Session) - assert client.session.headers['Authorization'] == 'Bearer test-token' - assert client.session.headers['Content-Type'] == 'application/x-www-form-urlencoded' - assert client.session.verify == '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' - assert client.ca_cert_path == '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' - mock_disable_warnings.assert_not_called() - - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_init_failure_ca_cert_not_found(self, mock_k8s_client_class): - """Test initialization failure when CA certificate file doesn't exist.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client.get_ca_cert_path.return_value = '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' - mock_k8s_client_class.return_value = mock_k8s_client - - with patch('os.path.exists', return_value=False): - # Test that MetricsException is raised when CA cert doesn't exist - with pytest.raises(MetricsException, match='CA_CERT not found at /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt'): - PrometheusClient(url='https://prometheus.example.com:9090') - - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_custom_timeout(self, mock_k8s_client_class): + # Create client + client = PrometheusClient( + url='https://prometheus.example.com:9090', token='test-token', ca_cert_path='/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' + ) + + # Check session configuration + assert isinstance(client.session, requests.Session) + assert client.session.headers['Authorization'] == 'Bearer test-token' + assert client.session.headers['Content-Type'] == 'application/x-www-form-urlencoded' + assert client.session.verify == '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' + + def test_init_with_ca_cert_path(self): + """Test initialization with CA certificate path.""" + # Create client with CA cert path + client = PrometheusClient( + url='https://prometheus.example.com:9090', token='test-token', ca_cert_path='/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' + ) + + # Verify CA cert is configured + assert client.session.verify == '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' + + def test_custom_timeout(self): """Test client with custom timeout.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - # Mock response mock_response_data = {'status': 'success', 'data': {'result': []}} @@ -419,21 +292,15 @@ def test_custom_timeout(self, mock_k8s_client_class): mock_get.return_value = mock_response # Create client with custom timeout - client = PrometheusClient(url='https://prometheus.example.com:9090', timeout=120) + client = PrometheusClient(url='https://prometheus.example.com:9090', timeout=120, token='test-token') client.query('test_metric') # Verify custom timeout is used assert client.timeout == 120 mock_get.assert_called_once_with('https://prometheus.example.com:9090/api/v1/query', params={'query': 'test_metric'}, timeout=120) - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_query_unknown_error_status(self, mock_k8s_client_class): + def test_query_unknown_error_status(self): """Test query with unknown error status from Prometheus.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - # Mock error response without error message mock_response_data = { 'status': 'error' @@ -447,19 +314,13 @@ def test_query_unknown_error_status(self, mock_k8s_client_class): mock_get.return_value = mock_response # Create client and test error - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') - with pytest.raises(MetricsException, match='Prometheus API error: Unknown error'): + with pytest.raises(Exception, match='Prometheus API error: Unknown error'): client.query('test_query') - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_query_missing_data_field(self, mock_k8s_client_class): + def test_query_missing_data_field(self): """Test query with missing data field in response.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - # Mock response without data field mock_response_data = { 'status': 'success' @@ -473,20 +334,14 @@ def test_query_missing_data_field(self, mock_k8s_client_class): mock_get.return_value = mock_response # Create client and execute query - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') result = client.query('test_metric') # Should return empty list when data field is missing assert result == [] - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_query_missing_result_field(self, mock_k8s_client_class): + def test_query_missing_result_field(self): """Test query with missing result field in data.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - # Mock response without result field mock_response_data = { 'status': 'success', @@ -500,20 +355,14 @@ def test_query_missing_result_field(self, mock_k8s_client_class): mock_get.return_value = mock_response # Create client and execute query - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') result = client.query('test_metric') # Should return empty list when result field is missing assert result == [] - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_query_range_success(self, mock_k8s_client_class): + def test_query_range_success(self): """Test successful query_range execution.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - # Mock response data for range query mock_response_data = { 'status': 'success', @@ -527,7 +376,7 @@ def test_query_range_success(self, mock_k8s_client_class): mock_get.return_value = mock_response # Create client and execute range query - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') result = client.query_range('test_metric', 1640995200, 1640995320, '1m') # Assertions @@ -538,14 +387,8 @@ def test_query_range_success(self, mock_k8s_client_class): timeout=30, ) - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_query_range_default_step(self, mock_k8s_client_class): + def test_query_range_default_step(self): """Test query_range with default step parameter.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - mock_response_data = {'status': 'success', 'data': {'result': []}} with patch.object(requests.Session, 'get') as mock_get: @@ -555,7 +398,7 @@ def test_query_range_default_step(self, mock_k8s_client_class): mock_get.return_value = mock_response # Create client and execute range query without step parameter - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') client.query_range('test_metric', 1640995200, 1640995320) # Should use default step of '5m' @@ -565,14 +408,8 @@ def test_query_range_default_step(self, mock_k8s_client_class): timeout=30, ) - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_query_range_prometheus_error(self, mock_k8s_client_class): + def test_query_range_prometheus_error(self): """Test query_range with Prometheus API error.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - # Mock error response mock_response_data = {'status': 'error', 'error': 'invalid query'} @@ -583,28 +420,22 @@ def test_query_range_prometheus_error(self, mock_k8s_client_class): mock_get.return_value = mock_response # Create client and execute range query - client = PrometheusClient(url='https://prometheus.example.com:9090') - result = client.query_range('invalid_query', 1640995200, 1640995320) + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') - # Should return None for error status - assert result is None + # Should raise exception for error status + with pytest.raises(Exception, match='Prometheus API error: invalid query'): + client.query_range('invalid_query', 1640995200, 1640995320) - @patch('metrics_utility.automation_controller_billing.prometheus_client.KubernetesClient') - def test_query_range_http_error(self, mock_k8s_client_class): + def test_query_range_http_error(self): """Test query_range with HTTP error.""" - # Setup mock - mock_k8s_client = MagicMock() - mock_k8s_client.get_current_token.return_value = 'test-token' - mock_k8s_client_class.return_value = mock_k8s_client - with patch.object(requests.Session, 'get') as mock_get: mock_response = MagicMock() mock_response.status_code = 400 - mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError('400 Bad Request') + mock_response.text = 'Bad Request' mock_get.return_value = mock_response # Create client and execute range query - client = PrometheusClient(url='https://prometheus.example.com:9090') + client = PrometheusClient(url='https://prometheus.example.com:9090', token='test-token') - with pytest.raises(MetricsException): + with pytest.raises(Exception, match='HTTP error 400: Bad Request'): client.query_range('test_metric', 1640995200, 1640995320) diff --git a/mock_awx/awx/main/utils/__init__.py b/mock_awx/awx/main/utils/__init__.py index 84d33637..ee1c7171 100644 --- a/mock_awx/awx/main/utils/__init__.py +++ b/mock_awx/awx/main/utils/__init__.py @@ -1,6 +1,3 @@ -from django.utils.dateparse import parse_datetime - - def get_awx_version(): "24.6.123" @@ -10,13 +7,3 @@ def get_awx_http_client_headers(): 'Content-Type': 'application/json', 'User-Agent': '{} {} ({})'.format('AWX', get_awx_version(), 'UNLICENSED'), } - - -def datetime_hook(d): - new_d = {} - for key, value in d.items(): - try: - new_d[key] = parse_datetime(value) - except TypeError: - new_d[key] = value - return new_d diff --git a/mock_awx/settings/__init__.py b/mock_awx/settings/__init__.py index f95b183b..c385e789 100644 --- a/mock_awx/settings/__init__.py +++ b/mock_awx/settings/__init__.py @@ -19,13 +19,6 @@ USE_L10N = True USE_TZ = True -# Collected -AUTOMATION_ANALYTICS_LAST_ENTRIES = '' +# collected but also used by Package INSTALL_UUID = '00000000-0000-0000-0000-000000000000' -LOG_AGGREGATOR_ENABLED = False -LOG_AGGREGATOR_LOGGERS = ['awx', 'activity_stream', 'job_events', 'system_tracking', 'broadcast_websocket', 'job_lifecycle'] -LOG_AGGREGATOR_TYPE = None -PENDO_TRACKING_STATE = 'off' -SUBSCRIPTION_USAGE_MODEL = '' SYSTEM_UUID = '00000000-0000-0000-0000-000000000000' -TOWER_URL_BASE = 'https://platformhost'