From f3f78ef4a6536c3b70d0be52d8c30e76b63d15aa Mon Sep 17 00:00:00 2001 From: Alan King Date: Wed, 13 Nov 2024 16:03:14 -0500 Subject: [PATCH 1/4] [ 233] Add Ruff linter GitHub Action Based on recommendations in Ruff's documentation: https://docs.astral.sh/ruff/integrations/#github-actions --- .github/workflows/ruff.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .github/workflows/ruff.yml diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml new file mode 100644 index 0000000..004052d --- /dev/null +++ b/.github/workflows/ruff.yml @@ -0,0 +1,8 @@ +name: ruff +on: [push, pull_request] +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/ruff-action@v1 From 482ad5558a50052fe133c00f2b2e8c994a3f066b Mon Sep 17 00:00:00 2001 From: Alan King Date: Thu, 14 Nov 2024 10:36:58 -0500 Subject: [PATCH 2/4] [ 233] Add Black formatter workflow Based on recommendations in Black's documentation: https://black.readthedocs.io/en/stable/integrations/github_actions.html --- .github/workflows/black.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .github/workflows/black.yml diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml new file mode 100644 index 0000000..12f8925 --- /dev/null +++ b/.github/workflows/black.yml @@ -0,0 +1,8 @@ +name: black +on: [push, pull_request] +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: psf/black@stable From f9f13ed5648c54e736077e318a9d78f32b5ab4b2 Mon Sep 17 00:00:00 2001 From: Alan King Date: Wed, 13 Nov 2024 16:34:26 -0500 Subject: [PATCH 3/4] [ 233] Address issues reported by Ruff --- cli.py | 3 - configure_ssl.py | 9 -- execute.py | 63 -------- federate.py | 5 - install.py | 56 ------- irods_config.py | 38 ----- irods_testing_environment/archive.py | 4 +- irods_testing_environment/context.py | 3 +- irods_testing_environment/database_setup.py | 23 +-- irods_testing_environment/execute.py | 6 - irods_testing_environment/federate.py | 13 +- irods_testing_environment/install/install.py | 31 ++-- irods_testing_environment/irods_config.py | 45 +++--- irods_testing_environment/irods_setup.py | 16 +- irods_testing_environment/logs.py | 1 - irods_testing_environment/negotiation_key.py | 15 +- irods_testing_environment/odbc_setup.py | 5 +- irods_testing_environment/services.py | 1 - irods_testing_environment/ssl_setup.py | 10 +- irods_testing_environment/test_manager.py | 12 +- irods_testing_environment/test_runner.py | 10 +- irods_testing_environment/test_utils.py | 8 +- negotiation_key.py | 153 ------------------- run_plugin_tests.py | 1 - run_topology_tests.py | 3 - run_unit_tests.py | 2 - setup.py | 110 ------------- 27 files changed, 93 insertions(+), 553 deletions(-) delete mode 100644 execute.py delete mode 100644 install.py delete mode 100644 irods_config.py delete mode 100644 negotiation_key.py delete mode 100644 setup.py diff --git a/cli.py b/cli.py index 10ccc60..035e4fd 100644 --- a/cli.py +++ b/cli.py @@ -1,9 +1,6 @@ # grown-up modules import textwrap -# local modules -from irods_testing_environment import context - def add_compose_args(parser): '''Add argparse options related to Docker Compose project. diff --git a/configure_ssl.py b/configure_ssl.py index 9837d12..dc196fc 100644 --- a/configure_ssl.py +++ b/configure_ssl.py @@ -1,19 +1,10 @@ # grown-up modules -import json import logging import os -from cryptography import x509 -from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import dh, rsa -from cryptography.x509.oid import NameOID - # local modules from irods_testing_environment.ssl_setup import configure_ssl_in_zone from irods_testing_environment import logs -from irods_testing_environment import context -from irods_testing_environment import execute -from irods_testing_environment import json_utils if __name__ == "__main__": import argparse diff --git a/execute.py b/execute.py deleted file mode 100644 index e456ccd..0000000 --- a/execute.py +++ /dev/null @@ -1,63 +0,0 @@ -# grown-up modules -import compose.cli.command -import docker -import logging -import os - -# local modules -import context - -if __name__ == "__main__": - import argparse - import logs - - parser = argparse.ArgumentParser(description='Run commands on a running container as iRODS service account.') - parser.add_argument('project_path', metavar='PROJECT_PATH', type=str, - help='Path to the docker-compose project on which packages will be installed.') - parser.add_argument('commands', metavar='COMMANDS', nargs='+', - help='Space-delimited list of commands to be run') - parser.add_argument('--run-on-container', '-t', metavar='TARGET_CONTAINER', dest='run_on', type=str, - help='The name of the container on which the command will run. By default, runs on all containers in project.') - parser.add_argument('--project-name', metavar='PROJECT_NAME', type=str, dest='project_name', - help='Name of the docker-compose project on which to install packages.') - parser.add_argument('--verbose', '-v', dest='verbosity', action='count', default=1, - help='Increase the level of output to stdout. CRITICAL and ERROR messages will always be printed.') - parser.add_argument('--user', '-u', metavar='USER', dest='user', default='root', - help='Name of the user to be when executing the commands.') - parser.add_argument('--workdir', '-w', metavar='WORKING_DIRECTORY', dest='workdir', default='/', - help='Working directory for execution of commands.') - - args = parser.parse_args() - - logs.configure(args.verbosity) - - ec = 0 - containers = list() - - docker_client = docker.from_env() - - try: - p = compose.cli.command.get_project(os.path.abspath(args.project_path), project_name=args.project_name) - - # Get the container on which the command is to be executed - containers = list() - if args.run_on: - containers.append(docker_client.containers.get(args.run_on)) - else: - containers = p.containers() - - # Serially execute the list of commands provided in the input - for c in containers: - if context.is_catalog_database_container(c): continue - - target_container = docker_client.containers.get(c.name) - for command in args.commands: - # TODO: on --continue, save only failure ec's/commands - ec = execute_command(target_container, command, user=args.user, workdir=args.workdir, stream_output=True) - - except Exception as e: - logging.critical(e) - - raise - - exit(ec) diff --git a/federate.py b/federate.py index 110a90e..5a9ce24 100644 --- a/federate.py +++ b/federate.py @@ -1,17 +1,12 @@ # grown-up modules import compose.cli.command import docker -import json import logging import os # local modules from irods_testing_environment import context -from irods_testing_environment import database_setup -from irods_testing_environment import execute from irods_testing_environment import irods_setup -from irods_testing_environment import irods_config -from irods_testing_environment import json_utils from irods_testing_environment import federate from irods_testing_environment import ssl_setup from irods_testing_environment.install import install diff --git a/install.py b/install.py deleted file mode 100644 index eb2c381..0000000 --- a/install.py +++ /dev/null @@ -1,56 +0,0 @@ -# grown-up modules -import compose.cli.command -import docker -import logging -import os - -# local modules -from irods_testing_environment import archive -from irods_testing_environment import context -from irods_testing_environment import execute -from irods_testing_environment.install import install -from irods_testing_environment import services - -if __name__ == "__main__": - import argparse - import textwrap - - import cli - from irods_testing_environment import logs - - parser = argparse.ArgumentParser(description='Install iRODS packages to a docker-compose project.') - - cli.add_common_args(parser) - cli.add_compose_args(parser) - cli.add_irods_package_args(parser) - - args = parser.parse_args() - - if args.package_directory and args.package_version: - print('package directory and package version are mutually exclusive') - exit(1) - - logs.configure(args.verbosity) - - project_directory = os.path.abspath(args.project_directory or os.getcwd()) - - ctx = context.context(docker.from_env(), - compose.cli.command.get_project( - project_dir=project_directory, - project_name=args.project_name)) - - logging.debug('provided project name [{0}], docker-compose project name [{1}]' - .format(args.project_name, ctx.compose_project.name)) - - if len(ctx.compose_project.containers()) is 0: - logging.critical( - 'no containers found for project [directory=[{0}], name=[{1}]]'.format( - os.path.abspath(project_directory), ctx.compose_project.name)) - - exit(1) - - exit(install.make_installer(ctx.platform_name()).install_irods_packages(ctx, - externals_directory=args.irods_externals_package_directory, - package_directory=args.package_directory, - package_version=args.package_version) - ) diff --git a/irods_config.py b/irods_config.py deleted file mode 100644 index 7f60d2f..0000000 --- a/irods_config.py +++ /dev/null @@ -1,38 +0,0 @@ -# grown-up modules -import compose.cli.command -import docker -import json -import logging -import os - -# local modules -import context -import execute -import json_utils - -if __name__ == "__main__": - import argparse - import logs - - import cli - - parser = argparse.ArgumentParser(description='Configure a running iRODS Zone for running tests.') - - cli.add_common_args(parser) - cli.add_compose_args(parser) - - args = parser.parse_args() - - docker_client = docker.from_env() - - compose_project = compose.cli.command.get_project(os.path.abspath(args.project_directory), - project_name=args.project_name) - - logs.configure(args.verbosity) - - try: - configure_irods_testing(docker_client, compose_project) - - except Exception as e: - logging.critical(e) - exit(1) diff --git a/irods_testing_environment/archive.py b/irods_testing_environment/archive.py index 5eaa687..2009b69 100644 --- a/irods_testing_environment/archive.py +++ b/irods_testing_environment/archive.py @@ -177,14 +177,14 @@ def copy_files_in_container(container, sources_and_destinations): sources_and_destinations -- a list of tuples of source paths and destination paths """ tarfile = create_archive([s for s, d in sources_and_destinations], 'ssl') - archive_dirname = copy_archive_to_container(container, tarfile) + _ = copy_archive_to_container(container, tarfile) for s, d in sources_and_destinations: logging.debug( 'copying source [{}] in container to destination in container [{}] [{}]'.format( s, d, container.name)) - if execute.execute_command(container, 'cp {} {}'.format(s, d)) is not 0: + if execute.execute_command(container, 'cp {} {}'.format(s, d)) != 0: raise RuntimeError('failed to copy file src [{}] dest [{}] [{}]' .format(s, d, container.name)) diff --git a/irods_testing_environment/context.py b/irods_testing_environment/context.py index 1f0b383..25e19b5 100644 --- a/irods_testing_environment/context.py +++ b/irods_testing_environment/context.py @@ -320,7 +320,8 @@ def is_irods_server_in_local_zone(container, local_zone): container -- the container to inspect (if not running iRODS, returns False) local_zone -- zone information against which to compare for the container running iRODS """ - if is_catalog_database_container(container): return False + if is_catalog_database_container(container): + return False if is_irods_catalog_provider_container(container): return service_instance(container.name) is local_zone.provider_service_instance diff --git a/irods_testing_environment/database_setup.py b/irods_testing_environment/database_setup.py index ae75d4c..0dd2cb2 100644 --- a/irods_testing_environment/database_setup.py +++ b/irods_testing_environment/database_setup.py @@ -1,5 +1,4 @@ # grown-up modules -import docker import logging import time @@ -145,7 +144,8 @@ def create_database(self, name, force_recreate=False): name -- name of the database to create force_recreate -- if True, drops any database by the specified name before creating """ - if force_recreate: self.drop_database(name) + if force_recreate: + self.drop_database(name) return 0 if self.database_exists(name) \ else self.execute_psql_command('create database \\\"{}\\\";'.format(name)) @@ -158,7 +158,8 @@ def create_user(self, username, password, force_recreate=False): password -- password for the new user force_recreate -- if True, drops any user by the specified username before creating """ - if force_recreate: self.drop_user(username) + if force_recreate: + self.drop_user(username) return 0 if self.user_exists(username) \ else self.execute_psql_command('create user {0} with password \'{1}\';' @@ -173,7 +174,7 @@ def grant_privileges(self, database, username): """ ec = self.execute_psql_command('grant all privileges on database \\\"{0}\\\" to {1};' .format(database, username)) - if ec is not 0: + if ec != 0: return ec return self.execute_psql_command('alter database \\\"{0}\\\" owner to {1};' .format(database, username)) @@ -275,7 +276,8 @@ def create_database(self, name, force_recreate=False): name -- name of the database to create force_recreate -- if True, drops any database by the specified name before creating """ - if force_recreate: self.drop_database(name) + if force_recreate: + self.drop_database(name) return 0 if self.database_exists(name) \ else self.execute_mysql_command('CREATE DATABASE {};'.format(name)) @@ -290,7 +292,8 @@ def create_user(self, username, password, force_recreate=False): """ user = '\'{}\'@\'{}\''.format(username, self.host) - if force_recreate: self.drop_user(user) + if force_recreate: + self.drop_user(user) return 0 if self.user_exists(username, password) \ else self.execute_mysql_command( @@ -351,7 +354,7 @@ def execute_mariadb_command(self, mariadb_cmd, user='root', password='testpasswo Arguments: mariadb_cmd -- the command to be passed to mariadb via --execute """ - return self.execute_mysql_command(mariadb_cmd, root, password) + return self.execute_mysql_command(mariadb_cmd, user, password) def make_strategy(database_image, container=None, database_port=None, root_password=None): @@ -395,15 +398,15 @@ def setup_catalog(ctx, strat = make_strategy(ctx.database(), db_container, database_port, root_password) ec = strat.create_database(database_name, force_recreate) - if ec is not 0: + if ec != 0: raise RuntimeError('failed to create database [{}]'.format(database_name)) ec = strat.create_user(database_user, database_password, force_recreate) - if ec is not 0: + if ec != 0: raise RuntimeError('failed to create user [{}]'.format(database_user)) ec = strat.grant_privileges(database_name, database_user) - if ec is not 0: + if ec != 0: raise RuntimeError('failed to grant privileges to user [{0}] on database [{1}]'.format(database_user, database_name)) strat.list_databases() diff --git a/irods_testing_environment/execute.py b/irods_testing_environment/execute.py index 7b42b0e..793b6d1 100644 --- a/irods_testing_environment/execute.py +++ b/irods_testing_environment/execute.py @@ -1,11 +1,5 @@ # grown-up modules -import compose.cli.command -import docker import logging -import os - -# local modules -from . import context def execute_command(container, command, user='', workdir=None, stream_output=None): """Execute `command` in `container` as `user` in `workdir`. diff --git a/irods_testing_environment/federate.py b/irods_testing_environment/federate.py index f3affe2..16f73d4 100644 --- a/irods_testing_environment/federate.py +++ b/irods_testing_environment/federate.py @@ -39,9 +39,11 @@ def federate_zones(ctx, zone_info_list, local_zone, include_consumers=True): """ # Every iRODS server in the Zone must be federated for c in ctx.compose_project.containers(): - if not context.is_irods_server_in_local_zone(c, local_zone): continue + if not context.is_irods_server_in_local_zone(c, local_zone): + continue - if not include_consumers and context.is_irods_catalog_consumer_container(c): continue + if not include_consumers and context.is_irods_catalog_consumer_container(c): + continue logging.debug('container [{}] zone [{}] provider instance [{}]' .format(c.name, @@ -53,7 +55,8 @@ def federate_zones(ctx, zone_info_list, local_zone, include_consumers=True): server_config = json_utils.get_json_from_file(container, context.server_config()) for remote_zone in zone_info_list: - if remote_zone.zone_name == local_zone.zone_name: continue + if remote_zone.zone_name == local_zone.zone_name: + continue logging.warning('federating remote zone [{}] with local zone [{}] on [{}]' .format(remote_zone.zone_name, local_zone.zone_name, container.name)) @@ -68,7 +71,7 @@ def federate_zones(ctx, zone_info_list, local_zone, include_consumers=True): remote_zone.provider_hostname(ctx), remote_zone.zone_port) - if execute.execute_command(container, make_remote_zone, user='irods') is not 0: + if execute.execute_command(container, make_remote_zone, user='irods') != 0: raise RuntimeError('failed to create remote zone [{}]' .format(container.name)) @@ -105,5 +108,5 @@ def form_federation_clique(ctx, zone_info_list, include_consumers=True): logging.error(e) rc = 1 - if rc is not 0: + if rc != 0: raise RuntimeError('failed to federate one or more iRODS Zones, ec=[{}]'.format(rc)) diff --git a/irods_testing_environment/install/install.py b/irods_testing_environment/install/install.py index f49fdb0..9de7dc2 100644 --- a/irods_testing_environment/install/install.py +++ b/irods_testing_environment/install/install.py @@ -52,7 +52,7 @@ def get_list_of_package_paths(self, package_directory, package_name_list=None): glob_list = glob.glob(glob_str) - if len(glob_list) is 0: + if len(glob_list) == 0: raise RuntimeError('no packages found [{}]'.format(glob_str)) # TODO: allow specifying a suffix or something instead of taking everything @@ -78,7 +78,8 @@ def install_packages_on_container_from_tarfile(self, container = ctx.docker_client.containers.get(container_name) # Only the iRODS containers need to have packages installed - if context.is_catalog_database_container(container): return 0 + if context.is_catalog_database_container(container): + return 0 archive.copy_archive_to_container(container, tarfile_path) @@ -92,12 +93,12 @@ def install_packages_on_container_from_tarfile(self, logging.warning('executing cmd [{0}] on container [{1}]'.format(cmd, container.name)) ec = execute.execute_command(container, self.update_command()) - if ec is not 0: + if ec != 0: logging.error('failed to update local repositories [{}]'.format(container.name)) return ec ec = execute.execute_command(container, cmd) - if ec is not 0: + if ec != 0: logging.error( 'failed to install packages on container [ec=[{0}], container=[{1}]'.format(ec, container.name)) return ec @@ -128,7 +129,7 @@ def install_packages(self, ctx, package_directory, containers, package_name_list container = futures_to_containers[f] try: ec = f.result() - if ec is not 0: + if ec != 0: logging.error('error while installing packages on container [{}]' .format(container.name)) rc = ec @@ -156,12 +157,12 @@ def install_packages_(ctx, docker_compose_container, packages_list): logging.warning('executing cmd [{0}] on container [{1}]'.format(cmd, container.name)) ec = execute.execute_command(container, self.update_command()) - if ec is not 0: + if ec != 0: logging.error('failed to update local repositories [{}]'.format(container.name)) return ec ec = execute.execute_command(container, cmd) - if ec is not 0: + if ec != 0: logging.error( 'failed to install packages on container [ec=[{0}], container=[{1}]'.format(ec, container.name)) @@ -189,7 +190,7 @@ def install_packages_(ctx, docker_compose_container, packages_list): container = futures_to_containers[f] try: ec = f.result() - if ec is not 0: + if ec != 0: logging.error('error while installing packages on container [{}]'.format(container.name)) rc = ec @@ -232,7 +233,7 @@ def install_irods_packages(self, os.path.abspath(externals_directory), ctx.irods_containers(), context.irods_externals_package_names()) - if ec is not 0: + if ec != 0: raise RuntimeError('failed to install externals') if package_directory: @@ -243,7 +244,7 @@ def install_irods_packages(self, os.path.abspath(package_directory), ctx.irods_containers(), context.irods_package_names(ctx.database_name())) - if ec is not 0: + if ec != 0: raise RuntimeError('failed to install iRODS packages') else: @@ -252,17 +253,11 @@ def install_irods_packages(self, .format(package_version)) ec = self.install_official_irods_packages(ctx, package_version, ctx.irods_containers()) - if ec is not 0: + if ec != 0: raise RuntimeError('failed to install iRODS packages') def make_installer(platform_name): - from . import almalinux_installer - from . import centos_installer - from . import debian_installer - from . import rockylinux_installer - from . import ubuntu_installer - name = '_'.join([platform_name, 'installer']) return eval('.'.join([name, name]))() @@ -288,6 +283,6 @@ def install_pip_package_from_repo(container, ec = execute.execute_command(container, ' '.join( [container_info.python(container), '-m', 'pip', 'install', repo_path])) - if ec is not 0: + if ec != 0: raise RuntimeError('Failed to install pip package [{}] [{}]' .format(repo_path, container.name)) diff --git a/irods_testing_environment/irods_config.py b/irods_testing_environment/irods_config.py index f3cbc3c..0980d3e 100644 --- a/irods_testing_environment/irods_config.py +++ b/irods_testing_environment/irods_config.py @@ -1,6 +1,4 @@ # grown-up modules -import compose.cli.command -import docker import json import logging import os @@ -119,7 +117,7 @@ def create_test_users(docker_client, docker_compose_container, usernames_and_pas for username, password in usernames_and_passwords: create_user = f'useradd {username}' - if execute.execute_command(container, create_user) is not 0: + if execute.execute_command(container, create_user) != 0: raise RuntimeError(f'[{container.name}] failed to create user [{username}]') if password is None or password == '': @@ -127,7 +125,7 @@ def create_test_users(docker_client, docker_compose_container, usernames_and_pas set_password = f'bash -c "echo \'{username}:{password}\' | chpasswd"' - if execute.execute_command(container, set_password) is not 0: + if execute.execute_command(container, set_password) != 0: raise RuntimeError(f'[{container.name}] failed to create hosts_config file') return 0 @@ -157,7 +155,7 @@ def create_test_users(docker_client, docker_compose_container, usernames_and_pas container = futures_to_containers[f] try: ec = f.result() - if ec is not 0: + if ec != 0: logging.error(f'[{container.name}] error while creating test user accounts') rc = ec else: @@ -168,7 +166,7 @@ def create_test_users(docker_client, docker_compose_container, usernames_and_pas logging.error(e) rc = 1 - if rc is not 0: + if rc != 0: raise RuntimeError('failed to create test user accounts on some service') @@ -202,7 +200,8 @@ def set_hostnames(docker_client, docker_compose_container, hosts_file): } for o in containers: - if o.name == container.name: continue + if o.name == container.name: + continue other = docker_client.containers.get(o.name) @@ -228,7 +227,7 @@ def set_hostnames(docker_client, docker_compose_container, hosts_file): create_hosts_config = 'bash -c \'echo "{}" > {}\''.format( json.dumps(hosts).replace('"', '\\"'), hosts_file) - if execute.execute_command(container, create_hosts_config) is not 0: + if execute.execute_command(container, create_hosts_config) != 0: raise RuntimeError('failed to create hosts_config file [{}]'.format(container.name)) return 0 @@ -251,7 +250,7 @@ def set_hostnames(docker_client, docker_compose_container, hosts_file): container = futures_to_containers[f] try: ec = f.result() - if ec is not 0: + if ec != 0: logging.error('error while configuring hosts_configs.json on container [{}]' .format(container.name)) rc = ec @@ -265,7 +264,7 @@ def set_hostnames(docker_client, docker_compose_container, hosts_file): logging.error(e) rc = 1 - if rc is not 0: + if rc != 0: raise RuntimeError('failed to configure hosts_config.json on some service') @@ -283,28 +282,28 @@ def modify_script(docker_client, docker_compose_container, script): make_script_executable = 'chmod 544 {}'.format(script) on_container = docker_client.containers.get(docker_compose_container.name) - if execute.execute_command(on_container, chown_msiexec) is not 0: + if execute.execute_command(on_container, chown_msiexec) != 0: raise RuntimeError('failed to change ownership to msiExecCmd_bin [{}]' .format(on_container.name)) if execute.execute_command(on_container, copy_from_template, user='irods', - workdir=context.irods_home()) is not 0: + workdir=context.irods_home()) != 0: raise RuntimeError('failed to copy univMSSInterface.sh template file [{}]' .format(on_container.name)) if execute.execute_command(on_container, remove_template_from_commands, user='irods', - workdir=context.irods_home()) is not 0: + workdir=context.irods_home()) != 0: raise RuntimeError('failed to modify univMSSInterface.sh template file [{}]' .format(on_container.name)) if execute.execute_command(on_container, make_script_executable, user='irods', - workdir=context.irods_home()) is not 0: + workdir=context.irods_home()) != 0: raise RuntimeError('failed to change permissions on univMSSInterface.sh [{}]' .format(on_container.name)) @@ -333,7 +332,7 @@ def modify_script(docker_client, docker_compose_container, script): container = futures_to_containers[f] try: ec = f.result() - if ec is not 0: + if ec != 0: logging.error('error while configuring univMSS script on container [{}]' .format(container.name)) rc = ec @@ -345,7 +344,7 @@ def modify_script(docker_client, docker_compose_container, script): logging.error(e) rc = 1 - if rc is not 0: + if rc != 0: raise RuntimeError('failed to configure univMSS script on some service') @@ -387,7 +386,7 @@ def configure_irods_federation_testing(ctx, remote_zone, zone_where_tests_will_r username = '#'.join(['zonehopper', zone_where_tests_will_run.zone_name]) mkuser = 'iadmin mkuser {} rodsuser'.format(username) logging.info('creating user [{}] in container [{}]'.format(username, container.name)) - if execute.execute_command(container, mkuser, user='irods') is not 0: + if execute.execute_command(container, mkuser, user='irods') != 0: raise RuntimeError('failed to create remote user [{}] [{}]' .format(username, container.name)) @@ -395,14 +394,14 @@ def configure_irods_federation_testing(ctx, remote_zone, zone_where_tests_will_r username = '#'.join(['zonehopper', remote_zone.zone_name]) mkuser = 'iadmin mkuser {} rodsuser'.format(username) logging.info('creating user [{}] in container [{}]'.format(username, container.name)) - if execute.execute_command(container, mkuser, user='irods') is not 0: + if execute.execute_command(container, mkuser, user='irods') != 0: raise RuntimeError('failed to create remote user [{}] [{}]' .format(username, container.name)) # set password of zonehopper# user moduser = 'iadmin moduser {} password 53CR37'.format(username) logging.info('setting password of user [{}] in container [{}]'.format(username, container.name)) - if execute.execute_command(container, moduser, user='irods') is not 0: + if execute.execute_command(container, moduser, user='irods') != 0: raise RuntimeError('failed to set password for remote user [{}] [{}]' .format(username, container.name)) @@ -412,7 +411,7 @@ def configure_irods_federation_testing(ctx, remote_zone, zone_where_tests_will_r ptname = 'federation_remote_passthrough' make_pt = 'iadmin mkresc {} passthru'.format(ptname) logging.info('creating passthrough resource [{}] [{}]'.format(make_pt, container.name)) - if execute.execute_command(container, make_pt, user='irods') is not 0: + if execute.execute_command(container, make_pt, user='irods') != 0: raise RuntimeError('failed to create passthrough resource [{}] [{}]' .format(ptname, container.name)) @@ -422,14 +421,14 @@ def configure_irods_federation_testing(ctx, remote_zone, zone_where_tests_will_r ufsname, context.container_hostname(container), os.path.join('/tmp', ufsname)) logging.info('creating unixfilesystem resource [{}] [{}]'.format(make_ufs, container.name)) - if execute.execute_command(container, make_ufs, user='irods') is not 0: + if execute.execute_command(container, make_ufs, user='irods') != 0: raise RuntimeError('failed to create unixfilesystem resource [{}] [{}]' .format(ufsname, container.name)) # make the hierarchy make_hier = 'iadmin addchildtoresc {} {}'.format(ptname, ufsname) logging.info('creating hierarchy [{}] [{}]'.format(make_hier, container.name)) - if execute.execute_command(container, make_hier, user='irods') is not 0: + if execute.execute_command(container, make_hier, user='irods') != 0: raise RuntimeError('failed to create hierarchy [{};{}] [{}]' .format(ptname, ufsname, container.name)) @@ -437,7 +436,7 @@ def configure_irods_federation_testing(ctx, remote_zone, zone_where_tests_will_r bug_3466_query = 'select alias, sqlStr from R_SPECIFIC_QUERY' asq = 'iadmin asq \'{}\' {}'.format(bug_3466_query, 'bug_3466_query') logging.info('creating specific query[{}] [{}]'.format(asq, container.name)) - if execute.execute_command(container, asq, user='irods') is not 0: + if execute.execute_command(container, asq, user='irods') != 0: raise RuntimeError('failed to create specific query [{}] [{}]' .format(bug_3466_query, container.name)) diff --git a/irods_testing_environment/irods_setup.py b/irods_testing_environment/irods_setup.py index 4337912..47c956d 100644 --- a/irods_testing_environment/irods_setup.py +++ b/irods_testing_environment/irods_setup.py @@ -1,6 +1,4 @@ # grown-up modules -import compose -import docker import logging import os @@ -578,7 +576,7 @@ def setup_irods_catalog_consumers(ctx, service_names=[context.irods_catalog_consumer_service()]) if consumer_service_instances: - if len(consumer_service_instances) is 0: + if len(consumer_service_instances) == 0: logging.warning('empty list of iRODS catalog service consumers to set up') return @@ -618,7 +616,7 @@ def setup_irods_catalog_consumers(ctx, logging.error(e) rc = 1 - if rc is not 0: + if rc != 0: raise RuntimeError('failed to set up one or more catalog service consumers, ec=[{}]' .format(rc)) @@ -698,7 +696,7 @@ def setup_irods_zones(ctx, logging.error(e) rc = 1 - if rc is not 0: + if rc != 0: raise RuntimeError('failed to set up one or more iRODS Zones, ec=[{}]'.format(rc)) @@ -712,13 +710,13 @@ def make_negotiation_key(prefix=''): Arguments: prefix -- optional prefix to use to make the key unique """ - negotation_key_size_in_bytes = 32 + negotiation_key_size_in_bytes = 32 - if len(prefix) > negotation_key_size_in_bytes: + if len(prefix) > negotiation_key_size_in_bytes: return prefix[:negotiation_key_size_in_bytes] - filler = '_' * negotation_key_size_in_bytes - return prefix + filler[:negotation_key_size_in_bytes - len(prefix)] + filler = '_' * negotiation_key_size_in_bytes + return prefix + filler[:negotiation_key_size_in_bytes - len(prefix)] def make_zone_key(zone_name): diff --git a/irods_testing_environment/logs.py b/irods_testing_environment/logs.py index 031df2a..f4f04dd 100644 --- a/irods_testing_environment/logs.py +++ b/irods_testing_environment/logs.py @@ -1,5 +1,4 @@ # grown-up modules -import docker import logging import os import sys diff --git a/irods_testing_environment/negotiation_key.py b/irods_testing_environment/negotiation_key.py index 4422687..f9a4186 100644 --- a/irods_testing_environment/negotiation_key.py +++ b/irods_testing_environment/negotiation_key.py @@ -1,8 +1,5 @@ # grown-up modules -import compose.cli.command -import docker import logging -import os # local modules from . import context @@ -11,13 +8,13 @@ def backup_file(container, file_path): backup_file_path = file_path + '.orig' - if execute.execute_command(container, 'cp {} {}'.format(file_path, backup_file_path)) is not 0: + if execute.execute_command(container, 'cp {} {}'.format(file_path, backup_file_path)) != 0: raise RuntimeError('failed to backup [{}] [{}]'.format(file_path, container.name)) def restore_file(container, file_path): backup_file_path = file_path + '.orig' - if execute.execute_command(container, 'cp {} {}'.format(backup_file_path, file_path)) is not 0: + if execute.execute_command(container, 'cp {} {}'.format(backup_file_path, file_path)) != 0: raise RuntimeError('failed to restore [{}] [{}]'.format(file_path, container.name)) @@ -39,7 +36,7 @@ def configure_ssl_in_server(container, server_ssl_negotiation): context.core_re() + '.orig', context.core_re()) - if execute.execute_command(container, add_acPreConnect) is not 0: + if execute.execute_command(container, add_acPreConnect) != 0: raise RuntimeError('failed to update core.re [{}]'.format(container.name)) @@ -48,13 +45,13 @@ def show_configurations(container, stream_output=False): show_server_config = 'bash -c "cat {} | jq \'.\'"'.format(context.server_config()) show_irods_env = 'bash -c "cat {} | jq \'.\'"'.format(context.service_account_irods_env()) - if execute.execute_command(container, show_core_re, stream_output=stream_output) is not 0: + if execute.execute_command(container, show_core_re, stream_output=stream_output) != 0: raise RuntimeError('failed to cat core.re [{}]'.format(container.name)) - if execute.execute_command(container, show_server_config, stream_output=stream_output) is not 0: + if execute.execute_command(container, show_server_config, stream_output=stream_output) != 0: raise RuntimeError('failed to cat server_config [{}]'.format(container.name)) - if execute.execute_command(container, show_irods_env, stream_output=stream_output) is not 0: + if execute.execute_command(container, show_irods_env, stream_output=stream_output) != 0: raise RuntimeError('failed to cat irods_environment [{}]'.format(container.name)) diff --git a/irods_testing_environment/odbc_setup.py b/irods_testing_environment/odbc_setup.py index 3b8055d..0d98bc5 100644 --- a/irods_testing_environment/odbc_setup.py +++ b/irods_testing_environment/odbc_setup.py @@ -205,7 +205,7 @@ def make_mysql_odbcinst_ini(csp_container, container_odbc_driver_dir): cmd = 'bash -c \'echo "{0}" > {1}\''.format(odbcinst_ini_contents, odbcinst_ini_path) ec = execute.execute_command(csp_container, cmd) - if ec is not 0: + if ec != 0: raise RuntimeError('failed to populate odbcinst.ini [ec=[{0}], container=[{1}]]' .format(ec, csp_container)) @@ -242,7 +242,6 @@ def download_mysql_odbc_driver(url, destination=None, always_download=False): destination -- destination path on local filesystem for the file to be downloaded """ import shutil - import tempfile import urllib.request if not destination: @@ -448,7 +447,7 @@ def make_mariadb_odbcinst_ini(csp_container, container_odbc_lib_dir): cmd = 'bash -c \'echo "{0}" > {1}\''.format(odbcinst_ini_contents, odbcinst_ini_path) ec = execute.execute_command(csp_container, cmd) - if ec is not 0: + if ec != 0: raise RuntimeError('failed to populate odbcinst.ini [ec=[{0}], container=[{1}]]' .format(ec, csp_container)) diff --git a/irods_testing_environment/services.py b/irods_testing_environment/services.py index bda9a65..5cd4ac9 100644 --- a/irods_testing_environment/services.py +++ b/irods_testing_environment/services.py @@ -1,5 +1,4 @@ # grown-up modules -import logging import os # local modules diff --git a/irods_testing_environment/ssl_setup.py b/irods_testing_environment/ssl_setup.py index fb332c1..61cd26c 100644 --- a/irods_testing_environment/ssl_setup.py +++ b/irods_testing_environment/ssl_setup.py @@ -105,7 +105,7 @@ def configure_ssl_on_server(container, cert_file = os.path.join(context.irods_config(), 'server.crt') irodsctl = os.path.join(context.irods_home(), 'irodsctl') - if execute.execute_command(container, '{} stop'.format(irodsctl), user='irods') is not 0: + if execute.execute_command(container, '{} stop'.format(irodsctl), user='irods') != 0: raise RuntimeError( 'failed to stop iRODS server before SSL configuration [{}]' .format(container.name)) @@ -142,7 +142,7 @@ def configure_ssl_on_server(container, negotiation_key.configure_ssl_in_server(container, 'CS_NEG_REQUIRE') # start the server again - if execute.execute_command(container, '{} start'.format(irodsctl), user='irods') is not 0: + if execute.execute_command(container, '{} start'.format(irodsctl), user='irods') != 0: raise RuntimeError( 'failed to start iRODS server after SSL configuration [{}]' .format(container.name)) @@ -152,12 +152,10 @@ def configure_ssl_on_server(container, def configure_ssl_in_zone(docker_client, compose_project): import concurrent.futures - import tempfile # Each irods_environment.json file is describing the cert this client will use and why # they think it is good. The testing environment is using a self-signed certificate, so # the certificate, key, and dhparams should be generated ONCE and copied to each server. - #ssl_files_dir = tempfile.mkdtemp() key, key_file = generate_ssl_certificate_key() cert_file = generate_ssl_self_signed_certificate(key) dhparams_file = generate_ssl_dh_params() @@ -192,7 +190,7 @@ def configure_ssl_in_zone(docker_client, compose_project): logging.error(e) rc = 1 - if rc is not 0: + if rc != 0: raise RuntimeError('failed to configure SSL on some service') cscs = compose_project.containers(service_names=[ @@ -218,7 +216,7 @@ def configure_ssl_in_zone(docker_client, compose_project): logging.error(e) rc = 1 - if rc is not 0: + if rc != 0: raise RuntimeError('failed to configure SSL on some service') finally: diff --git a/irods_testing_environment/test_manager.py b/irods_testing_environment/test_manager.py index ff0ca49..2679137 100644 --- a/irods_testing_environment/test_manager.py +++ b/irods_testing_environment/test_manager.py @@ -3,9 +3,6 @@ import queue import time -# local modules -from . import test_runner - class test_manager: """A class that manages a list of tests and `test_runners` for executing tests.""" @@ -62,7 +59,7 @@ def result_string(self): r = r + tr.result_string() tests_were_skipped = tests_were_skipped if tests_were_skipped else len(tr.skipped_tests()) > 0 - if self.return_code() is not 0: + if self.return_code() != 0: r = r + 'List of failed tests:\n\t{}\n'.format(' '.join([t or 'all tests' for t,_ in self.failed_tests()])) r = r + 'Return code:[{}]\n'.format(self.return_code()) @@ -96,7 +93,7 @@ def run(self, fail_fast=True, options=None, **kwargs): if options is None: options = [str() for _ in range(len(self.test_runners))] - if type(options) != list or len(options) != len(self.test_runners): + if not isinstance(options, list) or len(options) != len(self.test_runners): raise ValueError('options must be a list having a size equal to the number of concurrent executors') logging.info(f'options:{options}') @@ -128,7 +125,7 @@ def run(self, fail_fast=True, options=None, **kwargs): try: f.result() - if tr.rc is 0 and len(tr.failed_tests()) is 0: + if tr.rc == 0 and len(tr.failed_tests()) == 0: logging.error(f'[{tr.name()}]: tests completed successfully') else: logging.error(f'[{tr.name()}]: some tests failed') @@ -139,7 +136,8 @@ def run(self, fail_fast=True, options=None, **kwargs): tr.rc = 1 - if fail_fast: raise + if fail_fast: + raise end_time = time.time() diff --git a/irods_testing_environment/test_runner.py b/irods_testing_environment/test_runner.py index 4b8e3ac..2960a34 100644 --- a/irods_testing_environment/test_runner.py +++ b/irods_testing_environment/test_runner.py @@ -143,7 +143,7 @@ def run(self, test_queue, fail_fast=True, **kwargs): logging.info(f'[{self.name()}]: cmd [{ec}] [{cmd}]') - if ec is 0: + if ec == 0: self.passed_tests().append((t, duration)) logging.error(f'[{self.name()}]: test passed [[{duration:>9.4f}]s] [{t or "all tests"}]') @@ -162,7 +162,7 @@ def run(self, test_queue, fail_fast=True, **kwargs): self.duration = run_end - run_start - if self.rc is not 0: + if self.rc != 0: logging.error('[{}]: tests that failed [{}]'.format(self.name(), self.failed_tests())) @@ -197,7 +197,8 @@ def execute_test(self, test, options=None): cmd.extend(['--run_python_suite'] if test is None else ['--run_specific_test', test]) - if options: cmd.extend(options) + if options: + cmd.extend(options) return cmd, execute.execute_command(self.executor, ' '.join(cmd), @@ -318,6 +319,7 @@ def execute_test(self, url_base='https://github.com/irods', branch='main') - if options: cmd.extend(options) + if options: + cmd.extend(options) return cmd, execute.execute_command(self.executor, ' '.join(cmd)) diff --git a/irods_testing_environment/test_utils.py b/irods_testing_environment/test_utils.py index 96ecd71..edae554 100644 --- a/irods_testing_environment/test_utils.py +++ b/irods_testing_environment/test_utils.py @@ -1,15 +1,12 @@ # grown-up modules import logging import os -import tempfile import errno # local modules -from . import archive from . import container_info from . import context from . import execute -from . import services from . import test_manager def job_name(project_name, prefix=None): @@ -134,7 +131,8 @@ def run_python_test_suite(container, options=None): context.run_tests_script(), '--run_python_suite'] - if options: command.extend(options) + if options: + command.extend(options) ec = execute.execute_command(container, ' '.join(command), @@ -142,7 +140,7 @@ def run_python_test_suite(container, options=None): workdir=context.irods_home(), stream_output=True) - if ec is not 0: + if ec != 0: logging.warning('command exited with error code [{}] [{}] [{}]' .format(ec, command, container.name)) diff --git a/negotiation_key.py b/negotiation_key.py deleted file mode 100644 index d1d0db4..0000000 --- a/negotiation_key.py +++ /dev/null @@ -1,153 +0,0 @@ -# grown-up modules -import compose.cli.command -import docker -import logging -import os - -# local modules -import context -import execute -import json_utils - -def backup_file(container, file_path): - backup_file_path = file_path + '.orig' - if execute.execute_command(container, 'cp {} {}'.format(file_path, backup_file_path)) is not 0: - raise RuntimeError('failed to backup [{}] [{}]'.format(file_path, container.name)) - - -def restore_file(container, file_path): - backup_file_path = file_path + '.orig' - if execute.execute_command(container, 'cp {} {}'.format(backup_file_path, file_path)) is not 0: - raise RuntimeError('failed to restore [{}] [{}]'.format(file_path, container.name)) - - -def do_negotiation_key_tests(target_container, - remote_container, - client_policies, - server_policies, - negotiation_keys, - remote_server_policy='CS_NEG_REQUIRE'): - irods_env = json_utils.get_json_from_file(target_container, context.service_account_irods_env()) - server_config = json_utils.get_json_from_file(target_container, context.server_config()) - - rc = 0 - - configure_ssl_in_server(remote_container, remote_server_policy) - #show_configurations(remote_container, True) - - for client_ssl_negotiation in client_policies: - configure_ssl_in_client(target_container, client_ssl_negotiation, irods_env) - - for server_ssl_negotiation in server_policies: - configure_ssl_in_server(target_container, server_ssl_negotiation) - - for nk in negotiation_keys: - logging.warning( - 'irods_client_server_policy [{}] acPreConnect [{}] negotiation_key [{}]' - .format(client_ssl_negotiation, server_ssl_negotiation, nk)) - - configure_negotiation_key(target_container, nk, server_config) - - show_configurations(target_container) - - ec = execute.execute_command(target_container, 'ils', user='irods', stream_output=True) - if ec is not 0: - logging.error('running "ils" resulted in an error [{}] [{}] [{}] [{}]' - .format(ec, client_ssl_negotiation, server_ssl_negotiation, nk)) - rc = ec - else: - logging.warning('success!') - - return rc - - -def test_negotiation_key(target_container, remote_container): - logging.info('target [{}] remote [{}]'.format(target_container.name, remote_container.name)) - - execute.execute_command(target_container, 'apt install -y jq') - execute.execute_command(remote_container, 'apt install -y jq') - - irods_client_server_policies = ['CS_NEG_DONT_CARE', - 'CS_NEG_REFUSE', - 'CS_NEG_REQUIRE'] - - negotiation_keys = [None, # missing - '', # empty - 'too_short', # too short - '32_byte_server_negotiation_key__too_long', # too long - '32_byte_server_negotiation_key__'] # valid - - backup_file(remote_container, context.service_account_irods_env()) - backup_file(remote_container, context.server_config()) - backup_file(remote_container, context.core_re()) - backup_file(target_container, context.service_account_irods_env()) - backup_file(target_container, context.server_config()) - backup_file(target_container, context.core_re()) - - try: - return do_negotiation_key_tests(target_container, - remote_container, - irods_client_server_policies, - irods_client_server_policies, - negotiation_keys) - - finally: - restore_file(remote_container, context.service_account_irods_env()) - restore_file(remote_container, context.server_config()) - restore_file(remote_container, context.core_re()) - restore_file(target_container, context.service_account_irods_env()) - restore_file(target_container, context.server_config()) - restore_file(target_container, context.core_re()) - - return 1 - - -if __name__ == "__main__": - import argparse - - import cli - import logs - - parser = argparse.ArgumentParser(description='Run negotiation_key test.') - - cli.add_common_args(parser) - cli.add_compose_args(parser) - - args = parser.parse_args() - - docker_client = docker.from_env() - - project_directory = os.path.abspath(args.project_directory or os.getcwd()) - - compose_project = compose.cli.command.get_project(project_dir=project_directory, - project_name=args.project_name) - - logs.configure(args.verbosity) - - try: - if True: - exit( - test_negotiation_key( - docker_client.containers.get( - context.irods_catalog_consumer_container(compose_project.name) - ), - docker_client.containers.get( - context.irods_catalog_provider_container(compose_project.name) - ) - ) - ) - else: - exit( - test_negotiation_key( - docker_client.containers.get( - context.irods_catalog_provider_container(compose_project.name) - ), - docker_client.containers.get( - context.irods_catalog_consumer_container(compose_project.name) - ) - ) - ) - - except Exception as e: - logging.critical(e) - exit(1) diff --git a/run_plugin_tests.py b/run_plugin_tests.py index d02b986..d4a3428 100644 --- a/run_plugin_tests.py +++ b/run_plugin_tests.py @@ -4,7 +4,6 @@ import docker import logging import os -import textwrap # local modules from irods_testing_environment import archive diff --git a/run_topology_tests.py b/run_topology_tests.py index 283c3a0..8710d9e 100644 --- a/run_topology_tests.py +++ b/run_topology_tests.py @@ -7,10 +7,7 @@ # local modules from irods_testing_environment import archive from irods_testing_environment import context -from irods_testing_environment import execute -from irods_testing_environment import install from irods_testing_environment import irods_config -from irods_testing_environment import irods_setup from irods_testing_environment import services from irods_testing_environment import ssl_setup from irods_testing_environment import test_utils diff --git a/run_unit_tests.py b/run_unit_tests.py index 7459373..6e231f1 100644 --- a/run_unit_tests.py +++ b/run_unit_tests.py @@ -5,7 +5,6 @@ import os # local modules -from irods_testing_environment import archive from irods_testing_environment import context from irods_testing_environment import irods_config from irods_testing_environment import services @@ -13,7 +12,6 @@ if __name__ == "__main__": import argparse - import textwrap import cli from irods_testing_environment import logs diff --git a/setup.py b/setup.py deleted file mode 100644 index c74fc45..0000000 --- a/setup.py +++ /dev/null @@ -1,110 +0,0 @@ -# grown-up modules -import compose.cli.command -import docker -import logging -import os - -# local modules -from irods_testing_environment import context -from irods_testing_environment import database_setup -from irods_testing_environment import irods_setup - -if __name__ == "__main__": - import argparse - import textwrap - - import cli - from irods_testing_environment import logs - - parser = argparse.ArgumentParser(description='Setup the iRODS catalog, catalog service provider, and catalog service consumers on a running docker-compose project.') - - cli.add_common_args(parser) - cli.add_compose_args(parser) - cli.add_database_config_args(parser) - - parser.add_argument('--irods-zone-name', - metavar='ZONE_NAME', - dest='zone_name', default='tempZone', - help='Desired name for the iRODS Zone being set up.') - - parser.add_argument('--catalog-service-instance', - metavar='CATALOG_SERVICE_INSTANCE_NUM', - dest='catalog_instance', type=int, default=1, - help=textwrap.dedent('''\ - The Compose service instance number of the database server \ - hosting the iRODS catalog.''')) - - parser.add_argument('--irods-catalog-provider-service-instance', - metavar='IRODS_CATALOG_PROVIDER_INSTANCE_NUM', - dest='irods_csp_instance', type=int, default=1, - help=textwrap.dedent('''\ - The Compose service instance number of the iRODS catalog service \ - provider to set up.''')) - - parser.add_argument('--irods-catalog-consumer-service-instances', - metavar='IRODS_CATALOG_CONSUMER_INSTANCE_NUM', - dest='irods_csc_instances', type=int, nargs='+', - help=textwrap.dedent('''\ - The Compose service instance numbers of the iRODS catalog service \ - consumers to set up.''')) - - parser.add_argument('--exclude-catalog-setup', - dest='setup_catalog', action='store_false', - help='Skip setup of iRODS tables and user in the database.') - - parser.add_argument('--exclude-irods-catalog-provider-setup', - dest='setup_csp', action='store_false', - help='Skip iRODS setup script on the catalog service provider.') - - parser.add_argument('--exclude-irods-catalog-consumers-setup', - dest='setup_cscs', action='store_false', - help='Skip iRODS setup script on the catalog service consumers.') - - parser.add_argument('--force-recreate', - dest='force_recreate', action='store_true', - help=textwrap.dedent('''\ - Force recreating the iRODS catalog and database user by \ - dropping existing database and deleting existing user. \ - NOTE: iRODS setup script must be run again to use iRODS.''')) - - args = parser.parse_args() - - logs.configure(args.verbosity) - - project_directory = os.path.abspath(args.project_directory or os.getcwd()) - - ctx = context.context(docker.from_env(), - compose.cli.command.get_project( - project_dir=project_directory, - project_name=args.project_name)) - - logging.debug('provided project name [{}], docker-compose project name [{}]' - .format(args.project_name, ctx.compose_project.name)) - - if len(ctx.compose_project.containers()) is 0: - logging.critical('no containers found for project [directory=[{}], name=[{}]]' - .format(os.path.abspath(project_directory), ctx.compose_project.name)) - - exit(1) - - try: - if args.setup_catalog: - database_setup.setup_catalog(ctx, - service_instance=args.catalog_instance, - force_recreate=args.force_recreate) - - if args.setup_csp: - irods_setup.setup_irods_catalog_provider(ctx, - args.catalog_instance, - args.irods_csp_instance, - args.odbc_driver) - - if args.setup_cscs: - irods_setup.setup_irods_catalog_consumers(ctx, - args.irods_csp_instance, - args.irods_csc_instances) - - except Exception as e: - logging.critical(e) - raise - From c875835bebc745e5f899efb8c7b126e5d716b186 Mon Sep 17 00:00:00 2001 From: Alan King Date: Wed, 13 Nov 2024 16:38:47 -0500 Subject: [PATCH 4/4] [ 233] Run Black formatter on codebase --- cli.py | 277 +++++---- configure_ssl.py | 9 +- federate.py | 120 ++-- irods_testing_environment/archive.py | 125 ++-- irods_testing_environment/container_info.py | 2 +- irods_testing_environment/context.py | 132 ++-- irods_testing_environment/database_setup.py | 245 +++++--- irods_testing_environment/execute.py | 19 +- irods_testing_environment/federate.py | 81 ++- .../install/almalinux_installer.py | 16 +- .../install/centos_installer.py | 16 +- .../install/debian_installer.py | 16 +- irods_testing_environment/install/install.py | 249 +++++--- .../install/rockylinux_installer.py | 16 +- .../install/ubuntu_installer.py | 16 +- irods_testing_environment/irods_config.py | 436 ++++++++----- irods_testing_environment/irods_setup.py | 587 +++++++++++------- irods_testing_environment/json_utils.py | 8 +- irods_testing_environment/logs.py | 45 +- irods_testing_environment/negotiation_key.py | 100 ++- irods_testing_environment/odbc_setup.py | 216 +++++-- irods_testing_environment/services.py | 106 ++-- irods_testing_environment/ssl_setup.py | 219 ++++--- irods_testing_environment/test_manager.py | 71 ++- irods_testing_environment/test_runner.py | 172 ++--- irods_testing_environment/test_utils.py | 85 +-- run_core_tests.py | 101 +-- run_federation_tests.py | 146 +++-- run_plugin_tests.py | 135 ++-- run_topology_tests.py | 165 +++-- run_unit_tests.py | 61 +- stand_it_up.py | 71 ++- 32 files changed, 2496 insertions(+), 1567 deletions(-) diff --git a/cli.py b/cli.py index 035e4fd..e54822b 100644 --- a/cli.py +++ b/cli.py @@ -1,55 +1,75 @@ # grown-up modules import textwrap + def add_compose_args(parser): - '''Add argparse options related to Docker Compose project. + """Add argparse options related to Docker Compose project. Arguments: parser -- argparse.ArgumentParser to augment - ''' - parser.add_argument('--project-directory', - metavar='PATH_TO_PROJECT_DIRECTORY', - dest='project_directory', - default='.', - help='Path to Compose project on which packages will be installed.') - - parser.add_argument('--project-name', - metavar='PROJECT_NAME', - dest='project_name', - help='Name of Compose project on which to install packages.') + """ + parser.add_argument( + "--project-directory", + metavar="PATH_TO_PROJECT_DIRECTORY", + dest="project_directory", + default=".", + help="Path to Compose project on which packages will be installed.", + ) + + parser.add_argument( + "--project-name", + metavar="PROJECT_NAME", + dest="project_name", + help="Name of Compose project on which to install packages.", + ) + def add_irods_package_args(parser): - '''Add argparse options related to to-be-installed iRODS packages. + """Add argparse options related to to-be-installed iRODS packages. Arguments: parser -- argparse.ArgumentParser to augment - ''' - parser.add_argument('--irods-package-directory', - metavar='PATH_TO_DIRECTORY_WITH_PACKAGES', - dest='package_directory', - help='Path to local directory which contains iRODS packages.') - - parser.add_argument('--irods-package-version', - metavar='PACKAGE_VERSION_TO_DOWNLOAD', - dest='package_version', - help=textwrap.dedent('''\ + """ + parser.add_argument( + "--irods-package-directory", + metavar="PATH_TO_DIRECTORY_WITH_PACKAGES", + dest="package_directory", + help="Path to local directory which contains iRODS packages.", + ) + + parser.add_argument( + "--irods-package-version", + metavar="PACKAGE_VERSION_TO_DOWNLOAD", + dest="package_version", + help=textwrap.dedent( + """\ Version of official iRODS packages to download and install. \ If neither this or --irods-package-directory are specified, \ the latest available version will be installed. Can be used \ - with --use-static-image to avoid downloading packages.''')) - - parser.add_argument('--irods-externals-package-directory', - metavar='PATH_TO_DIRECTORY_WITH_IRODS_EXTERNALS_PACKAGES', - dest='irods_externals_package_directory', - help='Path to local directory which contains iRODS externals packages.') - - parser.add_argument('--use-static-image', - dest='install_packages', default=True, action='store_false', - help=textwrap.dedent('''\ + with --use-static-image to avoid downloading packages.""" + ), + ) + + parser.add_argument( + "--irods-externals-package-directory", + metavar="PATH_TO_DIRECTORY_WITH_IRODS_EXTERNALS_PACKAGES", + dest="irods_externals_package_directory", + help="Path to local directory which contains iRODS externals packages.", + ) + + parser.add_argument( + "--use-static-image", + dest="install_packages", + default=True, + action="store_false", + help=textwrap.dedent( + """\ If specified, a Docker image which has pre-installed iRODS \ packages will be used instead of installing the packages at \ runtime. This is incompatible with --irods-package-directory. \ - --irods-package-version must be specified to use this option.''')) + --irods-package-version must be specified to use this option.""" + ), + ) def add_irods_plugin_args(parser): @@ -58,23 +78,27 @@ def add_irods_plugin_args(parser): Arguments: parser -- argparse.ArgumentParser to augment """ - parser.add_argument('plugin_name', - metavar='PLUGIN_GIT_REPOSITORY_NAME', - help='Repository name for the plugin being installed.') - - parser.add_argument('--plugin-package-directory', - metavar='PATH_TO_DIRECTORY_WITH_PACKAGES', - dest='plugin_package_directory', - help='Path to local directory which contains iRODS plugin packages.') + parser.add_argument( + "plugin_name", + metavar="PLUGIN_GIT_REPOSITORY_NAME", + help="Repository name for the plugin being installed.", + ) + + parser.add_argument( + "--plugin-package-directory", + metavar="PATH_TO_DIRECTORY_WITH_PACKAGES", + dest="plugin_package_directory", + help="Path to local directory which contains iRODS plugin packages.", + ) # TODO: implement support - #parser.add_argument('--plugin-package-version', - #metavar='PACKAGE_VERSION_TO_DOWNLOAD', - #dest='plugin_package_version', - #help=textwrap.dedent('''\ - #Version of official iRODS plugin packages to download and install. \ - #If neither this or --plugin-package-directory are specified, \ - #the latest available version will be installed.''')) + # parser.add_argument('--plugin-package-version', + # metavar='PACKAGE_VERSION_TO_DOWNLOAD', + # dest='plugin_package_version', + # help=textwrap.dedent('''\ + # Version of official iRODS plugin packages to download and install. \ + # If neither this or --plugin-package-directory are specified, \ + # the latest available version will be installed.''')) def add_irods_test_args(parser): @@ -83,89 +107,134 @@ def add_irods_test_args(parser): Arguments: parser -- argparse.ArgumentParser to augment """ - parser.add_argument('--tests', - metavar='TESTS', - nargs='+', - help=textwrap.dedent('''\ + parser.add_argument( + "--tests", + metavar="TESTS", + nargs="+", + help=textwrap.dedent( + """\ Space-delimited list of tests to be run. If not provided, \ - ALL tests will be run (--run_python-suite).''')) - - parser.add_argument('--output-directory', '-o', - metavar='FULLPATH_TO_DIRECTORY_FOR_OUTPUT', - dest='output_directory', - help='Full path to local directory for output from execution. \ + ALL tests will be run (--run_python-suite).""" + ), + ) + + parser.add_argument( + "--output-directory", + "-o", + metavar="FULLPATH_TO_DIRECTORY_FOR_OUTPUT", + dest="output_directory", + help="Full path to local directory for output from execution. \ Individual job runs will appear as subdirectories in this \ - directory. Defaults to temporary directory.') - - parser.add_argument('--job-name', '-j', - metavar='JOB_NAME', - dest='job_name', - help='Name of the directory where output from a specific job will \ - appear within the output directory. Defaults to a UUID.') - - parser.add_argument('--fail-fast', - dest='fail_fast', action='store_true', - help=textwrap.dedent('''\ + directory. Defaults to temporary directory.", + ) + + parser.add_argument( + "--job-name", + "-j", + metavar="JOB_NAME", + dest="job_name", + help="Name of the directory where output from a specific job will \ + appear within the output directory. Defaults to a UUID.", + ) + + parser.add_argument( + "--fail-fast", + dest="fail_fast", + action="store_true", + help=textwrap.dedent( + """\ If indicated, exits on the first test that returns a non-zero exit \ - code.''')) - - parser.add_argument('--concurrent-test-executor-count', - dest='executor_count', type=int, default=1, - help=textwrap.dedent('''\ - Number of concurrent executors to run tests at the same time.''')) - - parser.add_argument('--discard-logs', - dest='save_logs', default=True, action='store_false', - help=textwrap.dedent('''\ + code.""" + ), + ) + + parser.add_argument( + "--concurrent-test-executor-count", + dest="executor_count", + type=int, + default=1, + help=textwrap.dedent( + """\ + Number of concurrent executors to run tests at the same time.""" + ), + ) + + parser.add_argument( + "--discard-logs", + dest="save_logs", + default=True, + action="store_false", + help=textwrap.dedent( + """\ Indicates that the logs should not be collected from the \ - containers.''')) + containers.""" + ), + ) - parser.add_argument('--leak-containers', - action='store_false', dest='cleanup_containers', - help='If indicated, the containers will not be torn down.') + parser.add_argument( + "--leak-containers", + action="store_false", + dest="cleanup_containers", + help="If indicated, the containers will not be torn down.", + ) - parser.add_argument('--skip-setup', - action='store_false', dest='do_setup', - help='If indicated, the iRODS servers will not be set up.') + parser.add_argument( + "--skip-setup", + action="store_false", + dest="do_setup", + help="If indicated, the iRODS servers will not be set up.", + ) def add_database_config_args(parser): - '''Add argparse options related to setting up and configuring iRODS. + """Add argparse options related to setting up and configuring iRODS. Arguments: parser -- argparse.ArgumentParser to augment - ''' - parser.add_argument('--odbc-driver-path', - metavar='PATH_TO_ODBC_DRIVER_ARCHIVE', - dest='odbc_driver', - help=textwrap.dedent('''\ + """ + parser.add_argument( + "--odbc-driver-path", + metavar="PATH_TO_ODBC_DRIVER_ARCHIVE", + dest="odbc_driver", + help=textwrap.dedent( + """\ Path to the ODBC driver archive file on the local machine. \ - If not provided, the driver will be downloaded.''')) + If not provided, the driver will be downloaded.""" + ), + ) + def add_common_args(parser): - '''Add argparse options common to irods_testing_environment scripts. + """Add argparse options common to irods_testing_environment scripts. Arguments: parser -- argparse.ArgumentParser to augment - ''' - parser.add_argument('--verbose', '-v', - dest='verbosity', action='count', default=1, - help=textwrap.dedent('''\ + """ + parser.add_argument( + "--verbose", + "-v", + dest="verbosity", + action="count", + default=1, + help=textwrap.dedent( + """\ Increase the level of output to stdout. \ CRITICAL and ERROR messages will always be printed. \ - Add more to see more log messages (e.g. -vvv displays DEBUG).''')) + Add more to see more log messages (e.g. -vvv displays DEBUG).""" + ), + ) def log_irods_version_and_commit_id(container): - '''Prints the version and commit_id found in the JSON version file. + """Prints the version and commit_id found in the JSON version file. Arguments: container -- Container which has the version file - ''' + """ import logging from irods_testing_environment import irods_config version = ".".join([str(i) for i in irods_config.get_irods_version(container)]) sha = irods_config.get_irods_commit_id(container) - logging.error(f'version:[{version}]') - logging.error(f'sha:[{sha}]') + logging.error(f"version:[{version}]") + logging.error(f"sha:[{sha}]") diff --git a/configure_ssl.py b/configure_ssl.py index dc196fc..6148b52 100644 --- a/configure_ssl.py +++ b/configure_ssl.py @@ -13,7 +13,9 @@ import cli - parser = argparse.ArgumentParser(description='Configure SSL in a running iRODS Zone.') + parser = argparse.ArgumentParser( + description="Configure SSL in a running iRODS Zone." + ) cli.add_common_args(parser) cli.add_compose_args(parser) @@ -22,8 +24,9 @@ docker_client = docker.from_env() - compose_project = compose.cli.command.get_project(os.path.abspath(args.project_directory), - project_name=args.project_name) + compose_project = compose.cli.command.get_project( + os.path.abspath(args.project_directory), project_name=args.project_name + ) logs.configure(args.verbosity) diff --git a/federate.py b/federate.py index 5a9ce24..135155e 100644 --- a/federate.py +++ b/federate.py @@ -18,66 +18,92 @@ import cli from irods_testing_environment import logs - parser = argparse.ArgumentParser(description='Stand up and federate two or more iRODS zones.') + parser = argparse.ArgumentParser( + description="Stand up and federate two or more iRODS zones." + ) cli.add_common_args(parser) cli.add_compose_args(parser) cli.add_database_config_args(parser) cli.add_irods_package_args(parser) - parser.add_argument('--consumers-per-zone', - metavar='IRODS_CATALOG_CONSUMER_INSTANCES_PER_ZONE', - dest='consumers_per_zone', type=int, default=0, - help=textwrap.dedent('''\ + parser.add_argument( + "--consumers-per-zone", + metavar="IRODS_CATALOG_CONSUMER_INSTANCES_PER_ZONE", + dest="consumers_per_zone", + type=int, + default=0, + help=textwrap.dedent( + """\ Number of iRODS Catalog Service Consumer service instances per \ - Zone.''')) - - parser.add_argument('--federate-consumers', - dest='federate_consumers', action='store_true', - help=textwrap.dedent('''\ + Zone.""" + ), + ) + + parser.add_argument( + "--federate-consumers", + dest="federate_consumers", + action="store_true", + help=textwrap.dedent( + """\ If indicated, the iRODS Catalog Service Consumers for each Zone \ will be federated with each of the other Zones in addition to the \ iRODS Catalog Service Providers (which are required to be \ - federated).''')) - - parser.add_argument('--zone-names', - metavar='IRODS_ZONE_NAME', - nargs='+', dest='zone_names', - help='Space-delimited list of zone names to set up.') - - parser.add_argument('--skip-setup', - action='store_false', dest='do_setup', - help='If indicated, the Zones will not be set up, only federated.') - - parser.add_argument('--use-ssl', - dest='use_ssl', action='store_true', - help=textwrap.dedent('''\ + federated).""" + ), + ) + + parser.add_argument( + "--zone-names", + metavar="IRODS_ZONE_NAME", + nargs="+", + dest="zone_names", + help="Space-delimited list of zone names to set up.", + ) + + parser.add_argument( + "--skip-setup", + action="store_false", + dest="do_setup", + help="If indicated, the Zones will not be set up, only federated.", + ) + + parser.add_argument( + "--use-ssl", + dest="use_ssl", + action="store_true", + help=textwrap.dedent( + """\ Indicates that SSL should be configured and enabled in each Zone.\ - ''')) + """ + ), + ) args = parser.parse_args() if not args.package_version and not args.install_packages: - print('--irods-package-version is required when using --use-static-image') + print("--irods-package-version is required when using --use-static-image") exit(1) if args.package_directory and args.package_version: - print('--package-directory and --package-version are incompatible') + print("--package-directory and --package-version are incompatible") exit(1) - zone_names = args.zone_names or ['tempZone', 'otherZone'] + zone_names = args.zone_names or ["tempZone", "otherZone"] project_directory = os.path.abspath(args.project_directory or os.getcwd()) if not args.install_packages: - os.environ['dockerfile'] = 'release.Dockerfile' + os.environ["dockerfile"] = "release.Dockerfile" if args.package_version: - os.environ['irods_package_version'] = args.package_version + os.environ["irods_package_version"] = args.package_version - ctx = context.context(docker.from_env(), - compose.cli.command.get_project( - project_dir=project_directory, - project_name=args.project_name)) + ctx = context.context( + docker.from_env(), + compose.cli.command.get_project( + project_dir=project_directory, project_name=args.project_name + ), + ) logs.configure(args.verbosity) @@ -86,23 +112,28 @@ if args.do_setup: # Bring up the services - logging.debug('bringing up project [{}]'.format(ctx.compose_project.name)) + logging.debug("bringing up project [{}]".format(ctx.compose_project.name)) ctx.compose_project.build() - ctx.compose_project.up(scale_override={ - context.irods_catalog_database_service(): zone_count, - context.irods_catalog_provider_service(): zone_count, - context.irods_catalog_consumer_service(): consumer_count - }) + ctx.compose_project.up( + scale_override={ + context.irods_catalog_database_service(): zone_count, + context.irods_catalog_provider_service(): zone_count, + context.irods_catalog_consumer_service(): consumer_count, + } + ) # The catalog consumers are only determined after the containers are running - zone_info_list = irods_setup.get_info_for_zones(ctx, zone_names, args.consumers_per_zone) + zone_info_list = irods_setup.get_info_for_zones( + ctx, zone_names, args.consumers_per_zone + ) if args.install_packages: install.make_installer(ctx.platform_name()).install_irods_packages( ctx, externals_directory=args.irods_externals_package_directory, package_directory=args.package_directory, - package_version=args.package_version) + package_version=args.package_version, + ) irods_setup.setup_irods_zones(ctx, zone_info_list, odbc_driver=args.odbc_driver) @@ -110,7 +141,8 @@ ssl_setup.configure_ssl_in_zone(ctx.docker_client, ctx.compose_project) else: - zone_info_list = irods_setup.get_info_for_zones(ctx, zone_names, args.consumers_per_zone) + zone_info_list = irods_setup.get_info_for_zones( + ctx, zone_names, args.consumers_per_zone + ) federate.form_federation_clique(ctx, zone_info_list, args.federate_consumers) - diff --git a/irods_testing_environment/archive.py b/irods_testing_environment/archive.py index 2009b69..68f5083 100644 --- a/irods_testing_environment/archive.py +++ b/irods_testing_environment/archive.py @@ -7,7 +7,8 @@ # local modules from . import execute -def create_archive(members, filename='foo', extension='tar'): + +def create_archive(members, filename="foo", extension="tar"): """Create a local archive file with the files in `members` and return a path to the file. Arguments: @@ -16,14 +17,14 @@ def create_archive(members, filename='foo', extension='tar'): # TODO: allow for path to be specified # TODO: allow for type of archive to be specified # Create a tarfile with the packages - tarfile_name = '.'.join([filename, extension]) + tarfile_name = ".".join([filename, extension]) tarfile_path = os.path.join(tempfile.mkdtemp(), tarfile_name) - logging.debug('creating tarfile [{}]'.format(tarfile_path)) + logging.debug("creating tarfile [{}]".format(tarfile_path)) - with tarfile.open(tarfile_path, 'w') as f: + with tarfile.open(tarfile_path, "w") as f: for m in members: - logging.debug('adding member [{0}] to tarfile'.format(m)) + logging.debug("adding member [{0}] to tarfile".format(m)) f.add(m) return tarfile_path @@ -44,39 +45,44 @@ def extract_archive(path_to_archive, path_to_extraction=None): p = os.path.abspath(path_to_archive) - logging.debug('extracting archive [{}] [{}]'.format(p, dest)) + logging.debug("extracting archive [{}] [{}]".format(p, dest)) + + with tarfile.open(p, "r") as f: - with tarfile.open(p, 'r') as f: def is_within_directory(directory, target): - + abs_directory = os.path.abspath(directory) abs_target = os.path.abspath(target) - + prefix = os.path.commonprefix([abs_directory, abs_target]) - + return prefix == abs_directory - + def safe_extract(tar, path=".", members=None, *, numeric_owner=False): - + for member in tar.getmembers(): member_path = os.path.join(path, member.name) if not is_within_directory(path, member_path): raise Exception("Attempted Path Traversal in Tar File") - - tar.extractall(path, members, numeric_owner=numeric_owner) - - + + tar.extractall(path, members, numeric_owner=numeric_owner) + safe_extract(f, path=dest) return dest -def path_to_archive_in_container(archive_file_path_on_host, extension='tar'): +def path_to_archive_in_container(archive_file_path_on_host, extension="tar"): """Return path to directory containing extracted archive when copied to container.""" - return '/' + os.path.basename(os.path.abspath(archive_file_path_on_host))[:(len(extension) + 1) * -1] + return ( + "/" + + os.path.basename(os.path.abspath(archive_file_path_on_host))[ + : (len(extension) + 1) * -1 + ] + ) -def copy_archive_to_container(container, archive_file_path_on_host, extension='tar'): +def copy_archive_to_container(container, archive_file_path_on_host, extension="tar"): """Copy local archive file into the specified container in extracted form. Returns the absolute path inside the container where the archive file was extracted. @@ -87,21 +93,28 @@ def copy_archive_to_container(container, archive_file_path_on_host, extension='t """ dir_path = path_to_archive_in_container(archive_file_path_on_host, extension) - logging.debug('putting archive [{0}] in container [{1}] at [{2}]'.format( - archive_file_path_on_host, container.name, dir_path)) + logging.debug( + "putting archive [{0}] in container [{1}] at [{2}]".format( + archive_file_path_on_host, container.name, dir_path + ) + ) - with open(archive_file_path_on_host, 'rb') as tf: - if not container.put_archive('/', tf): - raise RuntimeError('failed to put archive in container [{}]'.format(container.name)) + with open(archive_file_path_on_host, "rb") as tf: + if not container.put_archive("/", tf): + raise RuntimeError( + "failed to put archive in container [{}]".format(container.name) + ) return dir_path -def copy_from_container(container, - path_to_source_on_container, - path_to_destination_directory_on_host=None, - cleanup=True, - extract=True): +def copy_from_container( + container, + path_to_source_on_container, + path_to_destination_directory_on_host=None, + cleanup=True, + extract=True, +): """Copies a file or directory from a path inside the specified container to the local host. This functions just like `docker cp` on the CLI if the default options are used except @@ -135,7 +148,7 @@ def copy_from_container(container, """ if cleanup and not extract: raise ValueError( - 'cleanup without extraction is a no-op so these are considered incompatible options' + "cleanup without extraction is a no-op so these are considered incompatible options" ) if path_to_destination_directory_on_host: @@ -143,15 +156,18 @@ def copy_from_container(container, else: dest = os.path.join(tempfile.mkdtemp()) - logging.debug('copying file [{}] in container [{}] to [{}]' - .format(path_to_source_on_container, container.name, dest)) + logging.debug( + "copying file [{}] in container [{}] to [{}]".format( + path_to_source_on_container, container.name, dest + ) + ) - archive_path = os.path.join(dest, container.name + '.tar') + archive_path = os.path.join(dest, container.name + ".tar") try: bits, _ = container.get_archive(path_to_source_on_container) - with open(archive_path, 'wb') as f: + with open(archive_path, "wb") as f: for chunk in bits: f.write(chunk) @@ -176,23 +192,27 @@ def copy_files_in_container(container, sources_and_destinations): container -- the docker.Container in which files will be copied sources_and_destinations -- a list of tuples of source paths and destination paths """ - tarfile = create_archive([s for s, d in sources_and_destinations], 'ssl') + tarfile = create_archive([s for s, d in sources_and_destinations], "ssl") _ = copy_archive_to_container(container, tarfile) for s, d in sources_and_destinations: logging.debug( - 'copying source [{}] in container to destination in container [{}] [{}]'.format( - s, d, container.name)) + "copying source [{}] in container to destination in container [{}] [{}]".format( + s, d, container.name + ) + ) - if execute.execute_command(container, 'cp {} {}'.format(s, d)) != 0: - raise RuntimeError('failed to copy file src [{}] dest [{}] [{}]' - .format(s, d, container.name)) + if execute.execute_command(container, "cp {} {}".format(s, d)) != 0: + raise RuntimeError( + "failed to copy file src [{}] dest [{}] [{}]".format( + s, d, container.name + ) + ) -def collect_files_from_containers(docker_client, - containers, - paths_to_copy_from_containers, - output_directory_on_host): +def collect_files_from_containers( + docker_client, containers, paths_to_copy_from_containers, output_directory_on_host +): """Collect files from containers into a single output directory on the host. Arguments: @@ -202,11 +222,13 @@ def collect_files_from_containers(docker_client, output_directory_on_host -- the output directory on the host where files will be copied """ for c in containers: - od = os.path.join(output_directory_on_host, 'logs', c.name) + od = os.path.join(output_directory_on_host, "logs", c.name) if not os.path.exists(od): os.makedirs(od) - logging.info(f'saving files in [{paths_to_copy_from_containers}] to [{od}] [{c.name}]') + logging.info( + f"saving files in [{paths_to_copy_from_containers}] to [{od}] [{c.name}]" + ) source_container = docker_client.containers.get(c.name) @@ -222,5 +244,12 @@ def put_string_to_file(container, target_file, string): target_file -- the path inside the container with the contents to overwrite string -- contents to echo into the target file """ - if execute.execute_command(container, f'bash -c \'echo "{string}" > {target_file}\'') != 0: - raise RuntimeError(f'[{container.name}] failed to put string to file [{target_file}]') + if ( + execute.execute_command( + container, f"bash -c 'echo \"{string}\" > {target_file}'" + ) + != 0 + ): + raise RuntimeError( + f"[{container.name}] failed to put string to file [{target_file}]" + ) diff --git a/irods_testing_environment/container_info.py b/irods_testing_environment/container_info.py index b4c2cc9..8963f7d 100644 --- a/irods_testing_environment/container_info.py +++ b/irods_testing_environment/container_info.py @@ -4,4 +4,4 @@ def python(container): major, minor, patch = irods_config.get_irods_version(container) - return 'python' if minor < 3 else 'python3' + return "python" if minor < 3 else "python3" diff --git a/irods_testing_environment/context.py b/irods_testing_environment/context.py index 25e19b5..c1d685f 100644 --- a/irods_testing_environment/context.py +++ b/irods_testing_environment/context.py @@ -1,5 +1,6 @@ class context(object): """Class for holding Docker/Compose environment and container context information.""" + def __init__(self, docker_client=None, compose_project=None): """Construct a context object. @@ -9,6 +10,7 @@ def __init__(self, docker_client=None, compose_project=None): compose_project -- compose.project information """ import docker + self.docker_client = docker_client or docker.from_env() self.compose_project = compose_project self.platform_image_tag = None @@ -22,12 +24,17 @@ def platform(self, platform_service_name=None, platform_service_instance=1): platform_service_instance -- service instance to target for platform derivation """ if not self.platform_image_tag: - self.platform_image_tag = base_image(self.docker_client.containers.get( - container_name(self.compose_project.name, - platform_service_name or irods_catalog_provider_service(), - platform_service_instance))) - - return self.platform_image_tag.split('/')[-1] + self.platform_image_tag = base_image( + self.docker_client.containers.get( + container_name( + self.compose_project.name, + platform_service_name or irods_catalog_provider_service(), + platform_service_instance, + ) + ) + ) + + return self.platform_image_tag.split("/")[-1] def database(self, database_service_instance=1): """Return database Docker image from the database service in `self.compose_project`. @@ -36,10 +43,13 @@ def database(self, database_service_instance=1): database_service_instance -- service instance to target for database derivation """ if not self.database_image_tag: - self.database_image_tag = base_image(self.docker_client.containers.get( - irods_catalog_database_container(self.compose_project.name))) + self.database_image_tag = base_image( + self.docker_client.containers.get( + irods_catalog_database_container(self.compose_project.name) + ) + ) - return self.database_image_tag.split('/')[-1] + return self.database_image_tag.split("/")[-1] def platform_name(self): """Return the repo name for the OS platform image for this Compose project.""" @@ -51,86 +61,97 @@ def database_name(self): def irods_containers(self): """Return the set of containers running iRODS servers in this Compose project.""" - return [c for c in self.compose_project.containers() - if not is_catalog_database_container(c)] + return [ + c + for c in self.compose_project.containers() + if not is_catalog_database_container(c) + ] + def is_database_plugin(package_name): """Return whether the provided package name is the iRODS database plugin package.""" - return 'irods-database-plugin-' in package_name + return "irods-database-plugin-" in package_name def irods_externals_package_names(): """Return list of iRODS externals package names. For now, just a glob-able string.""" - return ['irods-externals'] + return ["irods-externals"] def irods_package_names(database_name=None): """Return list of iRODS packages (with database plugin if `database_name` provided).""" - irods_package_names = ['irods-runtime', 'irods-icommands', 'irods-server'] + irods_package_names = ["irods-runtime", "irods-icommands", "irods-server"] if database_name: - if database_name == 'mariadb': - database_name = 'mysql' - irods_package_names.append('irods-database-plugin-{}'.format(database_name)) + if database_name == "mariadb": + database_name = "mysql" + irods_package_names.append("irods-database-plugin-{}".format(database_name)) return irods_package_names def irods_catalog_database_service(): """Return name of the iRODS catalog database server docker-compose service.""" - return 'catalog' + return "catalog" def irods_catalog_provider_service(): """Return name of the iRODS catalog service provider docker-compose service.""" - return 'irods-catalog-provider' + return "irods-catalog-provider" def irods_catalog_consumer_service(): """Return name of the iRODS catalog service consumer docker-compose service.""" - return 'irods-catalog-consumer' + return "irods-catalog-consumer" def irods_home(): """Return the path to the iRODS Linux user's home directory.""" import os - return os.path.join('/var', 'lib', 'irods') + + return os.path.join("/var", "lib", "irods") def irods_config(): """Return the path to the iRODS configuration directory.""" import os - return os.path.join('/etc', 'irods') + + return os.path.join("/etc", "irods") def server_config(): """Return the path to the iRODS server_config.json file.""" import os - return os.path.join(irods_config(), 'server_config.json') + + return os.path.join(irods_config(), "server_config.json") def core_re(): """Return the path to the iRODS core.re file.""" import os - return os.path.join(irods_config(), 'core.re') + + return os.path.join(irods_config(), "core.re") def service_account_irods_env(): """Return the path to the iRODS service account client environment file.""" import os - return os.path.join(irods_home(), '.irods', 'irods_environment.json') + + return os.path.join(irods_home(), ".irods", "irods_environment.json") def run_tests_script(): """Return the path to the script which runs the python tests.""" import os - return os.path.join(irods_home(), 'scripts', 'run_tests.py') + + return os.path.join(irods_home(), "scripts", "run_tests.py") def unit_tests(): """Return the path to the directory containing packaged unit tests.""" import os - return os.path.join(irods_home(), 'unit_tests') + + return os.path.join(irods_home(), "unit_tests") def sanitize(repo_or_tag): @@ -139,9 +160,7 @@ def sanitize(repo_or_tag): Arguments: repo_or_tag -- input string which should represent a docker image repo or tag """ - return (repo_or_tag.replace('.', '') - .replace(':', '') - .replace('/', '')) + return repo_or_tag.replace(".", "").replace(":", "").replace("/", "") def project_name(container_name): @@ -154,7 +173,7 @@ def project_name(container_name): Arguments: container_name -- the name of the container from which the project name is extracted """ - return container_name.split('_')[0] + return container_name.split("_")[0] def service_name(container_name): @@ -163,7 +182,7 @@ def service_name(container_name): Arguments: container_name -- the name of the container from which the service name is extracted """ - return container_name.split('_')[1] + return container_name.split("_")[1] def service_instance(container_name): @@ -172,7 +191,7 @@ def service_instance(container_name): Arguments: container_name -- the name of the container from which the service instance is extracted """ - return int(container_name.split('_')[2]) + return int(container_name.split("_")[2]) def container_name(project_name, service_name, service_instance=1): @@ -188,7 +207,7 @@ def container_name(project_name, service_name, service_instance=1): service_name -- name of the service in the docker-compose project (2) service_instance -- number of the instance of the service instance (3) """ - return '_'.join([sanitize(project_name), service_name, str(service_instance)]) + return "_".join([sanitize(project_name), service_name, str(service_instance)]) def base_image(container, tag=0): @@ -201,11 +220,13 @@ def base_image(container, tag=0): container -- docker.container from which the OS platform is to be extracted tag -- The index in the list of Tags for the retrieved base Docker image (default: first) """ - return [image for image in - container.client.images.get( - container.client.api.inspect_container(container.name)['Config']['Image'] - ).history() - if '' not in image['Id']][-1]['Tags'][tag] + return [ + image + for image in container.client.images.get( + container.client.api.inspect_container(container.name)["Config"]["Image"] + ).history() + if "" not in image["Id"] + ][-1]["Tags"][tag] def container_hostname(container): @@ -214,7 +235,7 @@ def container_hostname(container): Arguments: container -- docker.container from which the hostname is to be extracted """ - return container.client.api.inspect_container(container.name)['Config']['Hostname'] + return container.client.api.inspect_container(container.name)["Config"]["Hostname"] def container_ip(container, network_name=None): @@ -224,12 +245,9 @@ def container_ip(container, network_name=None): container -- docker.container from which the IP is to be extracted network_name -- name of the docker network to inspect (if None, default network is used) """ - return (container.client.api.inspect_container(container.name) - ['NetworkSettings'] - ['Networks'] - [network_name or '_'.join([project_name(container.name), 'default'])] - ['IPAddress'] - ) + return container.client.api.inspect_container(container.name)["NetworkSettings"][ + "Networks" + ][network_name or "_".join([project_name(container.name), "default"])]["IPAddress"] def image_repo_and_tag_string(image_repo_and_tag): @@ -243,7 +261,7 @@ def image_repo_and_tag_string(image_repo_and_tag): if isinstance(image_tag, str): return image_repo_and_tag - return ':'.join(image_repo_and_tag) + return ":".join(image_repo_and_tag) def image_repo_and_tag(image_repo_and_tag_string): @@ -255,7 +273,7 @@ def image_repo_and_tag(image_repo_and_tag_string): if isinstance(image_repo_and_tag_string, list): return image_repo_and_tag_string - return image_repo_and_tag_string.split(':') + return image_repo_and_tag_string.split(":") def image_repo(image_repo_and_tag_string): @@ -275,7 +293,9 @@ def irods_catalog_provider_container(project_name, service_instance=1): project_name -- name of the Compose project to inspect service_instance -- the service instance number for the iRODS CSP service """ - return container_name(project_name, irods_catalog_provider_service(), service_instance) + return container_name( + project_name, irods_catalog_provider_service(), service_instance + ) def irods_catalog_consumer_container(project_name, service_instance=1): @@ -285,7 +305,9 @@ def irods_catalog_consumer_container(project_name, service_instance=1): project_name -- name of the Compose project to inspect service_instance -- the service instance number for the iRODS CSC service """ - return container_name(project_name, irods_catalog_consumer_service(), service_instance) + return container_name( + project_name, irods_catalog_consumer_service(), service_instance + ) def irods_catalog_database_container(project_name, service_instance=1): @@ -295,7 +317,9 @@ def irods_catalog_database_container(project_name, service_instance=1): project_name -- name of the Compose project to inspect service_instance -- the service instance number for the database service """ - return container_name(project_name, irods_catalog_database_service(), service_instance) + return container_name( + project_name, irods_catalog_database_service(), service_instance + ) def is_catalog_database_container(container): @@ -329,7 +353,9 @@ def is_irods_server_in_local_zone(container, local_zone): if is_irods_catalog_consumer_container(container): return service_instance(container.name) in local_zone.consumer_service_instances - raise NotImplementedError('service name is not supported [{}]'.format(container.name)) + raise NotImplementedError( + "service name is not supported [{}]".format(container.name) + ) def project_hostnames(docker_client, compose_project): @@ -340,6 +366,6 @@ def project_hostnames(docker_client, compose_project): compose_project -- compose.Project from which hostnames will be derived """ return { - c.name : container_hostname(docker_client.containers.get(c.name)) + c.name: container_hostname(docker_client.containers.get(c.name)) for c in compose_project.containers() } diff --git a/irods_testing_environment/database_setup.py b/irods_testing_environment/database_setup.py index 0dd2cb2..5f3a35e 100644 --- a/irods_testing_environment/database_setup.py +++ b/irods_testing_environment/database_setup.py @@ -6,6 +6,7 @@ from . import context from . import execute + def database_server_port(database_image): """Return the default port for the database server indicated by `database_image`. @@ -14,12 +15,12 @@ def database_server_port(database_image): """ db = context.image_repo(database_image) - if 'postgres' in db: + if "postgres" in db: return 5432 - elif 'mysql' in db or 'mariadb' in db: + elif "mysql" in db or "mariadb" in db: return 3306 else: - raise NotImplementedError('database not supported [{}]'.format(database_image)) + raise NotImplementedError("database not supported [{}]".format(database_image)) class database_setup_strategy(object): @@ -27,6 +28,7 @@ class database_setup_strategy(object): This class should not be instantiated directly. """ + def create_database(self, name, force_recreate=False): """Create a database. @@ -36,7 +38,7 @@ def create_database(self, name, force_recreate=False): name -- name of the database to create force_recreate -- if True, drops any database by the specified name before creating """ - raise NotImplementedError('method not implemented for database strategy') + raise NotImplementedError("method not implemented for database strategy") def create_user(self, username, password, force_recreate=False): """Create a user for the database. @@ -48,7 +50,7 @@ def create_user(self, username, password, force_recreate=False): password -- password for the new user force_recreate -- if True, drops any user by the specified username before creating """ - raise NotImplementedError('method not implemented for database strategy') + raise NotImplementedError("method not implemented for database strategy") def grant_privileges(self, database, username): """Grant all privileges on database to user called `username`. @@ -59,7 +61,7 @@ def grant_privileges(self, database, username): database -- name of the database on which privileges are being granted username -- name of the user for whom privileges are being granted """ - raise NotImplementedError('method not implemented for database strategy') + raise NotImplementedError("method not implemented for database strategy") def drop_database(self, name): """Drop a database called `name`. @@ -69,7 +71,7 @@ def drop_database(self, name): Arguments: name -- name of the database to drop """ - raise NotImplementedError('method not implemented for database strategy') + raise NotImplementedError("method not implemented for database strategy") def drop_user(self, username): """Drop a user named `username`. @@ -79,18 +81,19 @@ def drop_user(self, username): Arguments: username -- name of the user to drop """ - raise NotImplementedError('method not implemented for database strategy') + raise NotImplementedError("method not implemented for database strategy") def list_databases(self): """List databases. This method must be overridden. """ - raise NotImplementedError('method not implemented for database strategy') + raise NotImplementedError("method not implemented for database strategy") class postgres_database_setup_strategy(database_setup_strategy): """Database setup strategy for postgres""" + def __init__(self, container=None, root_password=None, port=None): """Construct a postgres_database_setup_strategy. @@ -100,7 +103,7 @@ def __init__(self, container=None, root_password=None, port=None): port -- port on which the postgres server is listening (default: 5432) """ self.container = container - self.root_password = root_password if root_password else 'testpassword' + self.root_password = root_password if root_password else "testpassword" self.port = port if port else 5432 def execute_psql_command(self, psql_cmd): @@ -109,17 +112,17 @@ def execute_psql_command(self, psql_cmd): Arguments: psql_cmd -- command to be passed to psql via --command """ - cmd = 'psql --port {0} --command \"{1}\"'.format(self.port, psql_cmd) - return execute.execute_command(self.container, cmd, user='postgres') + cmd = 'psql --port {0} --command "{1}"'.format(self.port, psql_cmd) + return execute.execute_command(self.container, cmd, user="postgres") - def connect_to_database(self, name='postgres', as_user='postgres'): + def connect_to_database(self, name="postgres", as_user="postgres"): """Connect to database named `name` as user `as_user`. Arguments: name -- name of the database to check as_user -- name of the user/role connecting to the database """ - return self.execute_psql_command('\c \'{}\' \'{}\';'.format(name, as_user)) + return self.execute_psql_command("\c '{}' '{}';".format(name, as_user)) def database_exists(self, name): """Confirm existence of a database by attempting to connect to it. @@ -147,8 +150,11 @@ def create_database(self, name, force_recreate=False): if force_recreate: self.drop_database(name) - return 0 if self.database_exists(name) \ - else self.execute_psql_command('create database \\\"{}\\\";'.format(name)) + return ( + 0 + if self.database_exists(name) + else self.execute_psql_command('create database \\"{}\\";'.format(name)) + ) def create_user(self, username, password, force_recreate=False): """Create a user for the database. @@ -161,9 +167,13 @@ def create_user(self, username, password, force_recreate=False): if force_recreate: self.drop_user(username) - return 0 if self.user_exists(username) \ - else self.execute_psql_command('create user {0} with password \'{1}\';' - .format(username, password)) + return ( + 0 + if self.user_exists(username) + else self.execute_psql_command( + "create user {0} with password '{1}';".format(username, password) + ) + ) def grant_privileges(self, database, username): """Grant all privileges on database to user called `username`. @@ -172,12 +182,16 @@ def grant_privileges(self, database, username): database -- name of the database on which privileges are being granted username -- name of the user for whom privileges are being granted """ - ec = self.execute_psql_command('grant all privileges on database \\\"{0}\\\" to {1};' - .format(database, username)) + ec = self.execute_psql_command( + 'grant all privileges on database \\"{0}\\" to {1};'.format( + database, username + ) + ) if ec != 0: return ec - return self.execute_psql_command('alter database \\\"{0}\\\" owner to {1};' - .format(database, username)) + return self.execute_psql_command( + 'alter database \\"{0}\\" owner to {1};'.format(database, username) + ) def drop_database(self, name): """Drop a database called `name`. @@ -185,7 +199,7 @@ def drop_database(self, name): Arguments: name -- name of the database to drop """ - return self.execute_psql_command('drop database \\\"{}\\\";'.format(name)) + return self.execute_psql_command('drop database \\"{}\\";'.format(name)) def drop_user(self, username): """Drop a user named `username`. @@ -193,15 +207,16 @@ def drop_user(self, username): Arguments: username -- name of the user to drop """ - return self.execute_psql_command('drop user {};'.format(username)) + return self.execute_psql_command("drop user {};".format(username)) def list_databases(self): """List databases.""" - return self.execute_psql_command('\l') + return self.execute_psql_command("\l") class mysql_database_setup_strategy(database_setup_strategy): """Database setup strategy for mysql""" + def __init__(self, container=None, root_password=None, port=None, db_exec=None): """Construct a mysql_database_setup_strategy. @@ -209,30 +224,32 @@ def __init__(self, container=None, root_password=None, port=None, db_exec=None): container -- docker.client.container running the database root_password -- password for the root database user database_port -- port on which the postgres server is listening (default: 3306) - db_exec -- name of the standard command-line client executable + db_exec -- name of the standard command-line client executable """ self.container = container - self.root_password = root_password if root_password else 'testpassword' + self.root_password = root_password if root_password else "testpassword" self.port = port if port else 3306 - self.db_exec = db_exec if db_exec else 'mysql' + self.db_exec = db_exec if db_exec else "mysql" # TODO: 'irods'@'%' is generated by the docker entrypoint for mysql container... # should be 'irods'@'localhost', but that doesn't work right now - self.host = '%' + self.host = "%" - def execute_mysql_command(self, mysql_cmd, user='root', password='testpassword'): + def execute_mysql_command(self, mysql_cmd, user="root", password="testpassword"): """Execute a mysql command as the postgres user. Arguments: mysql_cmd -- the command to be passed to mysql via --execute """ - return execute.execute_command(self.container, - '{0} --host 127.0.0.1 --port {1} --user {2} --password={3} --execute \"{4}\"' - .format(self.db_exec, self.port, user, password, mysql_cmd)) - - def connect_to_database(self, - name='information_schema', - as_user='root', - with_password='testpassword'): + return execute.execute_command( + self.container, + '{0} --host 127.0.0.1 --port {1} --user {2} --password={3} --execute "{4}"'.format( + self.db_exec, self.port, user, password, mysql_cmd + ), + ) + + def connect_to_database( + self, name="information_schema", as_user="root", with_password="testpassword" + ): """Connect to database named `name` as user `as_user`. Arguments: @@ -240,18 +257,20 @@ def connect_to_database(self, as_user -- name of the user/role connecting to the database """ - logging.debug('checking if database is accepting connection ...') + logging.debug("checking if database is accepting connection ...") # Make sure the database is ready for connections. - while self.execute_mysql_command('SHOW DATABASES;') != 0: - logging.debug('database is not accepting connections yet. retrying in 5 seconds ...') + while self.execute_mysql_command("SHOW DATABASES;") != 0: + logging.debug( + "database is not accepting connections yet. retrying in 5 seconds ..." + ) time.sleep(5) - logging.debug('database is ready!') + logging.debug("database is ready!") - return self.execute_mysql_command('\\r \'{}\''.format(name), - user=as_user, - password=with_password) + return self.execute_mysql_command( + "\\r '{}'".format(name), user=as_user, password=with_password + ) def database_exists(self, name): """Confirm existence of a database by attempting to connect to it. @@ -261,7 +280,7 @@ def database_exists(self, name): """ return self.connect_to_database(name=name) == 0 - def user_exists(self, username, password='testpassword'): + def user_exists(self, username, password="testpassword"): """Confirm existence of a user/role by attempting to connect to database as `username`. Arguments: @@ -279,8 +298,11 @@ def create_database(self, name, force_recreate=False): if force_recreate: self.drop_database(name) - return 0 if self.database_exists(name) \ - else self.execute_mysql_command('CREATE DATABASE {};'.format(name)) + return ( + 0 + if self.database_exists(name) + else self.execute_mysql_command("CREATE DATABASE {};".format(name)) + ) def create_user(self, username, password, force_recreate=False): """Create a user for the database. @@ -290,15 +312,18 @@ def create_user(self, username, password, force_recreate=False): password -- password for the new user force_recreate -- if True, drops any user by the specified username before creating """ - user = '\'{}\'@\'{}\''.format(username, self.host) + user = "'{}'@'{}'".format(username, self.host) if force_recreate: self.drop_user(user) - return 0 if self.user_exists(username, password) \ - else self.execute_mysql_command( - 'CREATE USER {} IDENTIFIED BY \'{}\';' - .format(user, password)) + return ( + 0 + if self.user_exists(username, password) + else self.execute_mysql_command( + "CREATE USER {} IDENTIFIED BY '{}';".format(user, password) + ) + ) def grant_privileges(self, database, username): """Grant all privileges on database to user called `username`. @@ -307,9 +332,11 @@ def grant_privileges(self, database, username): database -- name of the database on which privileges are being granted username -- name of the user for whom privileges are being granted """ - user = '\'{}\'@\'{}\''.format(username, self.host) + user = "'{}'@'{}'".format(username, self.host) - return self.execute_mysql_command('GRANT ALL ON {}.* to {};'.format(database, user)) + return self.execute_mysql_command( + "GRANT ALL ON {}.* to {};".format(database, user) + ) def drop_database(self, name): """Drop a database called `name`. @@ -317,7 +344,7 @@ def drop_database(self, name): Arguments: name -- name of the database to drop """ - return self.execute_mysql_command('DROP DATABASE {};'.format(name)) + return self.execute_mysql_command("DROP DATABASE {};".format(name)) def drop_user(self, username): """Drop a user named `username`. @@ -325,17 +352,18 @@ def drop_user(self, username): Arguments: username -- name of the user to drop """ - user = '\'{}\'@\'{}\''.format(username, self.host) + user = "'{}'@'{}'".format(username, self.host) - return self.execute_mysql_command('DROP USER {};'.format(user)) + return self.execute_mysql_command("DROP USER {};".format(user)) def list_databases(self): """List databases.""" - return self.execute_mysql_command('SHOW DATABASES;') + return self.execute_mysql_command("SHOW DATABASES;") class mariadb_database_setup_strategy(mysql_database_setup_strategy): """Database setup strategy for mariadb""" + def __init__(self, container=None, root_password=None, port=None, db_exec=None): """Construct a mysql_database_setup_strategy. @@ -343,12 +371,16 @@ def __init__(self, container=None, root_password=None, port=None, db_exec=None): container -- docker.client.container running the database root_password -- password for the root database user database_port -- port on which the postgres server is listening (default: 3306) - db_exec -- name of the standard command-line client executable + db_exec -- name of the standard command-line client executable """ - db_exec = db_exec if db_exec else 'mariadb' - super(mariadb_database_setup_strategy, self).__init__(container, root_password, port, db_exec) + db_exec = db_exec if db_exec else "mariadb" + super(mariadb_database_setup_strategy, self).__init__( + container, root_password, port, db_exec + ) - def execute_mariadb_command(self, mariadb_cmd, user='root', password='testpassword'): + def execute_mariadb_command( + self, mariadb_cmd, user="root", password="testpassword" + ): """Execute a mariadb command. Arguments: @@ -357,7 +389,9 @@ def execute_mariadb_command(self, mariadb_cmd, user='root', password='testpasswo return self.execute_mysql_command(mariadb_cmd, user, password) -def make_strategy(database_image, container=None, database_port=None, root_password=None): +def make_strategy( + database_image, container=None, database_port=None, root_password=None +): """Make a database setup strategy for the given database type. Arguments: @@ -366,18 +400,21 @@ def make_strategy(database_image, container=None, database_port=None, root_passw database_port -- port on which the database service is listening database_root_password -- password the database root user """ - strat_name = context.image_repo(database_image) + '_database_setup_strategy' + strat_name = context.image_repo(database_image) + "_database_setup_strategy" return eval(strat_name)(container, database_port, root_password) -def setup_catalog(ctx, - force_recreate=False, - database_port=None, - service_instance=1, - database_name='ICAT', - database_user='irods', - database_password='testpassword', - root_password=None): + +def setup_catalog( + ctx, + force_recreate=False, + database_port=None, + service_instance=1, + database_name="ICAT", + database_user="irods", + database_password="testpassword", + root_password=None, +): """Set up the iRODS catalog on the specified database service. Arguments: @@ -391,30 +428,37 @@ def setup_catalog(ctx, root_password -- password for the root database user """ db_container = ctx.docker_client.containers.get( - context.irods_catalog_database_container(ctx.compose_project.name, service_instance)) + context.irods_catalog_database_container( + ctx.compose_project.name, service_instance + ) + ) - logging.warning('setting up catalog [{}]'.format(db_container.name)) + logging.warning("setting up catalog [{}]".format(db_container.name)) strat = make_strategy(ctx.database(), db_container, database_port, root_password) ec = strat.create_database(database_name, force_recreate) if ec != 0: - raise RuntimeError('failed to create database [{}]'.format(database_name)) + raise RuntimeError("failed to create database [{}]".format(database_name)) ec = strat.create_user(database_user, database_password, force_recreate) if ec != 0: - raise RuntimeError('failed to create user [{}]'.format(database_user)) + raise RuntimeError("failed to create user [{}]".format(database_user)) ec = strat.grant_privileges(database_name, database_user) if ec != 0: - raise RuntimeError('failed to grant privileges to user [{0}] on database [{1}]'.format(database_user, database_name)) + raise RuntimeError( + "failed to grant privileges to user [{0}] on database [{1}]".format( + database_user, database_name + ) + ) strat.list_databases() -def wait_for_database_service(ctx, - database_service_instance=1, - seconds_between_retries=1, - retry_count=60): + +def wait_for_database_service( + ctx, database_service_instance=1, seconds_between_retries=1, retry_count=60 +): """Attempt to connect to the database service, looping until successful. Arguments: @@ -427,36 +471,47 @@ def wait_for_database_service(ctx, from . import container_info if retry_count < 0: - raise ValueError('retry_count must be a non-negative integer') + raise ValueError("retry_count must be a non-negative integer") irods_container = ctx.docker_client.containers.get( - context.irods_catalog_provider_container(ctx.compose_project.name)) + context.irods_catalog_provider_container(ctx.compose_project.name) + ) db_container = ctx.docker_client.containers.get( - context.irods_catalog_database_container(ctx.compose_project.name, database_service_instance)) + context.irods_catalog_database_container( + ctx.compose_project.name, database_service_instance + ) + ) db_address = context.container_ip(db_container) db_port = database_server_port(context.base_image(db_container)) - logging.info(f'waiting for catalog to be ready [{db_container.name}]') + logging.info(f"waiting for catalog to be ready [{db_container.name}]") retries = -1 while retries < retry_count: logging.debug( - f'[{irods_container.name}] trying database on [{db_container.name}] ip:[{db_address}] port:[{db_port}]') + f"[{irods_container.name}] trying database on [{db_container.name}] ip:[{db_address}] port:[{db_port}]" + ) - socket_cmd = str('import socket; ' - 's = socket.socket(socket.AF_INET, socket.SOCK_STREAM); ' - f'ec = s.connect_ex((\'{db_address}\', {db_port})); ' - 's.close(); print(ec); exit(ec)' + socket_cmd = str( + "import socket; " + "s = socket.socket(socket.AF_INET, socket.SOCK_STREAM); " + f"ec = s.connect_ex(('{db_address}', {db_port})); " + "s.close(); print(ec); exit(ec)" ) - cmd = ' '.join([container_info.python(irods_container), '-c', f'"{socket_cmd}"']) + cmd = " ".join( + [container_info.python(irods_container), "-c", f'"{socket_cmd}"'] + ) if execute.execute_command(irods_container, cmd) == 0: logging.debug( - f'[{irods_container.name}] database service ready on [{db_container.name}]') + f"[{irods_container.name}] database service ready on [{db_container.name}]" + ) return retries = retries + 1 time.sleep(seconds_between_retries) - raise RuntimeError(f'maximum retries reached attempting to connect to database [{db_container.name}]') + raise RuntimeError( + f"maximum retries reached attempting to connect to database [{db_container.name}]" + ) diff --git a/irods_testing_environment/execute.py b/irods_testing_environment/execute.py index 793b6d1..9689f89 100644 --- a/irods_testing_environment/execute.py +++ b/irods_testing_environment/execute.py @@ -1,7 +1,8 @@ # grown-up modules import logging -def execute_command(container, command, user='', workdir=None, stream_output=None): + +def execute_command(container, command, user="", workdir=None, stream_output=None): """Execute `command` in `container` as `user` in `workdir`. Running this is equivalent to the following: @@ -18,16 +19,20 @@ def execute_command(container, command, user='', workdir=None, stream_output=Non output will be streamed. Otherwise, the output will stream no matter what if True and it will not stream no matter what if False. """ - OUTPUT_ENCODING = 'utf-8' + OUTPUT_ENCODING = "utf-8" - logging.debug('executing on [{0}] [{1}]'.format(container.name, command)) + logging.debug("executing on [{0}] [{1}]".format(container.name, command)) if stream_output is None: log_level = logging.getLogger().getEffectiveLevel() stream_output = log_level <= logging.INFO - exec_instance = container.client.api.exec_create(container.id, command, user=user, workdir=workdir) - exec_out = container.client.api.exec_start(exec_instance['Id'], stream=stream_output) + exec_instance = container.client.api.exec_create( + container.id, command, user=user, workdir=workdir + ) + exec_out = container.client.api.exec_start( + exec_instance["Id"], stream=stream_output + ) try: # Stream output from the executing command. A StopIteration exception is raised @@ -37,9 +42,9 @@ def execute_command(container, command, user='', workdir=None, stream_output=Non logging.error(out) except StopIteration: - logging.debug('done') + logging.debug("done") if not stream_output: logging.debug(exec_out.decode(OUTPUT_ENCODING)) - return container.client.api.exec_inspect(exec_instance['Id'])['ExitCode'] + return container.client.api.exec_inspect(exec_instance["Id"])["ExitCode"] diff --git a/irods_testing_environment/federate.py b/irods_testing_environment/federate.py index 16f73d4..1f6f6f5 100644 --- a/irods_testing_environment/federate.py +++ b/irods_testing_environment/federate.py @@ -7,6 +7,7 @@ from . import irods_setup from . import json_utils + def make_federation_entry(ctx, local_zone, remote_zone): """Create an entry for the federation stanza to federate two zones together. @@ -16,13 +17,15 @@ def make_federation_entry(ctx, local_zone, remote_zone): remote_zone -- name of the remote iRODS zone with which `local_zone` is federating """ # TODO: Need to have strategies for different version of iRODS, this only works for 4.1/4.2, I think? - negotiation_key_prefix = '_'.join(sorted([local_zone.zone_name, remote_zone.zone_name])) + negotiation_key_prefix = "_".join( + sorted([local_zone.zone_name, remote_zone.zone_name]) + ) return { - 'catalog_provider_hosts': [remote_zone.provider_hostname(ctx)], - 'negotiation_key': irods_setup.make_negotiation_key(negotiation_key_prefix), - 'zone_key': irods_setup.make_zone_key(remote_zone.zone_name), - 'zone_name': remote_zone.zone_name, - 'zone_port': 1247 + "catalog_provider_hosts": [remote_zone.provider_hostname(ctx)], + "negotiation_key": irods_setup.make_negotiation_key(negotiation_key_prefix), + "zone_key": irods_setup.make_zone_key(remote_zone.zone_name), + "zone_name": remote_zone.zone_name, + "zone_port": 1247, } @@ -45,35 +48,47 @@ def federate_zones(ctx, zone_info_list, local_zone, include_consumers=True): if not include_consumers and context.is_irods_catalog_consumer_container(c): continue - logging.debug('container [{}] zone [{}] provider instance [{}]' - .format(c.name, - local_zone.zone_name, - local_zone.provider_service_instance)) + logging.debug( + "container [{}] zone [{}] provider instance [{}]".format( + c.name, local_zone.zone_name, local_zone.provider_service_instance + ) + ) container = ctx.docker_client.containers.get(c.name) - server_config = json_utils.get_json_from_file(container, context.server_config()) + server_config = json_utils.get_json_from_file( + container, context.server_config() + ) for remote_zone in zone_info_list: if remote_zone.zone_name == local_zone.zone_name: continue - logging.warning('federating remote zone [{}] with local zone [{}] on [{}]' - .format(remote_zone.zone_name, local_zone.zone_name, container.name)) + logging.warning( + "federating remote zone [{}] with local zone [{}] on [{}]".format( + remote_zone.zone_name, local_zone.zone_name, container.name + ) + ) - server_config['federation'].append(make_federation_entry(ctx, - local_zone, - remote_zone)) + server_config["federation"].append( + make_federation_entry(ctx, local_zone, remote_zone) + ) # Only make the remote Zone once per local Zone if context.is_irods_catalog_provider_container(container): - make_remote_zone = 'iadmin mkzone {} remote {}:{}'.format(remote_zone.zone_name, - remote_zone.provider_hostname(ctx), - remote_zone.zone_port) - - if execute.execute_command(container, make_remote_zone, user='irods') != 0: - raise RuntimeError('failed to create remote zone [{}]' - .format(container.name)) + make_remote_zone = "iadmin mkzone {} remote {}:{}".format( + remote_zone.zone_name, + remote_zone.provider_hostname(ctx), + remote_zone.zone_port, + ) + + if ( + execute.execute_command(container, make_remote_zone, user="irods") + != 0 + ): + raise RuntimeError( + "failed to create remote zone [{}]".format(container.name) + ) # Write out the server_config.json to the iRODS server container to complete the federation json_utils.put_json_to_file(container, context.server_config(), server_config) @@ -93,20 +108,30 @@ def form_federation_clique(ctx, zone_info_list, include_consumers=True): # configure federation between all zones (O(len(zone_names)^2)) with concurrent.futures.ThreadPoolExecutor() as executor: futures_to_containers = { - executor.submit(federate_zones, ctx, zone_info_list, z, include_consumers): - z for z in zone_info_list + executor.submit( + federate_zones, ctx, zone_info_list, z, include_consumers + ): z + for z in zone_info_list } for f in concurrent.futures.as_completed(futures_to_containers): z = futures_to_containers[f] try: f.result() - logging.debug('iRODS Zone federated successfully [{}]'.format(z.zone_name)) + logging.debug( + "iRODS Zone federated successfully [{}]".format(z.zone_name) + ) except Exception as e: - logging.error('exception raised while federating iRODS Zone [{}]'.format(z.zone_name)) + logging.error( + "exception raised while federating iRODS Zone [{}]".format( + z.zone_name + ) + ) logging.error(e) rc = 1 if rc != 0: - raise RuntimeError('failed to federate one or more iRODS Zones, ec=[{}]'.format(rc)) + raise RuntimeError( + "failed to federate one or more iRODS Zones, ec=[{}]".format(rc) + ) diff --git a/irods_testing_environment/install/almalinux_installer.py b/irods_testing_environment/install/almalinux_installer.py index cf50957..5a7d6a0 100644 --- a/irods_testing_environment/install/almalinux_installer.py +++ b/irods_testing_environment/install/almalinux_installer.py @@ -1,22 +1,18 @@ from . import install + class almalinux_installer(install.installer): def update_command(self): - return 'dnf update -y' - + return "dnf update -y" def install_local_packages_command(self): - return 'dnf --nogpgcheck -y install' - + return "dnf --nogpgcheck -y install" def install_official_packages_command(self): - return 'dnf -y install' - + return "dnf -y install" def filename_extension(self): - return 'rpm' - + return "rpm" def version_joinery(self): - return '-' - + return "-" diff --git a/irods_testing_environment/install/centos_installer.py b/irods_testing_environment/install/centos_installer.py index ec75620..3b0d522 100644 --- a/irods_testing_environment/install/centos_installer.py +++ b/irods_testing_environment/install/centos_installer.py @@ -1,22 +1,18 @@ from . import install + class centos_installer(install.installer): def update_command(self): - return 'yum update -y' - + return "yum update -y" def install_local_packages_command(self): - return 'yum --nogpgcheck -y install' - + return "yum --nogpgcheck -y install" def install_official_packages_command(self): - return 'yum -y install' - + return "yum -y install" def filename_extension(self): - return 'rpm' - + return "rpm" def version_joinery(self): - return '-' - + return "-" diff --git a/irods_testing_environment/install/debian_installer.py b/irods_testing_environment/install/debian_installer.py index f620171..6b9baa2 100644 --- a/irods_testing_environment/install/debian_installer.py +++ b/irods_testing_environment/install/debian_installer.py @@ -1,22 +1,18 @@ from . import install + class debian_installer(install.installer): def update_command(self): - return 'apt update' - + return "apt update" def install_local_packages_command(self): - return 'apt install -fy' - + return "apt install -fy" def install_official_packages_command(self): - return 'apt install -y' - + return "apt install -y" def filename_extension(self): - return 'deb' - + return "deb" def version_joinery(self): - return '=' - + return "=" diff --git a/irods_testing_environment/install/install.py b/irods_testing_environment/install/install.py index 9de7dc2..e4b0b13 100644 --- a/irods_testing_environment/install/install.py +++ b/irods_testing_environment/install/install.py @@ -8,52 +8,54 @@ from .. import context from .. import execute + class installer(object): def update_command(self): - raise NotImplementedError('method not implemented for installer strategy') - + raise NotImplementedError("method not implemented for installer strategy") def install_local_packages_command(self): - raise NotImplementedError('method not implemented for installer strategy') - + raise NotImplementedError("method not implemented for installer strategy") def install_official_packages_command(self): - raise NotImplementedError('method not implemented for installer strategy') - + raise NotImplementedError("method not implemented for installer strategy") def filename_extension(self): - raise NotImplementedError('method not implemented for installer strategy') - + raise NotImplementedError("method not implemented for installer strategy") def version_joinery(self): - raise NotImplementedError('method not implemented for installer strategy') - + raise NotImplementedError("method not implemented for installer strategy") def get_list_of_package_paths(self, package_directory, package_name_list=None): import glob if not package_directory: - raise RuntimeError('Attempting to install custom packages from unspecified location') + raise RuntimeError( + "Attempting to install custom packages from unspecified location" + ) # If nothing is provided, installs everything in package_directory if not package_name_list: - package_name_list = [''] + package_name_list = [""] package_path = os.path.abspath(package_directory) - logging.debug('listing for [{}]:\n{}'.format(package_path, os.listdir(package_path))) + logging.debug( + "listing for [{}]:\n{}".format(package_path, os.listdir(package_path)) + ) packages = list() for p in package_name_list: - glob_str = os.path.join(package_path, p + '*.{}'.format(self.filename_extension())) + glob_str = os.path.join( + package_path, p + "*.{}".format(self.filename_extension()) + ) - logging.debug('looking for packages like [{}]'.format(glob_str)) + logging.debug("looking for packages like [{}]".format(glob_str)) glob_list = glob.glob(glob_str) if len(glob_list) == 0: - raise RuntimeError('no packages found [{}]'.format(glob_str)) + raise RuntimeError("no packages found [{}]".format(glob_str)) # TODO: allow specifying a suffix or something instead of taking everything for item in glob_list: @@ -61,12 +63,9 @@ def get_list_of_package_paths(self, package_directory, package_name_list=None): return packages - - def install_packages_on_container_from_tarfile(self, - ctx, - container_name, - package_paths, - tarfile_path): + def install_packages_on_container_from_tarfile( + self, ctx, container_name, package_paths, tarfile_path + ): """Install specified packages from specified tarfile on specified container. Arguments: @@ -83,35 +82,47 @@ def install_packages_on_container_from_tarfile(self, archive.copy_archive_to_container(container, tarfile_path) - package_list = ' '.join([ - p for p in package_paths - if not context.is_database_plugin(p) or - context.is_irods_catalog_provider_container(container)]) + package_list = " ".join( + [ + p + for p in package_paths + if not context.is_database_plugin(p) + or context.is_irods_catalog_provider_container(container) + ] + ) - cmd = ' '.join([self.install_local_packages_command(), package_list]) + cmd = " ".join([self.install_local_packages_command(), package_list]) - logging.warning('executing cmd [{0}] on container [{1}]'.format(cmd, container.name)) + logging.warning( + "executing cmd [{0}] on container [{1}]".format(cmd, container.name) + ) ec = execute.execute_command(container, self.update_command()) if ec != 0: - logging.error('failed to update local repositories [{}]'.format(container.name)) + logging.error( + "failed to update local repositories [{}]".format(container.name) + ) return ec ec = execute.execute_command(container, cmd) if ec != 0: logging.error( - 'failed to install packages on container [ec=[{0}], container=[{1}]'.format(ec, container.name)) + "failed to install packages on container [ec=[{0}], container=[{1}]".format( + ec, container.name + ) + ) return ec return 0 - - def install_packages(self, ctx, package_directory, containers, package_name_list=None): + def install_packages( + self, ctx, package_directory, containers, package_name_list=None + ): import concurrent.futures packages = self.get_list_of_package_paths(package_directory, package_name_list) - logging.info('packages to install [{}]'.format(packages)) + logging.info("packages to install [{}]".format(packages)) tarfile_path = archive.create_archive(packages) @@ -120,8 +131,12 @@ def install_packages(self, ctx, package_directory, containers, package_name_list futures_to_containers = { executor.submit( self.install_packages_on_container_from_tarfile, - ctx, c.name, packages, tarfile_path - ): c for c in containers + ctx, + c.name, + packages, + tarfile_path, + ): c + for c in containers } logging.debug(futures_to_containers) @@ -130,41 +145,63 @@ def install_packages(self, ctx, package_directory, containers, package_name_list try: ec = f.result() if ec != 0: - logging.error('error while installing packages on container [{}]' - .format(container.name)) + logging.error( + "error while installing packages on container [{}]".format( + container.name + ) + ) rc = ec else: - logging.info('packages installed successfully [{}]' - .format(container.name)) + logging.info( + "packages installed successfully [{}]".format( + container.name + ) + ) except Exception as e: - logging.error('exception raised while installing packages [{}]' - .format(container.name)) + logging.error( + "exception raised while installing packages [{}]".format( + container.name + ) + ) logging.error(e) rc = 1 return rc - def install_official_irods_packages(self, ctx, version, containers): def install_packages_(ctx, docker_compose_container, packages_list): container = ctx.docker_client.containers.get(docker_compose_container.name) - package_list = ' '.join([p for p in packages_list if not context.is_database_plugin(p) or context.is_irods_catalog_provider_container(container)]) + package_list = " ".join( + [ + p + for p in packages_list + if not context.is_database_plugin(p) + or context.is_irods_catalog_provider_container(container) + ] + ) - cmd = ' '.join([self.install_official_packages_command(), package_list]) + cmd = " ".join([self.install_official_packages_command(), package_list]) - logging.warning('executing cmd [{0}] on container [{1}]'.format(cmd, container.name)) + logging.warning( + "executing cmd [{0}] on container [{1}]".format(cmd, container.name) + ) ec = execute.execute_command(container, self.update_command()) if ec != 0: - logging.error('failed to update local repositories [{}]'.format(container.name)) + logging.error( + "failed to update local repositories [{}]".format(container.name) + ) return ec ec = execute.execute_command(container, cmd) if ec != 0: logging.error( - 'failed to install packages on container [ec=[{0}], container=[{1}]'.format(ec, container.name)) + "failed to install packages on container [ec=[{0}], container=[{1}]".format( + ec, container.name + ) + ) return ec @@ -174,16 +211,21 @@ def install_packages_(ctx, docker_compose_container, packages_list): # If a version is not provided, just install the latest if version: - packages = ['{}{}{}'.format(p, self.version_joinery(), version) - for p in context.irods_package_names(ctx.database_name())] + packages = [ + "{}{}{}".format(p, self.version_joinery(), version) + for p in context.irods_package_names(ctx.database_name()) + ] else: packages = context.irods_package_names(ctx.database_name()) - logging.info('packages to install [{}]'.format(packages)) + logging.info("packages to install [{}]".format(packages)) rc = 0 with concurrent.futures.ThreadPoolExecutor() as executor: - futures_to_containers = {executor.submit(install_packages_, ctx, c, packages): c for c in containers} + futures_to_containers = { + executor.submit(install_packages_, ctx, c, packages): c + for c in containers + } logging.debug(futures_to_containers) for f in concurrent.futures.as_completed(futures_to_containers): @@ -191,24 +233,35 @@ def install_packages_(ctx, docker_compose_container, packages_list): try: ec = f.result() if ec != 0: - logging.error('error while installing packages on container [{}]'.format(container.name)) + logging.error( + "error while installing packages on container [{}]".format( + container.name + ) + ) rc = ec - logging.info('packages installed successfully [{}]'.format(container.name)) + logging.info( + "packages installed successfully [{}]".format(container.name) + ) except Exception as e: - logging.error('exception raised while installing packages [{}]'.format(container.name)) + logging.error( + "exception raised while installing packages [{}]".format( + container.name + ) + ) logging.error(e) rc = 1 return rc - - def install_irods_packages(self, - ctx, - externals_directory=None, - package_directory=None, - package_version=None): + def install_irods_packages( + self, + ctx, + externals_directory=None, + package_directory=None, + package_version=None, + ): """Install iRODS packages and external dependencies. `package_directory` and `package_version` cannot both be specified. @@ -226,47 +279,56 @@ def install_irods_packages(self, install (if None, the latest available version is used) """ if package_directory and package_version: - raise ValueError('package_directory and package_version are incompatible') + raise ValueError("package_directory and package_version are incompatible") if externals_directory: - ec = self.install_packages(ctx, - os.path.abspath(externals_directory), - ctx.irods_containers(), - context.irods_externals_package_names()) + ec = self.install_packages( + ctx, + os.path.abspath(externals_directory), + ctx.irods_containers(), + context.irods_externals_package_names(), + ) if ec != 0: - raise RuntimeError('failed to install externals') + raise RuntimeError("failed to install externals") if package_directory: - logging.warning('installing iRODS packages from directory [{}]' - .format(package_directory)) - - ec = self.install_packages(ctx, - os.path.abspath(package_directory), - ctx.irods_containers(), - context.irods_package_names(ctx.database_name())) + logging.warning( + "installing iRODS packages from directory [{}]".format( + package_directory + ) + ) + + ec = self.install_packages( + ctx, + os.path.abspath(package_directory), + ctx.irods_containers(), + context.irods_package_names(ctx.database_name()), + ) if ec != 0: - raise RuntimeError('failed to install iRODS packages') + raise RuntimeError("failed to install iRODS packages") else: # Even if no version was provided, we default to using the latest official release - logging.warning('installing official iRODS packages [{}]' - .format(package_version)) + logging.warning( + "installing official iRODS packages [{}]".format(package_version) + ) - ec = self.install_official_irods_packages(ctx, package_version, ctx.irods_containers()) + ec = self.install_official_irods_packages( + ctx, package_version, ctx.irods_containers() + ) if ec != 0: - raise RuntimeError('failed to install iRODS packages') + raise RuntimeError("failed to install iRODS packages") def make_installer(platform_name): - name = '_'.join([platform_name, 'installer']) + name = "_".join([platform_name, "installer"]) - return eval('.'.join([name, name]))() + return eval(".".join([name, name]))() -def install_pip_package_from_repo(container, - repo_name, - url_base='https://github.com/irods', - branch=None): +def install_pip_package_from_repo( + container, repo_name, url_base="https://github.com/irods", branch=None +): """Installs a pip package from a git repository cloned from the specified location. Arguments: @@ -276,13 +338,14 @@ def install_pip_package_from_repo(container, """ from .. import services - repo_path = services.clone_repository_to_container(container, - repo_name, - url_base=url_base, - branch=branch) - ec = execute.execute_command(container, ' '.join( - [container_info.python(container), - '-m', 'pip', 'install', repo_path])) + repo_path = services.clone_repository_to_container( + container, repo_name, url_base=url_base, branch=branch + ) + ec = execute.execute_command( + container, + " ".join([container_info.python(container), "-m", "pip", "install", repo_path]), + ) if ec != 0: - raise RuntimeError('Failed to install pip package [{}] [{}]' - .format(repo_path, container.name)) + raise RuntimeError( + "Failed to install pip package [{}] [{}]".format(repo_path, container.name) + ) diff --git a/irods_testing_environment/install/rockylinux_installer.py b/irods_testing_environment/install/rockylinux_installer.py index 5c857df..0394712 100644 --- a/irods_testing_environment/install/rockylinux_installer.py +++ b/irods_testing_environment/install/rockylinux_installer.py @@ -1,22 +1,18 @@ from . import install + class rockylinux_installer(install.installer): def update_command(self): - return 'dnf update -y' - + return "dnf update -y" def install_local_packages_command(self): - return 'dnf --nogpgcheck -y install' - + return "dnf --nogpgcheck -y install" def install_official_packages_command(self): - return 'dnf -y install' - + return "dnf -y install" def filename_extension(self): - return 'rpm' - + return "rpm" def version_joinery(self): - return '-' - + return "-" diff --git a/irods_testing_environment/install/ubuntu_installer.py b/irods_testing_environment/install/ubuntu_installer.py index 76d5fd9..8e0f21a 100644 --- a/irods_testing_environment/install/ubuntu_installer.py +++ b/irods_testing_environment/install/ubuntu_installer.py @@ -1,22 +1,18 @@ from . import install + class ubuntu_installer(install.installer): def update_command(self): - return 'apt update' - + return "apt update" def install_local_packages_command(self): - return 'apt install -fy' - + return "apt install -fy" def install_official_packages_command(self): - return 'apt install -y' - + return "apt install -y" def filename_extension(self): - return 'deb' - + return "deb" def version_joinery(self): - return '=' - + return "=" diff --git a/irods_testing_environment/irods_config.py b/irods_testing_environment/irods_config.py index 0980d3e..02d1e7e 100644 --- a/irods_testing_environment/irods_config.py +++ b/irods_testing_environment/irods_config.py @@ -23,6 +23,7 @@ # is declared here to extend the lifetime of the dict. irods_commit_id = dict() + def get_irods_zone_name(container): """Return the Zone name of the iRODS server running on `container`.""" global irods_zone @@ -32,7 +33,8 @@ def get_irods_zone_name(container): return irods_zone[container.name] irods_zone[container.name] = json_utils.get_json_from_file( - container, context.server_config())['zone_name'] + container, context.server_config() + )["zone_name"] return irods_zone[container.name] @@ -50,7 +52,7 @@ def get_irods_version(container): return irods_version[container.name] irods_version[container.name] = tuple( - int(i) for i in get_irods_version_info(container, 'irods_version').split('.') + int(i) for i in get_irods_version_info(container, "irods_version").split(".") ) return irods_version[container.name] @@ -68,7 +70,7 @@ def get_irods_commit_id(container): if container.name in irods_commit_id: return irods_commit_id[container.name] - irods_commit_id[container.name] = get_irods_version_info(container, 'commit_id') + irods_commit_id[container.name] = get_irods_version_info(container, "commit_id") return irods_commit_id[container.name] @@ -83,24 +85,24 @@ def get_irods_version_info(container, version_file_key): # The name of the version file changed in iRODS version 4.3.0. The testing environment supports # both 4.3.x and 4.2.x versions, so we want to check for both file names. version_file_locations = [ - os.path.join(context.irods_home(), 'version.json.dist'), - os.path.join(context.irods_home(), 'VERSION.json.dist') + os.path.join(context.irods_home(), "version.json.dist"), + os.path.join(context.irods_home(), "VERSION.json.dist"), ] for f in version_file_locations: # Check to see whether the file exists. If not, try the next one. if execute.execute_command(container, f'bash -c "[[ -f {f} ]]"') != 0: - logging.debug(f'[{container.name}]: version file [{f}] not found') + logging.debug(f"[{container.name}]: version file [{f}] not found") continue - logging.debug(f'[{container.name}]: version file [{f}] found') + logging.debug(f"[{container.name}]: version file [{f}] found") # If the file exists, extract the version string and store it in the version # dictionary under this container's name. return json_utils.get_json_from_file(container, f)[version_file_key] # If we reach here, that's no good. - raise RuntimeError(f'[{container.name}]: No iRODS version file found') + raise RuntimeError(f"[{container.name}]: No iRODS version file found") def configure_users_for_auth_tests(docker_client, compose_project): @@ -111,42 +113,51 @@ def configure_users_for_auth_tests(docker_client, compose_project): compose_project -- compose.Project in which the iRODS servers are running usernames_and_passwords -- a list of tuples of usernames/passwords (passwords can be empty) """ - def create_test_users(docker_client, docker_compose_container, usernames_and_passwords): + + def create_test_users( + docker_client, docker_compose_container, usernames_and_passwords + ): container = docker_client.containers.get(docker_compose_container.name) for username, password in usernames_and_passwords: - create_user = f'useradd {username}' + create_user = f"useradd {username}" if execute.execute_command(container, create_user) != 0: - raise RuntimeError(f'[{container.name}] failed to create user [{username}]') + raise RuntimeError( + f"[{container.name}] failed to create user [{username}]" + ) - if password is None or password == '': + if password is None or password == "": continue - set_password = f'bash -c "echo \'{username}:{password}\' | chpasswd"' + set_password = f"bash -c \"echo '{username}:{password}' | chpasswd\"" if execute.execute_command(container, set_password) != 0: - raise RuntimeError(f'[{container.name}] failed to create hosts_config file') + raise RuntimeError( + f"[{container.name}] failed to create hosts_config file" + ) return 0 import concurrent.futures - containers = compose_project.containers(service_names=[ - context.irods_catalog_provider_service(), - context.irods_catalog_consumer_service()]) + containers = compose_project.containers( + service_names=[ + context.irods_catalog_provider_service(), + context.irods_catalog_consumer_service(), + ] + ) rc = 0 with concurrent.futures.ThreadPoolExecutor() as executor: # TODO: get these names from the test file packaged with the server - usernames_and_passwords = [ - ('irodsauthuser', ';=iamnotasecret') - ] + usernames_and_passwords = [("irodsauthuser", ";=iamnotasecret")] futures_to_containers = { executor.submit( create_test_users, docker_client, c, usernames_and_passwords - ): c for c in containers + ): c + for c in containers } logging.debug(futures_to_containers) @@ -156,18 +167,24 @@ def create_test_users(docker_client, docker_compose_container, usernames_and_pas try: ec = f.result() if ec != 0: - logging.error(f'[{container.name}] error while creating test user accounts') + logging.error( + f"[{container.name}] error while creating test user accounts" + ) rc = ec else: - logging.info(f'[{container.name}] successfully created test user accounts') + logging.info( + f"[{container.name}] successfully created test user accounts" + ) except Exception as e: - logging.error(f'[{container.name}] exception raised while creating test users') + logging.error( + f"[{container.name}] exception raised while creating test users" + ) logging.error(e) rc = 1 if rc != 0: - raise RuntimeError('failed to create test user accounts on some service') + raise RuntimeError("failed to create test user accounts on some service") def configure_hosts_config(docker_client, compose_project): @@ -177,24 +194,26 @@ def configure_hosts_config(docker_client, compose_project): docker_client -- docker client for interacting with the docker-compose project compose_project -- compose.Project in which the iRODS servers are running """ + def set_hostnames(docker_client, docker_compose_container, hosts_file): container = docker_client.containers.get(docker_compose_container.name) if context.is_irods_catalog_provider_container(container): - alias = 'icat.example.org' + alias = "icat.example.org" else: - alias = 'resource{}.example.org'.format( - context.service_instance(docker_compose_container.name)) + alias = "resource{}.example.org".format( + context.service_instance(docker_compose_container.name) + ) hosts = { - 'host_entries': [ + "host_entries": [ { - 'address_type': 'local', - 'addresses': [ - {'address': context.container_hostname(container)}, - {'address': context.container_ip(container)}, - {'address': alias} - ] + "address_type": "local", + "addresses": [ + {"address": context.container_hostname(container)}, + {"address": context.container_ip(container)}, + {"address": alias}, + ], } ] } @@ -206,43 +225,53 @@ def set_hostnames(docker_client, docker_compose_container, hosts_file): other = docker_client.containers.get(o.name) if context.is_irods_catalog_provider_container(other): - remote_address = 'icat.example.org' + remote_address = "icat.example.org" else: - remote_address = 'resource{}.example.org'.format( - context.service_instance(other.name)) + remote_address = "resource{}.example.org".format( + context.service_instance(other.name) + ) - hosts['host_entries'].append( + hosts["host_entries"].append( { - 'address_type': 'remote', - 'addresses': [ - {'address': context.container_ip(other)}, - {'address': context.container_hostname(other)}, - {'address': remote_address} - ] + "address_type": "remote", + "addresses": [ + {"address": context.container_ip(other)}, + {"address": context.container_hostname(other)}, + {"address": remote_address}, + ], } ) - logging.info('json for hosts_config [{}] [{}]'.format(json.dumps(hosts), container.name)) + logging.info( + "json for hosts_config [{}] [{}]".format(json.dumps(hosts), container.name) + ) - create_hosts_config = 'bash -c \'echo "{}" > {}\''.format( - json.dumps(hosts).replace('"', '\\"'), hosts_file) + create_hosts_config = "bash -c 'echo \"{}\" > {}'".format( + json.dumps(hosts).replace('"', '\\"'), hosts_file + ) if execute.execute_command(container, create_hosts_config) != 0: - raise RuntimeError('failed to create hosts_config file [{}]'.format(container.name)) + raise RuntimeError( + "failed to create hosts_config file [{}]".format(container.name) + ) return 0 import concurrent.futures - containers = compose_project.containers(service_names=[ - context.irods_catalog_provider_service(), - context.irods_catalog_consumer_service()]) + containers = compose_project.containers( + service_names=[ + context.irods_catalog_provider_service(), + context.irods_catalog_consumer_service(), + ] + ) rc = 0 with concurrent.futures.ThreadPoolExecutor() as executor: - hosts_file = os.path.join('/etc', 'irods', 'hosts_config.json') + hosts_file = os.path.join("/etc", "irods", "hosts_config.json") futures_to_containers = { - executor.submit(set_hostnames, docker_client, c, hosts_file): c for c in containers + executor.submit(set_hostnames, docker_client, c, hosts_file): c + for c in containers } logging.debug(futures_to_containers) @@ -251,21 +280,30 @@ def set_hostnames(docker_client, docker_compose_container, hosts_file): try: ec = f.result() if ec != 0: - logging.error('error while configuring hosts_configs.json on container [{}]' - .format(container.name)) + logging.error( + "error while configuring hosts_configs.json on container [{}]".format( + container.name + ) + ) rc = ec else: - logging.info('hosts_config.json configured successfully [{}]' - .format(container.name)) + logging.info( + "hosts_config.json configured successfully [{}]".format( + container.name + ) + ) except Exception as e: - logging.error('exception raised while installing packages [{}]' - .format(container.name)) + logging.error( + "exception raised while installing packages [{}]".format( + container.name + ) + ) logging.error(e) rc = 1 if rc != 0: - raise RuntimeError('failed to configure hosts_config.json on some service') + raise RuntimeError("failed to configure hosts_config.json on some service") def configure_univmss_script(docker_client, compose_project): @@ -275,55 +313,86 @@ def configure_univmss_script(docker_client, compose_project): docker_client -- docker client for interacting with the docker-compose project compose_project -- compose.Project in which the iRODS servers are running """ + def modify_script(docker_client, docker_compose_container, script): - chown_msiexec = 'chown irods:irods {}'.format(os.path.dirname(script)) - copy_from_template = 'cp {0}.template {0}'.format(script) - remove_template_from_commands = 'sed -i \"s/template-//g\" {}'.format(script) - make_script_executable = 'chmod 544 {}'.format(script) + chown_msiexec = "chown irods:irods {}".format(os.path.dirname(script)) + copy_from_template = "cp {0}.template {0}".format(script) + remove_template_from_commands = 'sed -i "s/template-//g" {}'.format(script) + make_script_executable = "chmod 544 {}".format(script) on_container = docker_client.containers.get(docker_compose_container.name) if execute.execute_command(on_container, chown_msiexec) != 0: - raise RuntimeError('failed to change ownership to msiExecCmd_bin [{}]' - .format(on_container.name)) - - if execute.execute_command(on_container, - copy_from_template, - user='irods', - workdir=context.irods_home()) != 0: - raise RuntimeError('failed to copy univMSSInterface.sh template file [{}]' - .format(on_container.name)) - - if execute.execute_command(on_container, - remove_template_from_commands, - user='irods', - workdir=context.irods_home()) != 0: - raise RuntimeError('failed to modify univMSSInterface.sh template file [{}]' - .format(on_container.name)) - - if execute.execute_command(on_container, - make_script_executable, - user='irods', - workdir=context.irods_home()) != 0: - raise RuntimeError('failed to change permissions on univMSSInterface.sh [{}]' - .format(on_container.name)) + raise RuntimeError( + "failed to change ownership to msiExecCmd_bin [{}]".format( + on_container.name + ) + ) + + if ( + execute.execute_command( + on_container, + copy_from_template, + user="irods", + workdir=context.irods_home(), + ) + != 0 + ): + raise RuntimeError( + "failed to copy univMSSInterface.sh template file [{}]".format( + on_container.name + ) + ) + + if ( + execute.execute_command( + on_container, + remove_template_from_commands, + user="irods", + workdir=context.irods_home(), + ) + != 0 + ): + raise RuntimeError( + "failed to modify univMSSInterface.sh template file [{}]".format( + on_container.name + ) + ) + + if ( + execute.execute_command( + on_container, + make_script_executable, + user="irods", + workdir=context.irods_home(), + ) + != 0 + ): + raise RuntimeError( + "failed to change permissions on univMSSInterface.sh [{}]".format( + on_container.name + ) + ) return 0 import concurrent.futures - containers = compose_project.containers(service_names=[ - context.irods_catalog_provider_service(), - context.irods_catalog_consumer_service()]) + containers = compose_project.containers( + service_names=[ + context.irods_catalog_provider_service(), + context.irods_catalog_consumer_service(), + ] + ) rc = 0 with concurrent.futures.ThreadPoolExecutor() as executor: univmss_script = os.path.join( - context.irods_home(), 'msiExecCmd_bin', 'univMSSInterface.sh') + context.irods_home(), "msiExecCmd_bin", "univMSSInterface.sh" + ) futures_to_containers = { - executor.submit( - modify_script, docker_client, c, univmss_script - ): c for c in containers + executor.submit(modify_script, docker_client, c, univmss_script): c + for c in containers } logging.debug(futures_to_containers) @@ -333,19 +402,30 @@ def modify_script(docker_client, docker_compose_container, script): try: ec = f.result() if ec != 0: - logging.error('error while configuring univMSS script on container [{}]' - .format(container.name)) + logging.error( + "error while configuring univMSS script on container [{}]".format( + container.name + ) + ) rc = ec else: - logging.info('univMSS script configured successfully [{}]'.format(container.name)) + logging.info( + "univMSS script configured successfully [{}]".format( + container.name + ) + ) except Exception as e: - logging.error('exception raised while configuring univMSS script [{}]'.format(container.name)) + logging.error( + "exception raised while configuring univMSS script [{}]".format( + container.name + ) + ) logging.error(e) rc = 1 if rc != 0: - raise RuntimeError('failed to configure univMSS script on some service') + raise RuntimeError("failed to configure univMSS script on some service") def configure_irods_testing(docker_client, compose_project): @@ -375,70 +455,99 @@ def configure_irods_federation_testing(ctx, remote_zone, zone_where_tests_will_r container = ctx.docker_client.containers.get( context.irods_catalog_provider_container( ctx.compose_project.name, - service_instance=remote_zone.provider_service_instance + service_instance=remote_zone.provider_service_instance, ) ) - execute.execute_command(container, 'iadmin lu', user='irods') - execute.execute_command(container, 'iadmin lz', user='irods') + execute.execute_command(container, "iadmin lu", user="irods") + execute.execute_command(container, "iadmin lz", user="irods") # create zonehopper# user - username = '#'.join(['zonehopper', zone_where_tests_will_run.zone_name]) - mkuser = 'iadmin mkuser {} rodsuser'.format(username) - logging.info('creating user [{}] in container [{}]'.format(username, container.name)) - if execute.execute_command(container, mkuser, user='irods') != 0: - raise RuntimeError('failed to create remote user [{}] [{}]' - .format(username, container.name)) + username = "#".join(["zonehopper", zone_where_tests_will_run.zone_name]) + mkuser = "iadmin mkuser {} rodsuser".format(username) + logging.info( + "creating user [{}] in container [{}]".format(username, container.name) + ) + if execute.execute_command(container, mkuser, user="irods") != 0: + raise RuntimeError( + "failed to create remote user [{}] [{}]".format(username, container.name) + ) # create zonehopper# user - username = '#'.join(['zonehopper', remote_zone.zone_name]) - mkuser = 'iadmin mkuser {} rodsuser'.format(username) - logging.info('creating user [{}] in container [{}]'.format(username, container.name)) - if execute.execute_command(container, mkuser, user='irods') != 0: - raise RuntimeError('failed to create remote user [{}] [{}]' - .format(username, container.name)) + username = "#".join(["zonehopper", remote_zone.zone_name]) + mkuser = "iadmin mkuser {} rodsuser".format(username) + logging.info( + "creating user [{}] in container [{}]".format(username, container.name) + ) + if execute.execute_command(container, mkuser, user="irods") != 0: + raise RuntimeError( + "failed to create remote user [{}] [{}]".format(username, container.name) + ) # set password of zonehopper# user - moduser = 'iadmin moduser {} password 53CR37'.format(username) - logging.info('setting password of user [{}] in container [{}]'.format(username, container.name)) - if execute.execute_command(container, moduser, user='irods') != 0: - raise RuntimeError('failed to set password for remote user [{}] [{}]' - .format(username, container.name)) + moduser = "iadmin moduser {} password 53CR37".format(username) + logging.info( + "setting password of user [{}] in container [{}]".format( + username, container.name + ) + ) + if execute.execute_command(container, moduser, user="irods") != 0: + raise RuntimeError( + "failed to set password for remote user [{}] [{}]".format( + username, container.name + ) + ) - execute.execute_command(container, 'iadmin lu', user='irods') + execute.execute_command(container, "iadmin lu", user="irods") # create passthrough resource - ptname = 'federation_remote_passthrough' - make_pt = 'iadmin mkresc {} passthru'.format(ptname) - logging.info('creating passthrough resource [{}] [{}]'.format(make_pt, container.name)) - if execute.execute_command(container, make_pt, user='irods') != 0: - raise RuntimeError('failed to create passthrough resource [{}] [{}]' - .format(ptname, container.name)) + ptname = "federation_remote_passthrough" + make_pt = "iadmin mkresc {} passthru".format(ptname) + logging.info( + "creating passthrough resource [{}] [{}]".format(make_pt, container.name) + ) + if execute.execute_command(container, make_pt, user="irods") != 0: + raise RuntimeError( + "failed to create passthrough resource [{}] [{}]".format( + ptname, container.name + ) + ) # create the storage resource - ufsname = 'federation_remote_unixfilesystem_leaf' - make_ufs = 'iadmin mkresc {} unixfilesystem {}:{}'.format( - ufsname, context.container_hostname(container), - os.path.join('/tmp', ufsname)) - logging.info('creating unixfilesystem resource [{}] [{}]'.format(make_ufs, container.name)) - if execute.execute_command(container, make_ufs, user='irods') != 0: - raise RuntimeError('failed to create unixfilesystem resource [{}] [{}]' - .format(ufsname, container.name)) + ufsname = "federation_remote_unixfilesystem_leaf" + make_ufs = "iadmin mkresc {} unixfilesystem {}:{}".format( + ufsname, context.container_hostname(container), os.path.join("/tmp", ufsname) + ) + logging.info( + "creating unixfilesystem resource [{}] [{}]".format(make_ufs, container.name) + ) + if execute.execute_command(container, make_ufs, user="irods") != 0: + raise RuntimeError( + "failed to create unixfilesystem resource [{}] [{}]".format( + ufsname, container.name + ) + ) # make the hierarchy - make_hier = 'iadmin addchildtoresc {} {}'.format(ptname, ufsname) - logging.info('creating hierarchy [{}] [{}]'.format(make_hier, container.name)) - if execute.execute_command(container, make_hier, user='irods') != 0: - raise RuntimeError('failed to create hierarchy [{};{}] [{}]' - .format(ptname, ufsname, container.name)) + make_hier = "iadmin addchildtoresc {} {}".format(ptname, ufsname) + logging.info("creating hierarchy [{}] [{}]".format(make_hier, container.name)) + if execute.execute_command(container, make_hier, user="irods") != 0: + raise RuntimeError( + "failed to create hierarchy [{};{}] [{}]".format( + ptname, ufsname, container.name + ) + ) # add specific query to the local zone - bug_3466_query = 'select alias, sqlStr from R_SPECIFIC_QUERY' - asq = 'iadmin asq \'{}\' {}'.format(bug_3466_query, 'bug_3466_query') - logging.info('creating specific query[{}] [{}]'.format(asq, container.name)) - if execute.execute_command(container, asq, user='irods') != 0: - raise RuntimeError('failed to create specific query [{}] [{}]' - .format(bug_3466_query, container.name)) + bug_3466_query = "select alias, sqlStr from R_SPECIFIC_QUERY" + asq = "iadmin asq '{}' {}".format(bug_3466_query, "bug_3466_query") + logging.info("creating specific query[{}] [{}]".format(asq, container.name)) + if execute.execute_command(container, asq, user="irods") != 0: + raise RuntimeError( + "failed to create specific query [{}] [{}]".format( + bug_3466_query, container.name + ) + ) def configure_pam_for_auth_plugin(docker_client, compose_project): @@ -453,7 +562,9 @@ def configure_pam_for_auth_plugin(docker_client, compose_project): import concurrent.futures import textwrap - def configure_pam(docker_client, docker_compose_container, path_to_config, contents): + def configure_pam( + docker_client, docker_compose_container, path_to_config, contents + ): container = docker_client.containers.get(docker_compose_container.name) archive.put_string_to_file(container, path_to_config, contents) @@ -462,25 +573,31 @@ def configure_pam(docker_client, docker_compose_container, path_to_config, conte return 0 - path_to_config = os.path.join('/etc', 'pam.d', 'irods') + path_to_config = os.path.join("/etc", "pam.d", "irods") - contents = textwrap.dedent('''\ + contents = textwrap.dedent( + """\ auth required pam_env.so auth sufficient pam_unix.so auth requisite pam_succeed_if.so uid >= 500 quiet auth required pam_deny.so - ''') + """ + ) - containers = compose_project.containers(service_names=[ - context.irods_catalog_provider_service(), - context.irods_catalog_consumer_service()]) + containers = compose_project.containers( + service_names=[ + context.irods_catalog_provider_service(), + context.irods_catalog_consumer_service(), + ] + ) rc = 0 with concurrent.futures.ThreadPoolExecutor() as executor: futures_to_containers = { executor.submit( configure_pam, docker_client, c, path_to_config, contents - ): c for c in containers + ): c + for c in containers } for f in concurrent.futures.as_completed(futures_to_containers): @@ -488,16 +605,17 @@ def configure_pam(docker_client, docker_compose_container, path_to_config, conte try: ec = f.result() if ec != 0: - logging.error(f'[{container.name}] error configuring pam') + logging.error(f"[{container.name}] error configuring pam") rc = ec else: - logging.info(f'[{container.name}] successfully configured pam') + logging.info(f"[{container.name}] successfully configured pam") except Exception as e: - logging.error(f'[{container.name}] exception raised while configuring pam') + logging.error( + f"[{container.name}] exception raised while configuring pam" + ) logging.error(e) rc = 1 if rc != 0: - raise RuntimeError('failed to configure pam on some service') - + raise RuntimeError("failed to configure pam on some service") diff --git a/irods_testing_environment/irods_setup.py b/irods_testing_environment/irods_setup.py index 47c956d..3d672d4 100644 --- a/irods_testing_environment/irods_setup.py +++ b/irods_testing_environment/irods_setup.py @@ -9,16 +9,20 @@ from . import execute from . import irods_config + class zone_info(object): """Class to hold information about an iRODS Zone and the containers running the servers.""" - def __init__(self, - zone_name='tempZone', - zone_key='TEMPORARY_ZONE_KEY', - negotiation_key='32_byte_server_negotiation_key__', - zone_port=1247, - database_service_instance=1, - provider_service_instance=1, - consumer_service_instances=None): + + def __init__( + self, + zone_name="tempZone", + zone_key="TEMPORARY_ZONE_KEY", + negotiation_key="32_byte_server_negotiation_key__", + zone_port=1247, + database_service_instance=1, + provider_service_instance=1, + consumer_service_instances=None, + ): """Construct a zone_info object. Arguments: @@ -43,46 +47,43 @@ def __init__(self, self.provider_service_instance = provider_service_instance self.consumer_service_instances = consumer_service_instances - def provider_container(self, ctx): """Return Docker Container running the iRODS CSP.""" return ctx.docker_client.containers.get( context.irods_catalog_provider_container( ctx.compose_project.name, - service_instance=self.provider_service_instance) + service_instance=self.provider_service_instance, + ) ) - def consumer_container(self, ctx, instance): """Return Docker Container running an iRODS CSC with specified instance.""" return ctx.docker_client.containers.get( context.irods_catalog_consumer_container( - ctx.compose_project.name, - service_instance=instance) + ctx.compose_project.name, service_instance=instance + ) ) - def consumer_containers(self, ctx): """Return list of Docker Containers running the iRODS CSCs.""" - return [self.consumer_container(ctx, i) for i in self.consumer_service_instances] - + return [ + self.consumer_container(ctx, i) for i in self.consumer_service_instances + ] def provider_hostname(self, ctx): """Return hostname for the container running the iRODS CSP.""" return context.container_hostname(self.provider_container(ctx)) - def consumer_hostname(self, ctx, instance): """Return hostname for the container running an iRODS CSC with specified instance.""" return context.container_hostname( ctx.docker_client.containers.get( context.irods_catalog_provider_container( - ctx.compose_project.name, - service_instance=instance) + ctx.compose_project.name, service_instance=instance + ) ) ) - def consumer_hostnames(self, ctx): """Return list of hostnames for the containers running the iRODS CSCs.""" return [self.consumer_hostname(ctx, i) for i in self.consumer_service_instances] @@ -95,6 +96,7 @@ class setup_input_builder(object): To that end, each section of the setup script is its own method which sets the values to generate the input string. """ + def __init__(self): """Construct a setup input builder. @@ -102,64 +104,66 @@ def __init__(self): """ self.irods_version = None - self.service_account_name = '' - self.service_account_group = '' - self.catalog_service_role = '' + self.service_account_name = "" + self.service_account_group = "" + self.catalog_service_role = "" - self.odbc_driver = '' - self.database_server_hostname = 'localhost' + self.odbc_driver = "" + self.database_server_hostname = "localhost" self.database_server_port = 5432 - self.database_name = 'ICAT' - self.database_username = 'irods' - self.database_password = 'testpassword' - self.stored_passwords_salt = '' + self.database_name = "ICAT" + self.database_username = "irods" + self.database_password = "testpassword" + self.stored_passwords_salt = "" - self.zone_name = 'tempZone' + self.zone_name = "tempZone" self.zone_port = 1247 self.parallel_port_range_begin = 20000 self.parallel_port_range_end = 20199 self.control_plane_port = 1248 - self.schema_validation_base_uri = '' - self.admin_username = 'rods' - - self.zone_key = 'TEMPORARY_ZONE_KEY' - self.negotiation_key = '32_byte_server_negotiation_key__' - self.control_plane_key = '32_byte_server_control_plane_key' - self.admin_password = 'rods' - - self.provides_local_storage = 'y' - self.resource_name = '' - self.vault_directory = '' - - self.catalog_service_provider_host = 'localhost' - - def setup(self, - irods_version, - service_account_name= None, - service_account_group= None, - catalog_service_role= None, - odbc_driver= None, - database_server_hostname= None, - database_server_port= None, - database_name= None, - database_username= None, - database_password= None, - stored_passwords_salt= None, - zone_name= None, - catalog_service_provider_host= None, - zone_port= None, - parallel_port_range_begin= None, - parallel_port_range_end= None, - control_plane_port= None, - schema_validation_base_uri= None, - admin_username= None, - zone_key = None, - negotiation_key = None, - control_plane_key = None, - admin_password = None, - provides_local_storage = None, - resource_name = None, - vault_directory = None): + self.schema_validation_base_uri = "" + self.admin_username = "rods" + + self.zone_key = "TEMPORARY_ZONE_KEY" + self.negotiation_key = "32_byte_server_negotiation_key__" + self.control_plane_key = "32_byte_server_control_plane_key" + self.admin_password = "rods" + + self.provides_local_storage = "y" + self.resource_name = "" + self.vault_directory = "" + + self.catalog_service_provider_host = "localhost" + + def setup( + self, + irods_version, + service_account_name=None, + service_account_group=None, + catalog_service_role=None, + odbc_driver=None, + database_server_hostname=None, + database_server_port=None, + database_name=None, + database_username=None, + database_password=None, + stored_passwords_salt=None, + zone_name=None, + catalog_service_provider_host=None, + zone_port=None, + parallel_port_range_begin=None, + parallel_port_range_end=None, + control_plane_port=None, + schema_validation_base_uri=None, + admin_username=None, + zone_key=None, + negotiation_key=None, + control_plane_key=None, + admin_password=None, + provides_local_storage=None, + resource_name=None, + vault_directory=None, + ): """Set values for the service account section of the setup script. Returns this instance of the class. @@ -204,7 +208,9 @@ def setup(self, self.catalog_service_role = catalog_service_role or self.catalog_service_role self.odbc_driver = odbc_driver or self.odbc_driver - self.database_server_hostname = database_server_hostname or self.database_server_hostname + self.database_server_hostname = ( + database_server_hostname or self.database_server_hostname + ) self.database_server_port = database_server_port or self.database_server_port self.database_name = database_name or self.database_name self.database_username = database_username or self.database_username @@ -212,12 +218,20 @@ def setup(self, self.stored_passwords_salt = stored_passwords_salt or self.stored_passwords_salt self.zone_name = zone_name or self.zone_name - self.catalog_service_provider_host = catalog_service_provider_host or self.catalog_service_provider_host + self.catalog_service_provider_host = ( + catalog_service_provider_host or self.catalog_service_provider_host + ) self.zone_port = zone_port or self.zone_port - self.parallel_port_range_begin = parallel_port_range_begin or self.parallel_port_range_begin - self.parallel_port_range_end = parallel_port_range_end or self.parallel_port_range_end + self.parallel_port_range_begin = ( + parallel_port_range_begin or self.parallel_port_range_begin + ) + self.parallel_port_range_end = ( + parallel_port_range_end or self.parallel_port_range_end + ) self.control_plane_port = control_plane_port or self.control_plane_port - self.schema_validation_base_uri = schema_validation_base_uri or self.schema_validation_base_uri + self.schema_validation_base_uri = ( + schema_validation_base_uri or self.schema_validation_base_uri + ) self.admin_username = admin_username or self.admin_username self.zone_key = zone_key or self.zone_key @@ -225,13 +239,14 @@ def setup(self, self.control_plane_key = control_plane_key or self.control_plane_key self.admin_password = admin_password or self.admin_password - self.provides_local_storage = provides_local_storage or self.provides_local_storage + self.provides_local_storage = ( + provides_local_storage or self.provides_local_storage + ) self.resource_name = resource_name or self.resource_name self.vault_directory = vault_directory or self.vault_directory return self - def build_input_for_catalog_consumer(self): """Generate string to use as input for the setup script. @@ -244,7 +259,6 @@ def build_input_for_catalog_consumer(self): str(self.service_account_name), str(self.service_account_group), str(role), - str(self.zone_name), str(self.catalog_service_provider_host), str(self.zone_port), @@ -253,13 +267,12 @@ def build_input_for_catalog_consumer(self): str(self.control_plane_port), str(self.schema_validation_base_uri), str(self.admin_username), - 'y', # confirmation of inputs - + "y", # confirmation of inputs str(self.zone_key), str(self.negotiation_key), str(self.control_plane_key), str(self.admin_password), - '' # confirmation of inputs + "", # confirmation of inputs ] # Handle the difference between 4.2 servers and 4.3 servers. @@ -269,9 +282,9 @@ def build_input_for_catalog_consumer(self): input_args.insert(5, str(self.vault_directory)) else: input_args.append(str(self.vault_directory)) - input_args.append(str('')) # final confirmation + input_args.append(str("")) # final confirmation - return '\n'.join(input_args) + return "\n".join(input_args) def build_input_for_catalog_provider(self): """Generate string to use as input for the setup script. @@ -279,21 +292,19 @@ def build_input_for_catalog_provider(self): The script changes depending on the role, so the options used here are specific to setting up an iRODS catalog service provider. """ - role = '' + role = "" input_args = [ str(self.service_account_name), str(self.service_account_group), str(role), - str(self.odbc_driver), str(self.database_server_hostname), str(self.database_server_port), str(self.database_name), str(self.database_username), - 'y', # confirmation of inputs + "y", # confirmation of inputs str(self.database_password), str(self.stored_passwords_salt), - str(self.zone_name), str(self.zone_port), str(self.parallel_port_range_begin), @@ -301,13 +312,12 @@ def build_input_for_catalog_provider(self): str(self.control_plane_port), str(self.schema_validation_base_uri), str(self.admin_username), - 'y', # confirmation of inputs - + "y", # confirmation of inputs str(self.zone_key), str(self.negotiation_key), str(self.control_plane_key), str(self.admin_password), - '' # confirmation of inputs + "", # confirmation of inputs ] # Handle the difference between 4.2 servers and 4.3 servers. @@ -317,9 +327,9 @@ def build_input_for_catalog_provider(self): input_args.insert(13, str(self.vault_directory)) else: input_args.append(str(self.vault_directory)) - input_args.append(str('')) # final confirmation + input_args.append(str("")) # final confirmation - return '\n'.join(input_args) + return "\n".join(input_args) def build(self): """Build the string for the setup script input. @@ -329,24 +339,30 @@ def build(self): returned. """ build_for_role = { - 'provider': self.build_input_for_catalog_provider, - 'consumer': self.build_input_for_catalog_consumer + "provider": self.build_input_for_catalog_provider, + "consumer": self.build_input_for_catalog_consumer, } try: return build_for_role[self.catalog_service_role]() except KeyError: - raise NotImplementedError('unsupported catalog service role [{}]'.format(self.catalog_service_role)) + raise NotImplementedError( + "unsupported catalog service role [{}]".format( + self.catalog_service_role + ) + ) def configure_rsyslog(container): def restart_rsyslog(container): - rsyslog_bin_path = os.path.join('/usr', 'sbin', 'rsyslogd') + rsyslog_bin_path = os.path.join("/usr", "sbin", "rsyslogd") - ec = execute.execute_command(container, f'pkill {os.path.basename(rsyslog_bin_path)}') + ec = execute.execute_command( + container, f"pkill {os.path.basename(rsyslog_bin_path)}" + ) if ec != 0: - logging.info(f'[{container.name}] failed to kill rsyslogd') + logging.info(f"[{container.name}] failed to kill rsyslogd") # TODO: Remove multiple attempts when a more appropriate solution is found MAX_NUMBER_OF_ATTEMPTS = 3 @@ -355,13 +371,36 @@ def restart_rsyslog(container): logging.debug("[{}] Attempting startup of rsyslogd".format(container.name)) while num_attempts <= MAX_NUMBER_OF_ATTEMPTS: - logging.debug("[{}] running startup attempt [#{}]".format(container.name, num_attempts)) + logging.debug( + "[{}] running startup attempt [#{}]".format( + container.name, num_attempts + ) + ) ec = execute.execute_command(container, rsyslog_bin_path) - logging.debug("[{}] startup attempt [#{}] {status}.".format(container.name, num_attempts, status="succeeded" if ec == 0 else "failed")) + logging.debug( + "[{}] startup attempt [#{}] {status}.".format( + container.name, + num_attempts, + status="succeeded" if ec == 0 else "failed", + ) + ) - logging.debug("[{}] checking to see if rsyslogd started up in the background.".format(container.name)) - is_alive = execute.execute_command(container, f'pgrep {os.path.basename(rsyslog_bin_path)}') == 0 - logging.debug("[{}] result of checking if rsyslogd is running: [{}]".format(container.name, is_alive)) + logging.debug( + "[{}] checking to see if rsyslogd started up in the background.".format( + container.name + ) + ) + is_alive = ( + execute.execute_command( + container, f"pgrep {os.path.basename(rsyslog_bin_path)}" + ) + == 0 + ) + logging.debug( + "[{}] result of checking if rsyslogd is running: [{}]".format( + container.name, is_alive + ) + ) # If we started an instance successfully, or it's restarted by another mechanism, we're satisfied if ec == 0 or is_alive: @@ -371,11 +410,13 @@ def restart_rsyslog(container): # Ensure we don't end up in a loop of failed start attempts, and that we log the failure if num_attempts > MAX_NUMBER_OF_ATTEMPTS: - raise RuntimeError(f'[{container.name}] failed to start rsyslogd') + raise RuntimeError(f"[{container.name}] failed to start rsyslogd") import textwrap - rsyslog_config_file = os.path.join('/etc', 'rsyslog.d', '00-irods.conf') - rsyslog_config_contents = textwrap.dedent('''\ + + rsyslog_config_file = os.path.join("/etc", "rsyslog.d", "00-irods.conf") + rsyslog_config_contents = textwrap.dedent( + """\ \$FileCreateMode 0644 \$DirCreateMode 0755 \$Umask 0000 @@ -383,14 +424,19 @@ def restart_rsyslog(container): :programname,startswith,\\"irodsServer\\" /var/log/irods/irods.log;irods_format & stop :programname,startswith,\\"irodsDelayServer\\" /var/log/irods/irods.log;irods_format - & stop''') + & stop""" + ) - ec = execute.execute_command(container, f'bash -c \'echo "{rsyslog_config_contents}" > {rsyslog_config_file}\'') + ec = execute.execute_command( + container, + f"bash -c 'echo \"{rsyslog_config_contents}\" > {rsyslog_config_file}'", + ) if ec != 0: - raise RuntimeError(f'[{container.name}] failed to configure rsyslog') + raise RuntimeError(f"[{container.name}] failed to configure rsyslog") - logrotate_config_file = os.path.join('/etc', 'logrotate.d', 'irods') - logrotate_config_contents = textwrap.dedent('''\ + logrotate_config_file = os.path.join("/etc", "logrotate.d", "irods") + logrotate_config_contents = textwrap.dedent( + """\ /var/log/irods/irods.log { weekly rotate 26 @@ -401,23 +447,27 @@ def restart_rsyslog(container): notifempty missingok su root root - }''') + }""" + ) - ec = execute.execute_command(container, f'bash -c \'echo "{logrotate_config_contents}" > {logrotate_config_file}\'') + ec = execute.execute_command( + container, + f"bash -c 'echo \"{logrotate_config_contents}\" > {logrotate_config_file}'", + ) if ec != 0: - raise RuntimeError(f'[{container.name}] failed to configure logrotate') + raise RuntimeError(f"[{container.name}] failed to configure logrotate") restart_rsyslog(container) def stop_irods(container): - irodsctl = os.path.join(context.irods_home(), 'irodsctl') - return execute.execute_command(container, f'{irodsctl} stop', user='irods') + irodsctl = os.path.join(context.irods_home(), "irodsctl") + return execute.execute_command(container, f"{irodsctl} stop", user="irods") def restart_irods(container): - irodsctl = os.path.join(context.irods_home(), 'irodsctl') - return execute.execute_command(container, f'{irodsctl} restart', user='irods') + irodsctl = os.path.join(context.irods_home(), "irodsctl") + return execute.execute_command(container, f"{irodsctl} restart", user="irods") def setup_irods_server(container, setup_input): @@ -436,28 +486,39 @@ def setup_irods_server(container, setup_input): try: if stop_irods(container) != 0: - logging.debug(f'[{container.name}] failed to stop iRODS server before setup') + logging.debug( + f"[{container.name}] failed to stop iRODS server before setup" + ) except Exception as e: - error_msg = f'[{container.name}] failed to stop iRODS server before setup: {str(e)}' + error_msg = ( + f"[{container.name}] failed to stop iRODS server before setup: {str(e)}" + ) if "unable to find user irods" in str(e): # If the user didn't exist at this point then the service probably wasn't started. - logging.debug(error_msg) + logging.debug(error_msg) else: - logging.error(error_msg) - raise e + logging.error(error_msg) + raise e - ec = execute.execute_command(container, 'bash -c \'echo "{}" > /input\''.format(setup_input)) + ec = execute.execute_command( + container, "bash -c 'echo \"{}\" > /input'".format(setup_input) + ) if ec != 0: - raise RuntimeError('failed to create setup script input file [{}]'.format(container.name)) + raise RuntimeError( + "failed to create setup script input file [{}]".format(container.name) + ) - execute.execute_command(container, 'cat /input') + execute.execute_command(container, "cat /input") - path_to_setup_script = os.path.join(context.irods_home(), 'scripts', 'setup_irods.py') - run_setup_script = 'bash -c \'{} {} < /input\''.format(container_info.python(container), - path_to_setup_script) + path_to_setup_script = os.path.join( + context.irods_home(), "scripts", "setup_irods.py" + ) + run_setup_script = "bash -c '{} {} < /input'".format( + container_info.python(container), path_to_setup_script + ) ec = execute.execute_command(container, run_setup_script) if ec != 0: - raise RuntimeError('failed to set up iRODS server [{}]'.format(container.name)) + raise RuntimeError("failed to set up iRODS server [{}]".format(container.name)) # Only configure rsyslog for versions later than 4.3.0 as this was the first release # which uses syslog. In the future, maybe the syslog implementation used in the @@ -467,14 +528,18 @@ def setup_irods_server(container, setup_input): configure_rsyslog(container) if restart_irods(container) != 0: - raise RuntimeError(f'[{container.name}] failed to start iRODS server after setup') + raise RuntimeError( + f"[{container.name}] failed to start iRODS server after setup" + ) -def setup_irods_catalog_provider(ctx, - database_service_instance=1, - provider_service_instance=1, - odbc_driver=None, - **kwargs): +def setup_irods_catalog_provider( + ctx, + database_service_instance=1, + provider_service_instance=1, + odbc_driver=None, + **kwargs, +): """Set up iRODS catalog service provider in a docker-compose project. Arguments: @@ -490,7 +555,9 @@ def setup_irods_catalog_provider(ctx, ) ) - odbc_setup.configure_odbc_driver(ctx.platform(), ctx.database(), csp_container, odbc_driver) + odbc_setup.configure_odbc_driver( + ctx.platform(), ctx.database(), csp_container, odbc_driver + ) db_container = ctx.docker_client.containers.get( context.irods_catalog_database_container( @@ -498,27 +565,28 @@ def setup_irods_catalog_provider(ctx, ) ) - setup_input = (setup_input_builder() - .setup(irods_version=irods_config.get_irods_version(csp_container), - catalog_service_role='provider', - database_server_hostname=context.container_hostname(db_container), - database_server_port=database_setup.database_server_port(ctx.database()), - **kwargs + setup_input = ( + setup_input_builder() + .setup( + irods_version=irods_config.get_irods_version(csp_container), + catalog_service_role="provider", + database_server_hostname=context.container_hostname(db_container), + database_server_port=database_setup.database_server_port(ctx.database()), + **kwargs, ) .build() ) - logging.debug('input to setup script [{}]'.format(setup_input)) + logging.debug("input to setup script [{}]".format(setup_input)) - logging.warning('setting up iRODS catalog provider [{}]'.format(csp_container.name)) + logging.warning("setting up iRODS catalog provider [{}]".format(csp_container.name)) setup_irods_server(csp_container, setup_input) -def setup_irods_catalog_consumer(ctx, - provider_service_instance=1, - consumer_service_instance=1, - **kwargs): +def setup_irods_catalog_consumer( + ctx, provider_service_instance=1, consumer_service_instance=1, **kwargs +): """Set up iRODS catalog service consumer in a docker-compose project. Arguments: @@ -539,26 +607,27 @@ def setup_irods_catalog_consumer(ctx, ) ) - setup_input = (setup_input_builder() - .setup(irods_version=irods_config.get_irods_version(csc_container), - catalog_service_role='consumer', - catalog_service_provider_host=context.container_hostname(csp_container), - **kwargs) + setup_input = ( + setup_input_builder() + .setup( + irods_version=irods_config.get_irods_version(csc_container), + catalog_service_role="consumer", + catalog_service_provider_host=context.container_hostname(csp_container), + **kwargs, + ) .build() ) - logging.debug('input to setup script [{}]'.format(setup_input)) + logging.debug("input to setup script [{}]".format(setup_input)) - logging.warning('setting up iRODS catalog consumer [{}]' - .format(csc_container.name)) + logging.warning("setting up iRODS catalog consumer [{}]".format(csc_container.name)) setup_irods_server(csc_container, setup_input) -def setup_irods_catalog_consumers(ctx, - provider_service_instance=1, - consumer_service_instances=None, - **kwargs): +def setup_irods_catalog_consumers( + ctx, provider_service_instance=1, consumer_service_instances=None, **kwargs +): """Set up all iRODS catalog service consumers in a docker-compose project in parallel. Arguments: @@ -573,11 +642,12 @@ def setup_irods_catalog_consumers(ctx, import concurrent.futures catalog_consumer_containers = ctx.compose_project.containers( - service_names=[context.irods_catalog_consumer_service()]) + service_names=[context.irods_catalog_consumer_service()] + ) if consumer_service_instances: if len(consumer_service_instances) == 0: - logging.warning('empty list of iRODS catalog service consumers to set up') + logging.warning("empty list of iRODS catalog service consumers to set up") return consumer_service_instances = [ @@ -596,37 +666,51 @@ def setup_irods_catalog_consumers(ctx, futures_to_catalog_consumer_instances = { executor.submit( setup_irods_catalog_consumer, - ctx, provider_service_instance, instance, **kwargs - ): instance for instance in consumer_service_instances + ctx, + provider_service_instance, + instance, + **kwargs, + ): instance + for instance in consumer_service_instances } logging.debug(futures_to_catalog_consumer_instances) for f in concurrent.futures.as_completed(futures_to_catalog_consumer_instances): i = futures_to_catalog_consumer_instances[f] - container_name = context.irods_catalog_consumer_container(ctx.compose_project.name, - i + 1) + container_name = context.irods_catalog_consumer_container( + ctx.compose_project.name, i + 1 + ) try: f.result() - logging.debug('setup completed successfully [{}]'.format(container_name)) + logging.debug( + "setup completed successfully [{}]".format(container_name) + ) except Exception as e: - logging.error('exception raised while setting up iRODS [{}]' - .format(container_name)) + logging.error( + "exception raised while setting up iRODS [{}]".format( + container_name + ) + ) logging.error(e) rc = 1 if rc != 0: - raise RuntimeError('failed to set up one or more catalog service consumers, ec=[{}]' - .format(rc)) - -def setup_irods_zone(ctx, - force_recreate=False, - provider_service_instance=1, - database_service_instance=1, - consumer_service_instances=None, - odbc_driver=None, - **kwargs): + raise RuntimeError( + "failed to set up one or more catalog service consumers, ec=[{}]".format(rc) + ) + + +def setup_irods_zone( + ctx, + force_recreate=False, + provider_service_instance=1, + database_service_instance=1, + consumer_service_instances=None, + odbc_driver=None, + **kwargs, +): """Set up an iRODS Zone with the specified settings on the specified service instances. Arguments: @@ -642,65 +726,83 @@ def setup_irods_zone(ctx, odbc_driver -- path to the local archive file containing the ODBC driver """ database_setup.wait_for_database_service( - ctx, database_service_instance=database_service_instance) - - logging.info('setting up catalog database [{}]'.format(database_service_instance)) - database_setup.setup_catalog(ctx, - force_recreate=force_recreate, - service_instance=database_service_instance) - - logging.info('setting up catalog provider [{}] [{}]'.format(provider_service_instance, - database_service_instance)) - setup_irods_catalog_provider(ctx, - database_service_instance=database_service_instance, - provider_service_instance=provider_service_instance, - odbc_driver=odbc_driver, - **kwargs) - - logging.info('setting up catalog consumers [{}] [{}]'.format(provider_service_instance, - consumer_service_instances)) - setup_irods_catalog_consumers(ctx, - provider_service_instance=provider_service_instance, - consumer_service_instances=consumer_service_instances, - **kwargs) - -def setup_irods_zones(ctx, - zone_info_list, - odbc_driver=None): + ctx, database_service_instance=database_service_instance + ) + + logging.info("setting up catalog database [{}]".format(database_service_instance)) + database_setup.setup_catalog( + ctx, force_recreate=force_recreate, service_instance=database_service_instance + ) + + logging.info( + "setting up catalog provider [{}] [{}]".format( + provider_service_instance, database_service_instance + ) + ) + setup_irods_catalog_provider( + ctx, + database_service_instance=database_service_instance, + provider_service_instance=provider_service_instance, + odbc_driver=odbc_driver, + **kwargs, + ) + + logging.info( + "setting up catalog consumers [{}] [{}]".format( + provider_service_instance, consumer_service_instances + ) + ) + setup_irods_catalog_consumers( + ctx, + provider_service_instance=provider_service_instance, + consumer_service_instances=consumer_service_instances, + **kwargs, + ) + + +def setup_irods_zones(ctx, zone_info_list, odbc_driver=None): import concurrent.futures rc = 0 with concurrent.futures.ThreadPoolExecutor() as executor: futures_to_containers = { - executor.submit(setup_irods_zone, - ctx, - provider_service_instance=z.provider_service_instance, - database_service_instance=z.database_service_instance, - consumer_service_instances=z.consumer_service_instances, - odbc_driver=odbc_driver, - zone_name=z.zone_name, - zone_key=z.zone_key, - negotiation_key=z.negotiation_key, - ): z for i, z in enumerate(zone_info_list) + executor.submit( + setup_irods_zone, + ctx, + provider_service_instance=z.provider_service_instance, + database_service_instance=z.database_service_instance, + consumer_service_instances=z.consumer_service_instances, + odbc_driver=odbc_driver, + zone_name=z.zone_name, + zone_key=z.zone_key, + negotiation_key=z.negotiation_key, + ): z + for i, z in enumerate(zone_info_list) } for f in concurrent.futures.as_completed(futures_to_containers): zone = futures_to_containers[f] try: f.result() - logging.debug('iRODS Zone setup completed successfully [{}]'.format(zone)) + logging.debug( + "iRODS Zone setup completed successfully [{}]".format(zone) + ) except Exception as e: - logging.error('exception raised while setting up iRODS Zone [{}]'.format(zone)) + logging.error( + "exception raised while setting up iRODS Zone [{}]".format(zone) + ) logging.error(e) rc = 1 if rc != 0: - raise RuntimeError('failed to set up one or more iRODS Zones, ec=[{}]'.format(rc)) + raise RuntimeError( + "failed to set up one or more iRODS Zones, ec=[{}]".format(rc) + ) -def make_negotiation_key(prefix=''): +def make_negotiation_key(prefix=""): """Generate a 32-byte negotiation key with an optional prefix. The generated key will be 32 bytes in length. If a longer string is passed in, it will be @@ -715,13 +817,13 @@ def make_negotiation_key(prefix=''): if len(prefix) > negotiation_key_size_in_bytes: return prefix[:negotiation_key_size_in_bytes] - filler = '_' * negotiation_key_size_in_bytes - return prefix + filler[:negotiation_key_size_in_bytes - len(prefix)] + filler = "_" * negotiation_key_size_in_bytes + return prefix + filler[: negotiation_key_size_in_bytes - len(prefix)] def make_zone_key(zone_name): - zone_key_prefix = 'ZONE_KEY_FOR' - return '_'.join([zone_key_prefix, zone_name]) + zone_key_prefix = "ZONE_KEY_FOR" + return "_".join([zone_key_prefix, zone_name]) def get_info_for_zones(ctx, zone_names, consumer_service_instances_per_zone=0): @@ -733,23 +835,34 @@ def get_info_for_zones(ctx, zone_names, consumer_service_instances_per_zone=0): context.service_instance(c.name) for c in ctx.compose_project.containers() if context.is_irods_catalog_consumer_container(c) - and context.service_instance(c.name) > i * consumer_service_instances_per_zone - and context.service_instance(c.name) <= (i + 1) * consumer_service_instances_per_zone + and context.service_instance(c.name) + > i * consumer_service_instances_per_zone + and context.service_instance(c.name) + <= (i + 1) * consumer_service_instances_per_zone ] - logging.info('consumer service instances for [{}] [{}] (expected: [{}])' - .format(zn, consumer_service_instances, - list(range((i*consumer_service_instances_per_zone)+1, - ((i+1)*consumer_service_instances_per_zone)+1)) - )) + logging.info( + "consumer service instances for [{}] [{}] (expected: [{}])".format( + zn, + consumer_service_instances, + list( + range( + (i * consumer_service_instances_per_zone) + 1, + ((i + 1) * consumer_service_instances_per_zone) + 1, + ) + ), + ) + ) zone_info_list.append( - zone_info(database_service_instance=i + 1, - provider_service_instance=i + 1, - consumer_service_instances=consumer_service_instances, - zone_name=zn, - zone_key=make_zone_key(zn), - negotiation_key=make_negotiation_key(zn)) + zone_info( + database_service_instance=i + 1, + provider_service_instance=i + 1, + consumer_service_instances=consumer_service_instances, + zone_name=zn, + zone_key=make_zone_key(zn), + negotiation_key=make_negotiation_key(zn), + ) ) return zone_info_list diff --git a/irods_testing_environment/json_utils.py b/irods_testing_environment/json_utils.py index a7f181e..92f7ca3 100644 --- a/irods_testing_environment/json_utils.py +++ b/irods_testing_environment/json_utils.py @@ -5,6 +5,7 @@ # local modules from . import archive + def get_json_from_file(container, target_file): """Return a JSON structure read out from a JSON file on the specified container. @@ -14,8 +15,11 @@ def get_json_from_file(container, target_file): """ import shutil from . import archive - json_file = os.path.join(archive.copy_from_container(container, target_file), - os.path.basename(target_file)) + + json_file = os.path.join( + archive.copy_from_container(container, target_file), + os.path.basename(target_file), + ) try: with open(json_file) as f: diff --git a/irods_testing_environment/logs.py b/irods_testing_environment/logs.py index f4f04dd..99cf3e6 100644 --- a/irods_testing_environment/logs.py +++ b/irods_testing_environment/logs.py @@ -7,6 +7,7 @@ from . import archive from . import context + # TODO: Maybe this should be some kind of builder def configure(verbosity=1, log_filename=None): # CRITICAL messages will always be printed, but anything after that is a function of the number of -v @@ -18,31 +19,31 @@ def configure(verbosity=1, log_filename=None): handlers.append(logging.FileHandler(os.path.abspath(log_filename))) logging.basicConfig( - level = level if level > logging.NOTSET else logging.DEBUG, - format = '%(asctime)-15s - %(message)s', - handlers = handlers + level=level if level > logging.NOTSET else logging.DEBUG, + format="%(asctime)-15s - %(message)s", + handlers=handlers, ) def log_directory_for_version(version): """Return default iRODS log directory for the given `version`.""" - major,minor,patch = version + major, minor, patch = version if int(major) < 4: - raise NotImplementedError('nothing prior to iRODS 4.0.0 is supported right now') + raise NotImplementedError("nothing prior to iRODS 4.0.0 is supported right now") if int(major) > 4: - raise NotImplementedError('the detected iRODS version does not exist yet') + raise NotImplementedError("the detected iRODS version does not exist yet") if int(minor) < 2: - return os.path.join(context.irods_home(), 'iRODS', 'log') + return os.path.join(context.irods_home(), "iRODS", "log") elif int(minor) < 3: - return os.path.join(context.irods_home(), 'log') + return os.path.join(context.irods_home(), "log") else: # Until we change how logging works in iRODS, anything in 4.3 or beyond should have logs here. - return os.path.join('/var', 'log', 'irods') + return os.path.join("/var", "log", "irods") - raise NotImplementedError('the detected iRODS version does not exist yet') + raise NotImplementedError("the detected iRODS version does not exist yet") def collect_logs(docker_client, containers, output_directory): @@ -55,14 +56,20 @@ def collect_logs(docker_client, containers, output_directory): """ from . import irods_config - archive.collect_files_from_containers(docker_client, - containers, - [os.path.join(context.irods_home(), 'log')], - output_directory) + archive.collect_files_from_containers( + docker_client, + containers, + [os.path.join(context.irods_home(), "log")], + output_directory, + ) - major, minor, patch = irods_config.get_irods_version(docker_client.containers.get(containers[0].name)) + major, minor, patch = irods_config.get_irods_version( + docker_client.containers.get(containers[0].name) + ) if minor > 2: - archive.collect_files_from_containers(docker_client, - containers, - [log_directory_for_version((major,minor,patch))], - output_directory) + archive.collect_files_from_containers( + docker_client, + containers, + [log_directory_for_version((major, minor, patch))], + output_directory, + ) diff --git a/irods_testing_environment/negotiation_key.py b/irods_testing_environment/negotiation_key.py index f9a4186..cda17f0 100644 --- a/irods_testing_environment/negotiation_key.py +++ b/irods_testing_environment/negotiation_key.py @@ -6,63 +6,99 @@ from . import execute from . import json_utils + def backup_file(container, file_path): - backup_file_path = file_path + '.orig' - if execute.execute_command(container, 'cp {} {}'.format(file_path, backup_file_path)) != 0: - raise RuntimeError('failed to backup [{}] [{}]'.format(file_path, container.name)) + backup_file_path = file_path + ".orig" + if ( + execute.execute_command( + container, "cp {} {}".format(file_path, backup_file_path) + ) + != 0 + ): + raise RuntimeError( + "failed to backup [{}] [{}]".format(file_path, container.name) + ) def restore_file(container, file_path): - backup_file_path = file_path + '.orig' - if execute.execute_command(container, 'cp {} {}'.format(backup_file_path, file_path)) != 0: - raise RuntimeError('failed to restore [{}] [{}]'.format(file_path, container.name)) + backup_file_path = file_path + ".orig" + if ( + execute.execute_command( + container, "cp {} {}".format(backup_file_path, file_path) + ) + != 0 + ): + raise RuntimeError( + "failed to restore [{}] [{}]".format(file_path, container.name) + ) def configure_ssl_in_client(container, client_ssl_negotiation, irods_env=None): - env = irods_env or json_utils.get_json_from_file(container, - context.service_account_irods_env()) + env = irods_env or json_utils.get_json_from_file( + container, context.service_account_irods_env() + ) - env['irods_client_server_policy'] = client_ssl_negotiation + env["irods_client_server_policy"] = client_ssl_negotiation json_utils.put_json_to_file(container, context.service_account_irods_env(), env) def configure_ssl_in_server(container, server_ssl_negotiation): - acPreConnect = 'acPreConnect(*OUT) {{ *OUT=\\"{}\\"; }}'.format(server_ssl_negotiation) + acPreConnect = 'acPreConnect(*OUT) {{ *OUT=\\"{}\\"; }}'.format( + server_ssl_negotiation + ) - add_acPreConnect = 'bash -c \'echo "{}" > {}; cat {} {} > {}\''.format( + add_acPreConnect = "bash -c 'echo \"{}\" > {}; cat {} {} > {}'".format( acPreConnect, - context.core_re() + '.tmp', - context.core_re() + '.tmp', - context.core_re() + '.orig', - context.core_re()) + context.core_re() + ".tmp", + context.core_re() + ".tmp", + context.core_re() + ".orig", + context.core_re(), + ) if execute.execute_command(container, add_acPreConnect) != 0: - raise RuntimeError('failed to update core.re [{}]'.format(container.name)) + raise RuntimeError("failed to update core.re [{}]".format(container.name)) def show_configurations(container, stream_output=False): - show_core_re = 'bash -c \'cat {} | head -n30\''.format(context.core_re()) - show_server_config = 'bash -c "cat {} | jq \'.\'"'.format(context.server_config()) - show_irods_env = 'bash -c "cat {} | jq \'.\'"'.format(context.service_account_irods_env()) - - if execute.execute_command(container, show_core_re, stream_output=stream_output) != 0: - raise RuntimeError('failed to cat core.re [{}]'.format(container.name)) - - if execute.execute_command(container, show_server_config, stream_output=stream_output) != 0: - raise RuntimeError('failed to cat server_config [{}]'.format(container.name)) - - if execute.execute_command(container, show_irods_env, stream_output=stream_output) != 0: - raise RuntimeError('failed to cat irods_environment [{}]'.format(container.name)) + show_core_re = "bash -c 'cat {} | head -n30'".format(context.core_re()) + show_server_config = "bash -c \"cat {} | jq '.'\"".format(context.server_config()) + show_irods_env = "bash -c \"cat {} | jq '.'\"".format( + context.service_account_irods_env() + ) + + if ( + execute.execute_command(container, show_core_re, stream_output=stream_output) + != 0 + ): + raise RuntimeError("failed to cat core.re [{}]".format(container.name)) + + if ( + execute.execute_command( + container, show_server_config, stream_output=stream_output + ) + != 0 + ): + raise RuntimeError("failed to cat server_config [{}]".format(container.name)) + + if ( + execute.execute_command(container, show_irods_env, stream_output=stream_output) + != 0 + ): + raise RuntimeError( + "failed to cat irods_environment [{}]".format(container.name) + ) def configure_negotiation_key(container, negotiation_key, server_config=None): - config = server_config or json_utils.get_json_from_file(container, context.server_config()) + config = server_config or json_utils.get_json_from_file( + container, context.server_config() + ) if negotiation_key is not None: logging.info('adding "negotiation_key" [{}] to config'.format(negotiation_key)) - config['negotiation_key'] = negotiation_key - elif 'negotiation_key' in config: + config["negotiation_key"] = negotiation_key + elif "negotiation_key" in config: logging.info('deleting "negotiation_key" from config') - del config['negotiation_key'] + del config["negotiation_key"] json_utils.put_json_to_file(container, context.server_config(), config) diff --git a/irods_testing_environment/odbc_setup.py b/irods_testing_environment/odbc_setup.py index 0d98bc5..062ed8e 100644 --- a/irods_testing_environment/odbc_setup.py +++ b/irods_testing_environment/odbc_setup.py @@ -12,6 +12,7 @@ # Flip this bool to switch which ODBC driver is used for MariaDB projects. mariadb_use_mysql_odbc_driver = True + def configure_postgres_odbc_driver(csp_container, odbc_driver): """Configure ODBC driver for postgres. @@ -19,7 +20,10 @@ def configure_postgres_odbc_driver(csp_container, odbc_driver): csp_container -- docker container on which the iRODS catalog service provider is running odbc_driver -- path to local archive file containing the ODBC driver package """ - logging.debug('no ODBC driver setup required for postgres [{}]'.format(csp_container)) + logging.debug( + "no ODBC driver setup required for postgres [{}]".format(csp_container) + ) + def configure_odbc_driver_ubuntu_2004_postgres_14(csp_container, odbc_driver): """Configure ODBC driver for postgres 14 on ubuntu 20.04. @@ -30,6 +34,7 @@ def configure_odbc_driver_ubuntu_2004_postgres_14(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_ubuntu_2004_postgres_16(csp_container, odbc_driver): """Configure ODBC driver for postgres 16 on ubuntu 20.04. @@ -39,6 +44,7 @@ def configure_odbc_driver_ubuntu_2004_postgres_16(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_ubuntu_2204_postgres_14(csp_container, odbc_driver): """Configure ODBC driver for postgres 14 on ubuntu 22.04. @@ -48,6 +54,7 @@ def configure_odbc_driver_ubuntu_2204_postgres_14(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_ubuntu_2204_postgres_16(csp_container, odbc_driver): """Configure ODBC driver for postgres 16 on ubuntu 22.04. @@ -57,6 +64,7 @@ def configure_odbc_driver_ubuntu_2204_postgres_16(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_ubuntu_2404_postgres_14(csp_container, odbc_driver): """Configure ODBC driver for postgres 14 on ubuntu 24.04. @@ -66,6 +74,7 @@ def configure_odbc_driver_ubuntu_2404_postgres_14(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_ubuntu_2404_postgres_16(csp_container, odbc_driver): """Configure ODBC driver for postgres 16 on ubuntu 24.04. @@ -75,6 +84,7 @@ def configure_odbc_driver_ubuntu_2404_postgres_16(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_debian_11_postgres_14(csp_container, odbc_driver): """Configure ODBC driver for postgres 14 on debian 11. @@ -84,6 +94,7 @@ def configure_odbc_driver_debian_11_postgres_14(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_debian_11_postgres_16(csp_container, odbc_driver): """Configure ODBC driver for postgres 16 on debian 11. @@ -93,6 +104,7 @@ def configure_odbc_driver_debian_11_postgres_16(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_debian_12_postgres_14(csp_container, odbc_driver): """Configure ODBC driver for postgres 14 on debian 12. @@ -102,6 +114,7 @@ def configure_odbc_driver_debian_12_postgres_14(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_debian_12_postgres_16(csp_container, odbc_driver): """Configure ODBC driver for postgres 16 on debian 12. @@ -111,6 +124,7 @@ def configure_odbc_driver_debian_12_postgres_16(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_centos_7_postgres_14(csp_container, odbc_driver): """Configure ODBC driver for postgres 14 on centos 7. @@ -120,6 +134,7 @@ def configure_odbc_driver_centos_7_postgres_14(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_centos_7_postgres_16(csp_container, odbc_driver): """Configure ODBC driver for postgres 16 on centos 7. @@ -129,6 +144,7 @@ def configure_odbc_driver_centos_7_postgres_16(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_almalinux_8_postgres_14(csp_container, odbc_driver): """Configure ODBC driver for postgres 14 on almalinux 8. @@ -138,6 +154,7 @@ def configure_odbc_driver_almalinux_8_postgres_14(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_almalinux_8_postgres_16(csp_container, odbc_driver): """Configure ODBC driver for postgres 16 on almalinux 8. @@ -147,6 +164,7 @@ def configure_odbc_driver_almalinux_8_postgres_16(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_rockylinux_8_postgres_14(csp_container, odbc_driver): """Configure ODBC driver for postgres 14 on rockylinux 8. @@ -156,6 +174,7 @@ def configure_odbc_driver_rockylinux_8_postgres_14(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_rockylinux_8_postgres_16(csp_container, odbc_driver): """Configure ODBC driver for postgres 16 on rockylinux 8. @@ -165,6 +184,7 @@ def configure_odbc_driver_rockylinux_8_postgres_16(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_rockylinux_9_postgres_14(csp_container, odbc_driver): """Configure ODBC driver for postgres 14 on rockylinux 9. @@ -174,6 +194,7 @@ def configure_odbc_driver_rockylinux_9_postgres_14(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def configure_odbc_driver_rockylinux_9_postgres_16(csp_container, odbc_driver): """Configure ODBC driver for postgres 16 on rockylinux 9. @@ -183,6 +204,7 @@ def configure_odbc_driver_rockylinux_9_postgres_16(csp_container, odbc_driver): """ configure_postgres_odbc_driver(csp_container, odbc_driver) + def make_mysql_odbcinst_ini(csp_container, container_odbc_driver_dir): """Generate content for the /etc/odbcinst.ini configuration file used by mysql. @@ -190,28 +212,38 @@ def make_mysql_odbcinst_ini(csp_container, container_odbc_driver_dir): csp_container -- container running iRODS catalog service provider using the ODBC driver container_odbc_driver_dir -- path in `csp_container` containing the ODBC driver directory """ - odbcinst_ini_path = os.path.join('/etc', 'odbcinst.ini') + odbcinst_ini_path = os.path.join("/etc", "odbcinst.ini") # This is the same for both 8.0 and 8.4. - logging.debug('configuring odbcinst.ini with MySQL 8.x drivers') - odbcinst_ini_contents = textwrap.dedent("""\ + logging.debug("configuring odbcinst.ini with MySQL 8.x drivers") + odbcinst_ini_contents = textwrap.dedent( + """\ [MySQL ANSI] Description = MySQL ODBC 8.0 ANSI Driver Driver = {0}/lib/libmyodbc8a.so [MySQL Unicode] Description = MySQL ODBC 8.0 Unicode Driver - Driver = {0}/lib/libmyodbc8w.so""".format(container_odbc_driver_dir)) - - cmd = 'bash -c \'echo "{0}" > {1}\''.format(odbcinst_ini_contents, odbcinst_ini_path) + Driver = {0}/lib/libmyodbc8w.so""".format( + container_odbc_driver_dir + ) + ) + + cmd = "bash -c 'echo \"{0}\" > {1}'".format( + odbcinst_ini_contents, odbcinst_ini_path + ) ec = execute.execute_command(csp_container, cmd) if ec != 0: - raise RuntimeError('failed to populate odbcinst.ini [ec=[{0}], container=[{1}]]' - .format(ec, csp_container)) + raise RuntimeError( + "failed to populate odbcinst.ini [ec=[{0}], container=[{1}]]".format( + ec, csp_container + ) + ) + + execute.execute_command(csp_container, "cat {}".format(odbcinst_ini_path)) - execute.execute_command(csp_container, 'cat {}'.format(odbcinst_ini_path)) -def configure_mysql_odbc_driver(csp_container, odbc_driver, extension='tar.gz'): +def configure_mysql_odbc_driver(csp_container, odbc_driver, extension="tar.gz"): """Configure ODBC driver for mysql and return the ODBC driver path. Argument: @@ -220,20 +252,23 @@ def configure_mysql_odbc_driver(csp_container, odbc_driver, extension='tar.gz'): extension -- file extension for the archive file """ if not os.path.exists(odbc_driver): - raise RuntimeError('indicated ODBC driver does not exist [{}]'.format(odbc_driver)) + raise RuntimeError( + "indicated ODBC driver does not exist [{}]".format(odbc_driver) + ) - logging.info('looking for odbc driver [{}]'.format(odbc_driver)) + logging.info("looking for odbc driver [{}]".format(odbc_driver)) - container_odbc_driver_dir = archive.copy_archive_to_container(csp_container, - odbc_driver, - extension=extension) + container_odbc_driver_dir = archive.copy_archive_to_container( + csp_container, odbc_driver, extension=extension + ) - execute.execute_command(csp_container, 'ls -l {}'.format(container_odbc_driver_dir)) + execute.execute_command(csp_container, "ls -l {}".format(container_odbc_driver_dir)) make_mysql_odbcinst_ini(csp_container, container_odbc_driver_dir) return container_odbc_driver_dir + def download_mysql_odbc_driver(url, destination=None, always_download=False): """Downloads the file indicated by `url` and returns the path to the file. @@ -246,23 +281,28 @@ def download_mysql_odbc_driver(url, destination=None, always_download=False): if not destination: from urllib.parse import urlparse - destination = os.path.join('/tmp', os.path.basename(urlparse(url).path)) + + destination = os.path.join("/tmp", os.path.basename(urlparse(url).path)) destination = os.path.abspath(destination) if not always_download and os.path.exists(destination): - logging.info('destination mysql odbc already exists, not downloading [{}]' - .format(destination)) + logging.info( + "destination mysql odbc already exists, not downloading [{}]".format( + destination + ) + ) return destination - logging.info('downloading [{}] to [{}]'.format(url, destination)) + logging.info("downloading [{}] to [{}]".format(url, destination)) with urllib.request.urlopen(url) as r: - with open(destination, 'w+b') as f: + with open(destination, "w+b") as f: shutil.copyfileobj(r, f) return destination + def configure_odbc_driver_mysql_80(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.0. @@ -272,10 +312,12 @@ def configure_odbc_driver_mysql_80(csp_container, odbc_driver): """ if not odbc_driver: odbc_driver = download_mysql_odbc_driver( - 'https://dev.mysql.com/get/Downloads/Connector-ODBC/8.0/mysql-connector-odbc-8.0.33-linux-glibc2.28-x86-64bit.tar.gz') + "https://dev.mysql.com/get/Downloads/Connector-ODBC/8.0/mysql-connector-odbc-8.0.33-linux-glibc2.28-x86-64bit.tar.gz" + ) configure_mysql_odbc_driver(csp_container, os.path.abspath(odbc_driver)) + def configure_odbc_driver_mysql_84(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.4. @@ -285,10 +327,12 @@ def configure_odbc_driver_mysql_84(csp_container, odbc_driver): """ if not odbc_driver: odbc_driver = download_mysql_odbc_driver( - 'https://dev.mysql.com/get/Downloads/Connector-ODBC/8.4/mysql-connector-odbc-8.4.0-linux-glibc2.28-x86-64bit.tar.gz') + "https://dev.mysql.com/get/Downloads/Connector-ODBC/8.4/mysql-connector-odbc-8.4.0-linux-glibc2.28-x86-64bit.tar.gz" + ) configure_mysql_odbc_driver(csp_container, os.path.abspath(odbc_driver)) + def configure_odbc_driver_centos_7_mysql_80(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.0 on centos 7. @@ -299,10 +343,12 @@ def configure_odbc_driver_centos_7_mysql_80(csp_container, odbc_driver): # special case since EL7 uses an older version of glibc if not odbc_driver: odbc_driver = download_mysql_odbc_driver( - 'https://downloads.mysql.com/archives/get/p/10/file/mysql-connector-odbc-8.0.29-linux-glibc2.12-x86-64bit.tar.gz') + "https://downloads.mysql.com/archives/get/p/10/file/mysql-connector-odbc-8.0.29-linux-glibc2.12-x86-64bit.tar.gz" + ) configure_mysql_odbc_driver(csp_container, os.path.abspath(odbc_driver)) + def configure_odbc_driver_ubuntu_2004_mysql_80(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.0 on ubuntu 20.04. @@ -312,6 +358,7 @@ def configure_odbc_driver_ubuntu_2004_mysql_80(csp_container, odbc_driver): """ configure_odbc_driver_mysql_80(csp_container, odbc_driver) + def configure_odbc_driver_ubuntu_2204_mysql_80(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.0 on ubuntu 22.04. @@ -321,6 +368,7 @@ def configure_odbc_driver_ubuntu_2204_mysql_80(csp_container, odbc_driver): """ configure_odbc_driver_mysql_80(csp_container, odbc_driver) + def configure_odbc_driver_ubuntu_2204_mysql_84(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.4 on ubuntu 22.04. @@ -330,6 +378,7 @@ def configure_odbc_driver_ubuntu_2204_mysql_84(csp_container, odbc_driver): """ configure_odbc_driver_mysql_84(csp_container, odbc_driver) + def configure_odbc_driver_ubuntu_2404_mysql_80(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.0 on ubuntu 24.04. @@ -339,6 +388,7 @@ def configure_odbc_driver_ubuntu_2404_mysql_80(csp_container, odbc_driver): """ configure_odbc_driver_mysql_80(csp_container, odbc_driver) + def configure_odbc_driver_ubuntu_2404_mysql_84(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.4 on ubuntu 24.04. @@ -348,6 +398,7 @@ def configure_odbc_driver_ubuntu_2404_mysql_84(csp_container, odbc_driver): """ configure_odbc_driver_mysql_84(csp_container, odbc_driver) + def configure_odbc_driver_debian_11_mysql_80(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.0 on debian 11. @@ -357,6 +408,7 @@ def configure_odbc_driver_debian_11_mysql_80(csp_container, odbc_driver): """ configure_odbc_driver_mysql_80(csp_container, odbc_driver) + def configure_odbc_driver_debian_12_mysql_80(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.0 on debian 12. @@ -366,6 +418,7 @@ def configure_odbc_driver_debian_12_mysql_80(csp_container, odbc_driver): """ configure_odbc_driver_mysql_80(csp_container, odbc_driver) + def configure_odbc_driver_debian_12_mysql_84(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.4 on debian 12. @@ -375,6 +428,7 @@ def configure_odbc_driver_debian_12_mysql_84(csp_container, odbc_driver): """ configure_odbc_driver_mysql_84(csp_container, odbc_driver) + def configure_odbc_driver_almalinux_8_mysql_80(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.0 on almalinux 8. @@ -384,6 +438,7 @@ def configure_odbc_driver_almalinux_8_mysql_80(csp_container, odbc_driver): """ configure_odbc_driver_mysql_80(csp_container, odbc_driver) + def configure_odbc_driver_almalinux_8_mysql_84(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.4 on almalinux 8. @@ -393,6 +448,7 @@ def configure_odbc_driver_almalinux_8_mysql_84(csp_container, odbc_driver): """ configure_odbc_driver_mysql_84(csp_container, odbc_driver) + def configure_odbc_driver_rockylinux_8_mysql_80(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.0 on rockylinux 8. @@ -402,6 +458,7 @@ def configure_odbc_driver_rockylinux_8_mysql_80(csp_container, odbc_driver): """ configure_odbc_driver_mysql_80(csp_container, odbc_driver) + def configure_odbc_driver_rockylinux_8_mysql_84(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.4 on rockylinux 8. @@ -411,6 +468,7 @@ def configure_odbc_driver_rockylinux_8_mysql_84(csp_container, odbc_driver): """ configure_odbc_driver_mysql_84(csp_container, odbc_driver) + def configure_odbc_driver_rockylinux_9_mysql_80(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.0 on rockylinux 9. @@ -420,6 +478,7 @@ def configure_odbc_driver_rockylinux_9_mysql_80(csp_container, odbc_driver): """ configure_odbc_driver_mysql_80(csp_container, odbc_driver) + def configure_odbc_driver_rockylinux_9_mysql_84(csp_container, odbc_driver): """Configure ODBC driver for mysql 8.4 on rockylinux 9. @@ -429,6 +488,7 @@ def configure_odbc_driver_rockylinux_9_mysql_84(csp_container, odbc_driver): """ configure_odbc_driver_mysql_84(csp_container, odbc_driver) + def make_mariadb_odbcinst_ini(csp_container, container_odbc_lib_dir): """Generate content for the /etc/odbcinst.ini configuration file used by mariadb. @@ -436,22 +496,32 @@ def make_mariadb_odbcinst_ini(csp_container, container_odbc_lib_dir): csp_container -- container running iRODS catalog service provider using the ODBC driver container_odbc_lib_dir -- path in `csp_container` containing the ODBC driver library """ - odbcinst_ini_path = os.path.join('/etc', 'odbcinst.ini') + odbcinst_ini_path = os.path.join("/etc", "odbcinst.ini") - odbcinst_ini_contents = textwrap.dedent("""\ + odbcinst_ini_contents = textwrap.dedent( + """\ [MariaDB] Description = MariaDB ODBC Connector Driver = {0}/libmaodbc.so Threading = 0 - """.format(container_odbc_lib_dir)) - - cmd = 'bash -c \'echo "{0}" > {1}\''.format(odbcinst_ini_contents, odbcinst_ini_path) + """.format( + container_odbc_lib_dir + ) + ) + + cmd = "bash -c 'echo \"{0}\" > {1}'".format( + odbcinst_ini_contents, odbcinst_ini_path + ) ec = execute.execute_command(csp_container, cmd) if ec != 0: - raise RuntimeError('failed to populate odbcinst.ini [ec=[{0}], container=[{1}]]' - .format(ec, csp_container)) + raise RuntimeError( + "failed to populate odbcinst.ini [ec=[{0}], container=[{1}]]".format( + ec, csp_container + ) + ) + + execute.execute_command(csp_container, "cat {}".format(odbcinst_ini_path)) - execute.execute_command(csp_container, 'cat {}'.format(odbcinst_ini_path)) def configure_mariadb_odbc_driver_apt(csp_container, odbc_driver, package_url): """Configure ODBC driver package for mariadb (via apt) @@ -472,10 +542,11 @@ def configure_mariadb_odbc_driver_apt(csp_container, odbc_driver, package_url): odbc_driver_archive = archive.create_archive([odbc_driver]) archive.copy_archive_to_container(csp_container, odbc_driver_archive) - execute.execute_command(csp_container, 'apt-get update') - execute.execute_command(csp_container, 'apt-get install {}'.format(odbc_driver)) + execute.execute_command(csp_container, "apt-get update") + execute.execute_command(csp_container, "apt-get install {}".format(odbc_driver)) + + make_mariadb_odbcinst_ini(csp_container, "/usr/lib/x86_64-linux-gnu") - make_mariadb_odbcinst_ini(csp_container, '/usr/lib/x86_64-linux-gnu') def configure_mariadb_odbc_driver_dnf(csp_container, odbc_driver, package_url): """Configure ODBC driver package for mariadb (via yum) @@ -496,9 +567,10 @@ def configure_mariadb_odbc_driver_dnf(csp_container, odbc_driver, package_url): odbc_driver_archive = archive.create_archive([odbc_driver]) archive.copy_archive_to_container(csp_container, odbc_driver_archive) - execute.execute_command(csp_container, 'dnf install -y {}'.format(odbc_driver)) + execute.execute_command(csp_container, "dnf install -y {}".format(odbc_driver)) + + make_mariadb_odbcinst_ini(csp_container, "/usr/lib64") - make_mariadb_odbcinst_ini(csp_container, '/usr/lib64') def configure_odbc_driver_ubuntu_2004_mariadb(csp_container, odbc_driver): """Configure ODBC driver for mariadb on ubuntu 20.04. @@ -510,7 +582,9 @@ def configure_odbc_driver_ubuntu_2004_mariadb(csp_container, odbc_driver): configure_mariadb_odbc_driver_apt( csp_container, odbc_driver, - 'https://downloads.mariadb.com/Connectors/odbc/connector-odbc-3.2.2/mariadb-connector-odbc-3.2.2-ubu2004-amd64.deb') + "https://downloads.mariadb.com/Connectors/odbc/connector-odbc-3.2.2/mariadb-connector-odbc-3.2.2-ubu2004-amd64.deb", + ) + def configure_odbc_driver_ubuntu_2004_mariadb_106(csp_container, odbc_driver): """Configure ODBC driver for mariadb 10.6 on ubuntu 20.04. @@ -521,6 +595,7 @@ def configure_odbc_driver_ubuntu_2004_mariadb_106(csp_container, odbc_driver): """ configure_odbc_driver_ubuntu_2004_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_ubuntu_2004_mariadb_1011(csp_container, odbc_driver): """Configure ODBC driver for mariadb 10.11 on ubuntu 20.04. @@ -530,6 +605,7 @@ def configure_odbc_driver_ubuntu_2004_mariadb_1011(csp_container, odbc_driver): """ configure_odbc_driver_ubuntu_2004_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_ubuntu_2204_mariadb(csp_container, odbc_driver): """Configure ODBC driver for mariadb on ubuntu 22.04. @@ -540,7 +616,9 @@ def configure_odbc_driver_ubuntu_2204_mariadb(csp_container, odbc_driver): configure_mariadb_odbc_driver_apt( csp_container, odbc_driver, - 'https://downloads.mariadb.com/Connectors/odbc/connector-odbc-3.2.2/mariadb-connector-odbc-3.2.2-ubu2204-amd64.deb') + "https://downloads.mariadb.com/Connectors/odbc/connector-odbc-3.2.2/mariadb-connector-odbc-3.2.2-ubu2204-amd64.deb", + ) + def configure_odbc_driver_ubuntu_2204_mariadb_106(csp_container, odbc_driver): """Configure ODBC driver for mariadb 10.6 on ubuntu 22.04. @@ -551,6 +629,7 @@ def configure_odbc_driver_ubuntu_2204_mariadb_106(csp_container, odbc_driver): """ configure_odbc_driver_ubuntu_2204_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_ubuntu_2204_mariadb_1011(csp_container, odbc_driver): """Configure ODBC driver for mariadb 10.11 on ubuntu 22.04. @@ -560,6 +639,7 @@ def configure_odbc_driver_ubuntu_2204_mariadb_1011(csp_container, odbc_driver): """ configure_odbc_driver_ubuntu_2204_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_ubuntu_2204_mariadb_114(csp_container, odbc_driver): """Configure ODBC driver for mariadb 11.4 on ubuntu 22.04. @@ -569,6 +649,7 @@ def configure_odbc_driver_ubuntu_2204_mariadb_114(csp_container, odbc_driver): """ configure_odbc_driver_ubuntu_2204_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_ubuntu_2404_mariadb(csp_container, odbc_driver): """Configure ODBC driver for mariadb on ubuntu 24.04. @@ -580,7 +661,9 @@ def configure_odbc_driver_ubuntu_2404_mariadb(csp_container, odbc_driver): csp_container, odbc_driver, # package is for 22.04, but works on 24.04 - 'https://downloads.mariadb.com/Connectors/odbc/connector-odbc-3.2.2/mariadb-connector-odbc-3.2.2-ubu2204-amd64.deb') + "https://downloads.mariadb.com/Connectors/odbc/connector-odbc-3.2.2/mariadb-connector-odbc-3.2.2-ubu2204-amd64.deb", + ) + def configure_odbc_driver_ubuntu_2404_mariadb_1011(csp_container, odbc_driver): """Configure ODBC driver for mariadb 10.11 on ubuntu 24.04. @@ -591,6 +674,7 @@ def configure_odbc_driver_ubuntu_2404_mariadb_1011(csp_container, odbc_driver): """ configure_odbc_driver_ubuntu_2404_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_ubuntu_2404_mariadb_114(csp_container, odbc_driver): """Configure ODBC driver for mariadb 11.4 on ubuntu 24.04. @@ -600,6 +684,7 @@ def configure_odbc_driver_ubuntu_2404_mariadb_114(csp_container, odbc_driver): """ configure_odbc_driver_ubuntu_2404_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_debian_11_mariadb(csp_container, odbc_driver): """Configure ODBC driver for mariadb on debian 11. @@ -610,7 +695,9 @@ def configure_odbc_driver_debian_11_mariadb(csp_container, odbc_driver): configure_mariadb_odbc_driver_apt( csp_container, odbc_driver, - 'https://downloads.mariadb.com/Connectors/odbc/connector-odbc-3.2.2/mariadb-connector-odbc-3.2.2-deb11-amd64.deb') + "https://downloads.mariadb.com/Connectors/odbc/connector-odbc-3.2.2/mariadb-connector-odbc-3.2.2-deb11-amd64.deb", + ) + def configure_odbc_driver_debian_11_mariadb_106(csp_container, odbc_driver): """Configure ODBC driver for mariadb 10.6 debian 11. @@ -621,6 +708,7 @@ def configure_odbc_driver_debian_11_mariadb_106(csp_container, odbc_driver): """ configure_odbc_driver_debian_11_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_debian_11_mariadb_1011(csp_container, odbc_driver): """Configure ODBC driver for mariadb 10.11 debian 11. @@ -630,6 +718,7 @@ def configure_odbc_driver_debian_11_mariadb_1011(csp_container, odbc_driver): """ configure_odbc_driver_debian_11_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_debian_12_mariadb(csp_container, odbc_driver): """Configure ODBC driver for mariadb on debian 12. @@ -640,7 +729,9 @@ def configure_odbc_driver_debian_12_mariadb(csp_container, odbc_driver): configure_mariadb_odbc_driver_apt( csp_container, odbc_driver, - 'https://downloads.mariadb.com/Connectors/odbc/connector-odbc-3.2.2/mariadb-connector-odbc-3.2.2-deb12-amd64.deb') + "https://downloads.mariadb.com/Connectors/odbc/connector-odbc-3.2.2/mariadb-connector-odbc-3.2.2-deb12-amd64.deb", + ) + def configure_odbc_driver_debian_12_mariadb_1011(csp_container, odbc_driver): """Configure ODBC driver for mariadb 10.11 debian 12. @@ -651,6 +742,7 @@ def configure_odbc_driver_debian_12_mariadb_1011(csp_container, odbc_driver): """ configure_odbc_driver_debian_11_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_debian_12_mariadb_114(csp_container, odbc_driver): """Configure ODBC driver for mariadb 11.4 debian 12. @@ -660,6 +752,7 @@ def configure_odbc_driver_debian_12_mariadb_114(csp_container, odbc_driver): """ configure_odbc_driver_debian_11_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_el_8_mariadb(csp_container, odbc_driver): """Configure ODBC driver for mariadb on EL 8. @@ -670,7 +763,9 @@ def configure_odbc_driver_el_8_mariadb(csp_container, odbc_driver): configure_mariadb_odbc_driver_dnf( csp_container, odbc_driver, - 'https://downloads.mariadb.com/Connectors/odbc/connector-odbc-3.2.2/mariadb-connector-odbc-3.2.2-rhel8-amd64.rpm') + "https://downloads.mariadb.com/Connectors/odbc/connector-odbc-3.2.2/mariadb-connector-odbc-3.2.2-rhel8-amd64.rpm", + ) + def configure_odbc_driver_almalinux_8_mariadb_106(csp_container, odbc_driver): """Configure ODBC driver for mariadb 10.6 almalinux 8. @@ -681,6 +776,7 @@ def configure_odbc_driver_almalinux_8_mariadb_106(csp_container, odbc_driver): """ configure_odbc_driver_el_8_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_almalinux_8_mariadb_1011(csp_container, odbc_driver): """Configure ODBC driver for mariadb 10.11 almalinux 8. @@ -690,6 +786,7 @@ def configure_odbc_driver_almalinux_8_mariadb_1011(csp_container, odbc_driver): """ configure_odbc_driver_el_8_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_almalinux_8_mariadb_114(csp_container, odbc_driver): """Configure ODBC driver for mariadb 11.4 almalinux 8. @@ -699,6 +796,7 @@ def configure_odbc_driver_almalinux_8_mariadb_114(csp_container, odbc_driver): """ configure_odbc_driver_el_8_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_rockylinux_8_mariadb_106(csp_container, odbc_driver): """Configure ODBC driver for mariadb 10.6 rockylinux 8. @@ -708,6 +806,7 @@ def configure_odbc_driver_rockylinux_8_mariadb_106(csp_container, odbc_driver): """ configure_odbc_driver_el_8_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_rockylinux_8_mariadb_1011(csp_container, odbc_driver): """Configure ODBC driver for mariadb 10.11 rockylinux 8. @@ -717,6 +816,7 @@ def configure_odbc_driver_rockylinux_8_mariadb_1011(csp_container, odbc_driver): """ configure_odbc_driver_el_8_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_rockylinux_8_mariadb_114(csp_container, odbc_driver): """Configure ODBC driver for mariadb 11.4 rockylinux 8. @@ -726,6 +826,7 @@ def configure_odbc_driver_rockylinux_8_mariadb_114(csp_container, odbc_driver): """ configure_odbc_driver_el_8_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_el_9_mariadb(csp_container, odbc_driver): """Configure ODBC driver for mariadb on EL 9. @@ -736,7 +837,9 @@ def configure_odbc_driver_el_9_mariadb(csp_container, odbc_driver): configure_mariadb_odbc_driver_dnf( csp_container, odbc_driver, - 'https://downloads.mariadb.com/Connectors/odbc/connector-odbc-3.2.2/mariadb-connector-odbc-3.2.2-rhel9-amd64.rpm') + "https://downloads.mariadb.com/Connectors/odbc/connector-odbc-3.2.2/mariadb-connector-odbc-3.2.2-rhel9-amd64.rpm", + ) + def configure_odbc_driver_rockylinux_9_mariadb_1011(csp_container, odbc_driver): """Configure ODBC driver for mariadb 10.11 rockylinux 9. @@ -747,6 +850,7 @@ def configure_odbc_driver_rockylinux_9_mariadb_1011(csp_container, odbc_driver): """ configure_odbc_driver_el_9_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_rockylinux_9_mariadb_114(csp_container, odbc_driver): """Configure ODBC driver for mariadb 11.4 rockylinux 9. @@ -756,6 +860,7 @@ def configure_odbc_driver_rockylinux_9_mariadb_114(csp_container, odbc_driver): """ configure_odbc_driver_el_9_mariadb(csp_container, odbc_driver) + def configure_odbc_driver_centos_7_mariadb_106(csp_container, odbc_driver): """Configure ODBC driver for mariadb 10.6 centos 7. @@ -768,7 +873,10 @@ def configure_odbc_driver_centos_7_mariadb_106(csp_container, odbc_driver): # removed on resolution of #217. configure_odbc_driver_centos_7_mysql_80(csp_container, odbc_driver) -def configure_odbc_driver(platform_image, database_image, csp_container, odbc_driver=None): + +def configure_odbc_driver( + platform_image, database_image, csp_container, odbc_driver=None +): """Make an ODBC setup strategy for the given database type. Arguments: @@ -778,13 +886,17 @@ def configure_odbc_driver(platform_image, database_image, csp_container, odbc_dr odbc_driver -- if specified, the ODBC driver will be sought here """ import inspect + # generate the function name of the form: # configure_odbc_driver_platform-repo_platform-tag_database-repo_database-tag - func_name = '_'.join([inspect.currentframe().f_code.co_name, - context.sanitize(context.image_repo(platform_image)), - context.sanitize(context.image_tag(platform_image)), - context.sanitize(context.image_repo(database_image)), - context.sanitize(context.image_tag(database_image))]) + func_name = "_".join( + [ + inspect.currentframe().f_code.co_name, + context.sanitize(context.image_repo(platform_image)), + context.sanitize(context.image_tag(platform_image)), + context.sanitize(context.image_repo(database_image)), + context.sanitize(context.image_tag(database_image)), + ] + ) eval(func_name)(csp_container, odbc_driver) - diff --git a/irods_testing_environment/services.py b/irods_testing_environment/services.py index 5cd4ac9..54dbedc 100644 --- a/irods_testing_environment/services.py +++ b/irods_testing_environment/services.py @@ -6,15 +6,18 @@ from . import irods_setup from .install import install -def create_topologies(ctx, - zone_count, - externals_directory=None, - package_directory=None, - package_version=None, - odbc_driver=None, - zone_name='tempZone', - consumer_count=0, - install_packages=True): + +def create_topologies( + ctx, + zone_count, + externals_directory=None, + package_directory=None, + package_version=None, + odbc_driver=None, + zone_name="tempZone", + consumer_count=0, + install_packages=True, +): """Create several generic topologies of iRODS servers with the given inputs. This is a convenience function for standing up multiple, identical iRODS Zones with the @@ -31,18 +34,21 @@ def create_topologies(ctx, Zone """ ctx.compose_project.build() - ctx.compose_project.up(scale_override={ - context.irods_catalog_database_service(): zone_count, - context.irods_catalog_provider_service(): zone_count, - context.irods_catalog_consumer_service(): consumer_count * zone_count - }) + ctx.compose_project.up( + scale_override={ + context.irods_catalog_database_service(): zone_count, + context.irods_catalog_provider_service(): zone_count, + context.irods_catalog_consumer_service(): consumer_count * zone_count, + } + ) if install_packages: install.make_installer(ctx.platform_name()).install_irods_packages( - ctx, - externals_directory=externals_directory, - package_directory=package_directory, - package_version=package_version) + ctx, + externals_directory=externals_directory, + package_directory=package_directory, + package_version=package_version, + ) zone_names = [zone_name for i in range(zone_count)] @@ -52,13 +58,15 @@ def create_topologies(ctx, irods_setup.setup_irods_zones(ctx, zone_info_list, odbc_driver=odbc_driver) -def create_topology(ctx, - externals_directory=None, - package_directory=None, - package_version=None, - odbc_driver=None, - consumer_count=0, - install_packages=True): +def create_topology( + ctx, + externals_directory=None, + package_directory=None, + package_version=None, + odbc_driver=None, + consumer_count=0, + install_packages=True, +): """Create a generic topology of iRODS servers with the given inputs. This is a convenience function for standing up an iRODS Zone with the default @@ -73,21 +81,25 @@ def create_topology(ctx, consumer_count -- number of iRODS Catalog Service Consumers to create and set up for the Zone """ - return create_topologies(ctx, - zone_count=1, - externals_directory=externals_directory, - package_directory=package_directory, - package_version=package_version, - odbc_driver=odbc_driver, - consumer_count=consumer_count, - install_packages=install_packages) - - -def clone_repository_to_container(container, - repo_name, - url_base='https://github.com/irods', - branch=None, - destination_directory=None): + return create_topologies( + ctx, + zone_count=1, + externals_directory=externals_directory, + package_directory=package_directory, + package_version=package_version, + odbc_driver=odbc_driver, + consumer_count=consumer_count, + install_packages=install_packages, + ) + + +def clone_repository_to_container( + container, + repo_name, + url_base="https://github.com/irods", + branch=None, + destination_directory=None, +): """Clone the specified git repository to the specified container. Arguments: @@ -102,16 +114,16 @@ def clone_repository_to_container(container, from . import archive - url = os.path.join(url_base, '.'.join([repo_name, 'git'])) + url = os.path.join(url_base, ".".join([repo_name, "git"])) - repo_path = os.path.abspath(os.path.join( - destination_directory or tempfile.mkdtemp(), - repo_name)) + repo_path = os.path.abspath( + os.path.join(destination_directory or tempfile.mkdtemp(), repo_name) + ) Repo.clone_from(url=url, to_path=repo_path, branch=branch) - archive.copy_archive_to_container(container, - archive.create_archive( - [os.path.abspath(repo_path)], repo_name)) + archive.copy_archive_to_container( + container, archive.create_archive([os.path.abspath(repo_path)], repo_name) + ) return repo_path diff --git a/irods_testing_environment/ssl_setup.py b/irods_testing_environment/ssl_setup.py index 61cd26c..6cda0f3 100644 --- a/irods_testing_environment/ssl_setup.py +++ b/irods_testing_environment/ssl_setup.py @@ -13,24 +13,27 @@ from . import execute from . import json_utils + def generate_ssl_certificate_key(directory=None): - logging.info('generating private key for signing certificate') + logging.info("generating private key for signing certificate") key = rsa.generate_private_key( public_exponent=65537, key_size=2048, ) - keyfile = os.path.join((directory or os.getcwd()), 'server.key') + keyfile = os.path.join((directory or os.getcwd()), "server.key") - logging.info('writing private key to file [{}]'.format(keyfile)) + logging.info("writing private key to file [{}]".format(keyfile)) with open(keyfile, "wb") as f: - f.write(key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), - )) + f.write( + key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) return key, keyfile @@ -38,29 +41,34 @@ def generate_ssl_certificate_key(directory=None): def generate_ssl_self_signed_certificate(key, directory=None): import datetime - logging.info('generating self-signed certificate') + logging.info("generating self-signed certificate") - subject = issuer = x509.Name([ - x509.NameAttribute(NameOID.COUNTRY_NAME, u'US'), - x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, u'North Carolina'), - x509.NameAttribute(NameOID.LOCALITY_NAME, u'Chapel Hill'), - x509.NameAttribute(NameOID.ORGANIZATION_NAME, u'iRODS Consortium'), - ]) + subject = issuer = x509.Name( + [ + x509.NameAttribute(NameOID.COUNTRY_NAME, "US"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "North Carolina"), + x509.NameAttribute(NameOID.LOCALITY_NAME, "Chapel Hill"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "iRODS Consortium"), + ] + ) - cert = x509.CertificateBuilder() \ - .subject_name(subject) \ - .issuer_name(issuer) \ - .public_key(key.public_key()) \ - .serial_number(x509.random_serial_number()) \ - .not_valid_before(datetime.datetime.utcnow()) \ - .not_valid_after(datetime.datetime.utcnow() + datetime.timedelta(days=365)) \ - .add_extension(x509.SubjectAlternativeName([x509.DNSName(u"localhost")]), - critical=False) \ - .sign(key, hashes.SHA256()) + cert = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.utcnow()) + .not_valid_after(datetime.datetime.utcnow() + datetime.timedelta(days=365)) + .add_extension( + x509.SubjectAlternativeName([x509.DNSName("localhost")]), critical=False + ) + .sign(key, hashes.SHA256()) + ) - certfile = os.path.join((directory or os.getcwd()), 'server.crt') + certfile = os.path.join((directory or os.getcwd()), "server.crt") - logging.info('writing cert to file [{}]'.format(certfile)) + logging.info("writing cert to file [{}]".format(certfile)) with open(certfile, "wb") as f: f.write(cert.public_bytes(serialization.Encoding.PEM)) @@ -69,25 +77,31 @@ def generate_ssl_self_signed_certificate(key, directory=None): def generate_ssl_dh_params(generator=2, key_size=1024, directory=None): - logging.info('generating dh params') + logging.info("generating dh params") parameters = dh.generate_parameters(generator=2, key_size=key_size) - dhfile = os.path.join((directory or os.getcwd()), 'dhparams.pem') + dhfile = os.path.join((directory or os.getcwd()), "dhparams.pem") - logging.info('writing dhparams to file [{}]'.format(dhfile)) + logging.info("writing dhparams to file [{}]".format(dhfile)) - with open(dhfile, 'wb') as f: - f.write(parameters.parameter_bytes(encoding=serialization.Encoding.PEM, - format=serialization.ParameterFormat.PKCS3)) + with open(dhfile, "wb") as f: + f.write( + parameters.parameter_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.ParameterFormat.PKCS3, + ) + ) return dhfile -def configure_ssl_on_server(container, - path_to_key_file_on_host, - path_to_cert_file_on_host, - path_to_dhparams_file_on_host): +def configure_ssl_on_server( + container, + path_to_key_file_on_host, + path_to_cert_file_on_host, + path_to_dhparams_file_on_host, +): """Copy SSL files to the container and configure SSL on the iRODS server. Arguments: @@ -99,55 +113,70 @@ def configure_ssl_on_server(container, from . import archive from . import negotiation_key - key_file = os.path.join(context.irods_config(), 'server.key') - dhparams_file = os.path.join(context.irods_config(), 'dhparams.pem') - chain_file = os.path.join(context.irods_config(), 'chain.pem') - cert_file = os.path.join(context.irods_config(), 'server.crt') + key_file = os.path.join(context.irods_config(), "server.key") + dhparams_file = os.path.join(context.irods_config(), "dhparams.pem") + chain_file = os.path.join(context.irods_config(), "chain.pem") + cert_file = os.path.join(context.irods_config(), "server.crt") - irodsctl = os.path.join(context.irods_home(), 'irodsctl') - if execute.execute_command(container, '{} stop'.format(irodsctl), user='irods') != 0: + irodsctl = os.path.join(context.irods_home(), "irodsctl") + if ( + execute.execute_command(container, "{} stop".format(irodsctl), user="irods") + != 0 + ): raise RuntimeError( - 'failed to stop iRODS server before SSL configuration [{}]' - .format(container.name)) - - logging.warning('configuring SSL [{}]'.format(container.name)) - - archive.copy_files_in_container(container, - [(path_to_key_file_on_host, key_file), - (path_to_cert_file_on_host, chain_file), - (path_to_cert_file_on_host, cert_file), - (path_to_dhparams_file_on_host, dhparams_file)]) + "failed to stop iRODS server before SSL configuration [{}]".format( + container.name + ) + ) + + logging.warning("configuring SSL [{}]".format(container.name)) + + archive.copy_files_in_container( + container, + [ + (path_to_key_file_on_host, key_file), + (path_to_cert_file_on_host, chain_file), + (path_to_cert_file_on_host, cert_file), + (path_to_dhparams_file_on_host, dhparams_file), + ], + ) # add certificate chain file, certificate key file, and dh parameters file to iRODS # service account environment file - service_account_irods_env = os.path.join(context.irods_home(), - '.irods', 'irods_environment.json') + service_account_irods_env = os.path.join( + context.irods_home(), ".irods", "irods_environment.json" + ) irods_env = json_utils.get_json_from_file(container, service_account_irods_env) - logging.debug('env [{}] [{}]'.format(json.dumps(irods_env), container.name)) + logging.debug("env [{}] [{}]".format(json.dumps(irods_env), container.name)) - irods_env['irods_client_server_policy'] = 'CS_NEG_REQUIRE' - irods_env['irods_ssl_ca_certificate_file'] = cert_file - irods_env['irods_ssl_certificate_chain_file'] = chain_file - irods_env['irods_ssl_certificate_key_file'] = key_file - irods_env['irods_ssl_dh_params_file'] = dhparams_file - irods_env['irods_ssl_verify_server'] = 'cert' + irods_env["irods_client_server_policy"] = "CS_NEG_REQUIRE" + irods_env["irods_ssl_ca_certificate_file"] = cert_file + irods_env["irods_ssl_certificate_chain_file"] = chain_file + irods_env["irods_ssl_certificate_key_file"] = key_file + irods_env["irods_ssl_dh_params_file"] = dhparams_file + irods_env["irods_ssl_verify_server"] = "cert" - logging.debug('env [{}] [{}]'.format(json.dumps(irods_env), container.name)) + logging.debug("env [{}] [{}]".format(json.dumps(irods_env), container.name)) json_utils.put_json_to_file(container, service_account_irods_env, irods_env) # TODO: consider using a generator to restore the file here... negotiation_key.backup_file(container, context.core_re()) - negotiation_key.configure_ssl_in_server(container, 'CS_NEG_REQUIRE') + negotiation_key.configure_ssl_in_server(container, "CS_NEG_REQUIRE") # start the server again - if execute.execute_command(container, '{} start'.format(irodsctl), user='irods') != 0: + if ( + execute.execute_command(container, "{} start".format(irodsctl), user="irods") + != 0 + ): raise RuntimeError( - 'failed to start iRODS server after SSL configuration [{}]' - .format(container.name)) + "failed to start iRODS server after SSL configuration [{}]".format( + container.name + ) + ) - logging.warning('SSL configured successfully [{}]'.format(container.name)) + logging.warning("SSL configured successfully [{}]".format(container.name)) def configure_ssl_in_zone(docker_client, compose_project): @@ -167,16 +196,20 @@ def configure_ssl_in_zone(docker_client, compose_project): # catalog service consumers depends on being able to communicate with the catalog # service provider. If SSL is not configured first on the catalog service provider # the catalog service consumers will not be able to communicate with it. - csps = compose_project.containers(service_names=[ - context.irods_catalog_provider_service()]) + csps = compose_project.containers( + service_names=[context.irods_catalog_provider_service()] + ) with concurrent.futures.ThreadPoolExecutor() as executor: futures_to_containers = { - executor.submit(configure_ssl_on_server, - docker_client.containers.get(c.name), - key_file, - cert_file, - dhparams_file): c for c in csps + executor.submit( + configure_ssl_on_server, + docker_client.containers.get(c.name), + key_file, + cert_file, + dhparams_file, + ): c + for c in csps } for f in concurrent.futures.as_completed(futures_to_containers): @@ -185,24 +218,31 @@ def configure_ssl_in_zone(docker_client, compose_project): f.result() except Exception as e: - logging.error('exception raised while configuring SSL [{}]' - .format(container.name)) + logging.error( + "exception raised while configuring SSL [{}]".format( + container.name + ) + ) logging.error(e) rc = 1 if rc != 0: - raise RuntimeError('failed to configure SSL on some service') + raise RuntimeError("failed to configure SSL on some service") - cscs = compose_project.containers(service_names=[ - context.irods_catalog_consumer_service()]) + cscs = compose_project.containers( + service_names=[context.irods_catalog_consumer_service()] + ) with concurrent.futures.ThreadPoolExecutor() as executor: futures_to_containers = { - executor.submit(configure_ssl_on_server, - docker_client.containers.get(c.name), - key_file, - cert_file, - dhparams_file): c for c in cscs + executor.submit( + configure_ssl_on_server, + docker_client.containers.get(c.name), + key_file, + cert_file, + dhparams_file, + ): c + for c in cscs } for f in concurrent.futures.as_completed(futures_to_containers): @@ -211,13 +251,16 @@ def configure_ssl_in_zone(docker_client, compose_project): f.result() except Exception as e: - logging.error('exception raised while configuring SSL [{}]' - .format(container.name)) + logging.error( + "exception raised while configuring SSL [{}]".format( + container.name + ) + ) logging.error(e) rc = 1 if rc != 0: - raise RuntimeError('failed to configure SSL on some service') + raise RuntimeError("failed to configure SSL on some service") finally: os.unlink(key_file) diff --git a/irods_testing_environment/test_manager.py b/irods_testing_environment/test_manager.py index 2679137..5bc96db 100644 --- a/irods_testing_environment/test_manager.py +++ b/irods_testing_environment/test_manager.py @@ -3,10 +3,11 @@ import queue import time + class test_manager: """A class that manages a list of tests and `test_runners` for executing tests.""" - def __init__(self, containers, tests, test_type='irods_python_suite'): + def __init__(self, containers, tests, test_type="irods_python_suite"): """Constructor for `test_manager`. A note about passing `None` to `tests`: @@ -22,64 +23,73 @@ def __init__(self, containers, tests, test_type='irods_python_suite'): tests -- list of tests which will run on the `test_runners` test_type -- a string representing the name of the class implementing the test_runner """ - tr_name = '_'.join(['test_runner', test_type]) - tr = eval('.'.join(['test_runner', tr_name])) + tr_name = "_".join(["test_runner", test_type]) + tr = eval(".".join(["test_runner", tr_name])) self.test_runners = [tr(c) for c in containers] self.test_list = tests self.duration = -1 - logging.debug(f'tr:[{tr}], tests:[{tests}], runners:[{self.test_runners}]') - + logging.debug(f"tr:[{tr}], tests:[{tests}], runners:[{self.test_runners}]") def __str__(self): """Return a string representation of the list of `test_runners`.""" return str([str(tr) for tr in self.test_runners]) - def failed_tests(self): """Return a list of failed tests across the managed `test_runners`.""" return [t for tr in self.test_runners for t in tr.failed_tests()] - def return_code(self): """Return int representing the 'overall' return code from a test run. Return 0 if all `test_runners` have a return code of 0. Otherwise, 1. """ - logging.info('[{}]'.format([tr.rc for tr in self.test_runners])) - return 0 if [tr.rc for tr in self.test_runners].count(0) == len(self.test_runners) else 1 - + logging.info("[{}]".format([tr.rc for tr in self.test_runners])) + return ( + 0 + if [tr.rc for tr in self.test_runners].count(0) == len(self.test_runners) + else 1 + ) def result_string(self): """Return string showing tests that passed and failed from each `test_runner.`""" - r = '==== begin test run results ====\n' + r = "==== begin test run results ====\n" tests_were_skipped = False for tr in self.test_runners: r = r + tr.result_string() - tests_were_skipped = tests_were_skipped if tests_were_skipped else len(tr.skipped_tests()) > 0 + tests_were_skipped = ( + tests_were_skipped + if tests_were_skipped + else len(tr.skipped_tests()) > 0 + ) if self.return_code() != 0: - r = r + 'List of failed tests:\n\t{}\n'.format(' '.join([t or 'all tests' for t,_ in self.failed_tests()])) - r = r + 'Return code:[{}]\n'.format(self.return_code()) + r = r + "List of failed tests:\n\t{}\n".format( + " ".join([t or "all tests" for t, _ in self.failed_tests()]) + ) + r = r + "Return code:[{}]\n".format(self.return_code()) elif tests_were_skipped: - r = r + 'Some tests were skipped or did not complete...\n' + r = r + "Some tests were skipped or did not complete...\n" else: - r = r + 'All tests passed! :)\n' + r = r + "All tests passed! :)\n" if self.duration > 0: hours = int(self.duration / 60 / 60) minutes = self.duration / 60 - hours * 60 - r = r + 'time elapsed: [{:>9.4f}]seconds ([{:>4d}]hours [{:>7.4f}]minutes)\n'.format( - self.duration, hours, minutes) + r = ( + r + + "time elapsed: [{:>9.4f}]seconds ([{:>4d}]hours [{:>7.4f}]minutes)\n".format( + self.duration, hours, minutes + ) + ) - r = r + '==== end of test run results ====\n' + r = r + "==== end of test run results ====\n" return r - def run(self, fail_fast=True, options=None, **kwargs): """Run managed `test_runners` in parallel. @@ -94,9 +104,11 @@ def run(self, fail_fast=True, options=None, **kwargs): options = [str() for _ in range(len(self.test_runners))] if not isinstance(options, list) or len(options) != len(self.test_runners): - raise ValueError('options must be a list having a size equal to the number of concurrent executors') + raise ValueError( + "options must be a list having a size equal to the number of concurrent executors" + ) - logging.info(f'options:{options}') + logging.info(f"options:{options}") test_queue = queue.Queue() @@ -111,12 +123,9 @@ def run(self, fail_fast=True, options=None, **kwargs): with concurrent.futures.ThreadPoolExecutor() as executor: futures_to_test_runners = { executor.submit( - tr.run, - test_queue, - fail_fast, - options=options[i], - **kwargs - ): tr for i, tr in enumerate(self.test_runners) + tr.run, test_queue, fail_fast, options=options[i], **kwargs + ): tr + for i, tr in enumerate(self.test_runners) } for f in concurrent.futures.as_completed(futures_to_test_runners): @@ -126,12 +135,12 @@ def run(self, fail_fast=True, options=None, **kwargs): f.result() if tr.rc == 0 and len(tr.failed_tests()) == 0: - logging.error(f'[{tr.name()}]: tests completed successfully') + logging.error(f"[{tr.name()}]: tests completed successfully") else: - logging.error(f'[{tr.name()}]: some tests failed') + logging.error(f"[{tr.name()}]: some tests failed") except Exception as e: - logging.error(f'[{tr.name()}]: exception raised while running test') + logging.error(f"[{tr.name()}]: exception raised while running test") logging.error(e) tr.rc = 1 diff --git a/irods_testing_environment/test_runner.py b/irods_testing_environment/test_runner.py index 2960a34..21f49f3 100644 --- a/irods_testing_environment/test_runner.py +++ b/irods_testing_environment/test_runner.py @@ -8,6 +8,7 @@ from . import context from . import execute + class test_runner: """A class that manages a list of tests and can execute them on a managed container.""" @@ -32,85 +33,84 @@ def __init__(self, executing_container): # Start the duration time at -1 to indicate that no tests have run self.duration = -1 - def __str__(self): """Return a string representation of a map representing the data members.""" - return str({ - 'executing_container': self.name(), - 'return_code': self.rc, - 'test_list': self.tests, - 'passed_tests': self.passed_tests(), - 'failed_tests': self.failed_tests(), - 'duration': self.duration - }) - + return str( + { + "executing_container": self.name(), + "return_code": self.rc, + "test_list": self.tests, + "passed_tests": self.passed_tests(), + "failed_tests": self.failed_tests(), + "duration": self.duration, + } + ) def add_test(self, test): """Append `test` to the test list and return self.""" - logging.info('before [{}]'.format(self)) + logging.info("before [{}]".format(self)) self.tests.append(test) - logging.info('after [{}]'.format(self)) + logging.info("after [{}]".format(self)) return self - def name(self): """Return the name of the executing container.""" return self.executor.name - def test_list(self): """Return the list of tests for which this runner is responsible.""" return self.tests - def passed_tests(self): """Return the list of tests which have been executed and passed.""" return self.passed - def failed_tests(self): """Return the list of tests which have been executed and failed.""" return self.failed - def skipped_tests(self): """Return the list of tests which have not been executed.""" - executed_tests = [t for t,_ in self.passed_tests()] + [t for t,_ in self.failed_tests()] + executed_tests = [t for t, _ in self.passed_tests()] + [ + t for t, _ in self.failed_tests() + ] return list(filter(lambda t: t not in executed_tests, self.test_list())) - def result_string(self): """Return a string representing the results of running the test list.""" - r = '-----\nresults for [{}]\n'.format(self.name()) + r = "-----\nresults for [{}]\n".format(self.name()) - r = r + '\tpassed tests:\n' + r = r + "\tpassed tests:\n" for test, duration in self.passed_tests(): # TODO: a test list of type None may not indicate that all tests ran - r = r + '\t\t[[{:>9.4f}]s]\t[{}]\n'.format(duration, test or 'all tests') + r = r + "\t\t[[{:>9.4f}]s]\t[{}]\n".format(duration, test or "all tests") - r = r + '\tskipped tests:\n' + r = r + "\tskipped tests:\n" for t in self.skipped_tests(): # TODO: a test list of type None may not indicate that all tests ran - r = r + '\t\t[{}]\n'.format(t or 'all tests') + r = r + "\t\t[{}]\n".format(t or "all tests") - r = r + '\tfailed tests:\n' + r = r + "\tfailed tests:\n" for test, duration in self.failed_tests(): # TODO: a test list of type None may not indicate that all tests ran - r = r + '\t\t[[{:>9.4f}]s]\t[{}]\n'.format(duration, test or 'all tests') + r = r + "\t\t[[{:>9.4f}]s]\t[{}]\n".format(duration, test or "all tests") - r = r + '\treturn code:[{}]\n'.format(self.rc) + r = r + "\treturn code:[{}]\n".format(self.rc) if self.duration > 0: hours = int(self.duration / 60 / 60) minutes = self.duration / 60 - hours * 60 - r = r + '\ttime elapsed: [{:9.4}]seconds ([{:4}]hours [{:9.4}]minutes)\n'.format( - self.duration, hours, minutes) + r = ( + r + + "\ttime elapsed: [{:9.4}]seconds ([{:4}]hours [{:9.4}]minutes)\n".format( + self.duration, hours, minutes + ) + ) - r = r + '-----\n' + r = r + "-----\n" return r - def run(self, test_queue, fail_fast=True, **kwargs): """Execute tests from `test_queue` in executing container. @@ -129,7 +129,7 @@ def run(self, test_queue, fail_fast=True, **kwargs): t = test_queue.get(block=False) self.add_test(t) - logging.warning(f'[{self.name()}]: running test [{t}]') + logging.warning(f"[{self.name()}]: running test [{t}]") start = time.time() @@ -141,47 +141,53 @@ def run(self, test_queue, fail_fast=True, **kwargs): duration = end - start - logging.info(f'[{self.name()}]: cmd [{ec}] [{cmd}]') + logging.info(f"[{self.name()}]: cmd [{ec}] [{cmd}]") if ec == 0: self.passed_tests().append((t, duration)) - logging.error(f'[{self.name()}]: test passed [[{duration:>9.4f}]s] [{t or "all tests"}]') + logging.error( + f'[{self.name()}]: test passed [[{duration:>9.4f}]s] [{t or "all tests"}]' + ) else: self.rc = ec self.failed_tests().append((t, duration)) - logging.error(f'[{self.name()}]: test failed [[{duration:>9.4f}]s] [{t or "all tests"}]') + logging.error( + f'[{self.name()}]: test failed [[{duration:>9.4f}]s] [{t or "all tests"}]' + ) if fail_fast: - raise RuntimeError(f'[{self.name()}]: command failed [{cmd}]') + raise RuntimeError(f"[{self.name()}]: command failed [{cmd}]") except queue.Empty: - logging.info(f'[{self.name()}]: Queue is empty!') + logging.info(f"[{self.name()}]: Queue is empty!") run_end = time.time() self.duration = run_end - run_start if self.rc != 0: - logging.error('[{}]: tests that failed [{}]'.format(self.name(), self.failed_tests())) - + logging.error( + "[{}]: tests that failed [{}]".format(self.name(), self.failed_tests()) + ) def execute_test(self, test, options=None, **kwargs): """Execute `test` with return the command run and the return code.""" - raise NotImplementedError('test_runner is a base class and should not be used directly') + raise NotImplementedError( + "test_runner is a base class and should not be used directly" + ) class test_runner_irods_python_suite(test_runner): def __init__(self, executing_container): super(test_runner_irods_python_suite, self).__init__(executing_container) - @staticmethod def run_tests_command(container): """Return a list of strings used as a space-delimited invocation of the test runner.""" from . import container_info - return [container_info.python(container), context.run_tests_script()] + return [container_info.python(container), context.run_tests_script()] def execute_test(self, test, options=None): """Execute `test` with `options` and return the command run and the return code. @@ -195,23 +201,23 @@ def execute_test(self, test, options=None): """ cmd = self.run_tests_command(self.executor) - cmd.extend(['--run_python_suite'] if test is None else ['--run_specific_test', test]) + cmd.extend( + ["--run_python_suite"] if test is None else ["--run_specific_test", test] + ) if options: cmd.extend(options) - return cmd, execute.execute_command(self.executor, - ' '.join(cmd), - user='irods', - workdir=context.irods_home()) + return cmd, execute.execute_command( + self.executor, " ".join(cmd), user="irods", workdir=context.irods_home() + ) class test_runner_irods_unit_tests(test_runner): def __init__(self, executing_container): super(test_runner_irods_unit_tests, self).__init__(executing_container) - - def execute_test(self, test, options=None, reporter='junit'): + def execute_test(self, test, options=None, reporter="junit"): """Execute `test` and return the command run and the return code. If `test` is `None`, a `TypeError` is raised because the test runner requires that a @@ -223,25 +229,25 @@ def execute_test(self, test, options=None, reporter='junit'): reporter -- Catch2 reporter to use (options: console, compact, junit, xml) """ if test is None: - raise TypeError('unit tests must be specified by name - try using --tests') + raise TypeError("unit tests must be specified by name - try using --tests") - extension = 'xml' if reporter == 'junit' else 'out' - output_dir = os.path.join(context.irods_home(), 'log') - output_path = os.path.join(output_dir, f'{test}_{reporter}_report.{extension}') + extension = "xml" if reporter == "junit" else "out" + output_dir = os.path.join(context.irods_home(), "log") + output_path = os.path.join(output_dir, f"{test}_{reporter}_report.{extension}") - cmd = [os.path.join(context.unit_tests(), test)] + (options or ['--reporter', reporter, '--out', output_path]) + cmd = [os.path.join(context.unit_tests(), test)] + ( + options or ["--reporter", reporter, "--out", output_path] + ) - return cmd, execute.execute_command(self.executor, - ' '.join(cmd), - user='irods', - workdir=context.irods_home()) + return cmd, execute.execute_command( + self.executor, " ".join(cmd), user="irods", workdir=context.irods_home() + ) class test_runner_irods_plugin_tests(test_runner): def __init__(self, executing_container): super(test_runner_irods_plugin_tests, self).__init__(executing_container) - # TODO: this could likely just be implemented in yet another subclass def stage_test_hook_file_from_repo(self, repo_name, branch=None): """Run the test hook from the specified git repository. @@ -252,10 +258,13 @@ def stage_test_hook_file_from_repo(self, repo_name, branch=None): options -- list of strings representing script options to pass to the run_tests.py script """ from . import services - return os.path.join( - services.clone_repository_to_container(self.executor, repo_name, branch=branch), - 'irods_consortium_continuous_integration_test_hook.py') + return os.path.join( + services.clone_repository_to_container( + self.executor, repo_name, branch=branch + ), + "irods_consortium_continuous_integration_test_hook.py", + ) # TODO: this could likely just be implemented in yet another subclass def stage_custom_test_hook_file(self, path_to_test_hook_on_host): @@ -273,13 +282,14 @@ def stage_custom_test_hook_file(self, path_to_test_hook_on_host): return f - - def execute_test(self, - test, - options=None, - plugin_repo_name=None, - plugin_branch=None, - path_to_test_hook_on_host=None): + def execute_test( + self, + test, + options=None, + plugin_repo_name=None, + plugin_branch=None, + path_to_test_hook_on_host=None, + ): """Execute `test` and return the command run and the return code. If `test` is `None`, the test hook will be run without any options. This is an @@ -297,29 +307,35 @@ def execute_test(self, from . import container_info if path_to_test_hook_on_host: - path_to_test_hook_in_container = self.stage_custom_test_hook_file(path_to_test_hook_on_host) + path_to_test_hook_in_container = self.stage_custom_test_hook_file( + path_to_test_hook_on_host + ) else: - path_to_test_hook_in_container = self.stage_test_hook_file_from_repo(plugin_repo_name, plugin_branch) + path_to_test_hook_in_container = self.stage_test_hook_file_from_repo( + plugin_repo_name, plugin_branch + ) cmd = [container_info.python(self.executor), path_to_test_hook_in_container] if test is not None: - cmd.extend(['--test', test]) + cmd.extend(["--test", test]) if test != self.tests[0]: - cmd.append('--skip-setup') + cmd.append("--skip-setup") # Install irods_python_ci_utilities for the first test to run on this executor. This # will be true even if None is the test being run. if len(self.passed_tests()) == 0 and len(self.failed_tests()) == 0: from .install import install - install.install_pip_package_from_repo(self.executor, - 'irods_python_ci_utilities', - url_base='https://github.com/irods', - branch='main') + install.install_pip_package_from_repo( + self.executor, + "irods_python_ci_utilities", + url_base="https://github.com/irods", + branch="main", + ) if options: cmd.extend(options) - return cmd, execute.execute_command(self.executor, ' '.join(cmd)) + return cmd, execute.execute_command(self.executor, " ".join(cmd)) diff --git a/irods_testing_environment/test_utils.py b/irods_testing_environment/test_utils.py index edae554..64f0225 100644 --- a/irods_testing_environment/test_utils.py +++ b/irods_testing_environment/test_utils.py @@ -9,6 +9,7 @@ from . import execute from . import test_manager + def job_name(project_name, prefix=None): """Construct unique job name based on the docker-compose project name. @@ -21,11 +22,12 @@ def job_name(project_name, prefix=None): prefix -- optional prefix for the job name """ import uuid + # TODO: use timestamps, also if prefix: - return '_'.join([prefix, project_name, str(uuid.uuid4())]) + return "_".join([prefix, project_name, str(uuid.uuid4())]) - return '_'.join([project_name, str(uuid.uuid4())]) + return "_".join([project_name, str(uuid.uuid4())]) def make_output_directory(dirname, basename): @@ -45,6 +47,7 @@ def make_output_directory(dirname, basename): return directory + def run_unit_tests(containers, test_list=None, fail_fast=True): """Run a set of tests from the python test suite for iRODS. @@ -56,7 +59,7 @@ def run_unit_tests(containers, test_list=None, fail_fast=True): """ tests = test_list or get_unit_test_list(containers[0]) - tm = test_manager.test_manager(containers, tests, test_type='irods_unit_tests') + tm = test_manager.test_manager(containers, tests, test_type="irods_unit_tests") try: tm.run(fail_fast) @@ -67,12 +70,14 @@ def run_unit_tests(containers, test_list=None, fail_fast=True): return tm.return_code() -def run_plugin_tests(containers, - plugin_name, - path_to_test_hook_on_host=None, - test_list=None, - options=None, - fail_fast=True): +def run_plugin_tests( + containers, + plugin_name, + path_to_test_hook_on_host=None, + test_list=None, + options=None, + fail_fast=True, +): """Run a set of tests from the test hook for the specified iRODS plugin. Arguments: @@ -83,14 +88,18 @@ def run_plugin_tests(containers, options -- list of strings representing script options to pass to the run_tests.py script fail_fast -- if True, stop running after first failure; else, runs all tests """ - tm = test_manager.test_manager(containers, test_list, test_type='irods_plugin_tests') + tm = test_manager.test_manager( + containers, test_list, test_type="irods_plugin_tests" + ) try: - tm.run(fail_fast, - plugin_repo_name=plugin_name, - plugin_branch=None, - path_to_test_hook_on_host=path_to_test_hook_on_host, - options=options) + tm.run( + fail_fast, + plugin_repo_name=plugin_name, + plugin_branch=None, + path_to_test_hook_on_host=path_to_test_hook_on_host, + options=options, + ) finally: logging.error(tm.result_string()) @@ -127,22 +136,29 @@ def run_python_test_suite(container, options=None): container -- target container on which the test script will run options -- list of strings representing script options to pass to the run_tests.py script """ - command = [container_info.python(container), - context.run_tests_script(), - '--run_python_suite'] + command = [ + container_info.python(container), + context.run_tests_script(), + "--run_python_suite", + ] if options: command.extend(options) - ec = execute.execute_command(container, - ' '.join(command), - user='irods', - workdir=context.irods_home(), - stream_output=True) + ec = execute.execute_command( + container, + " ".join(command), + user="irods", + workdir=context.irods_home(), + stream_output=True, + ) if ec != 0: - logging.warning('command exited with error code [{}] [{}] [{}]' - .format(ec, command, container.name)) + logging.warning( + "command exited with error code [{}] [{}] [{}]".format( + ec, command, container.name + ) + ) return ec @@ -154,11 +170,10 @@ def get_unit_test_list(container): container -- target container from which test list will be extracted """ from . import json_utils - return json_utils.get_json_from_file(container, - os.path.join( - context.unit_tests(), - 'unit_tests_list.json') - ) + + return json_utils.get_json_from_file( + container, os.path.join(context.unit_tests(), "unit_tests_list.json") + ) def get_test_list(container): @@ -168,9 +183,7 @@ def get_test_list(container): container -- target container from which test list will be extracted """ from . import json_utils - return json_utils.get_json_from_file(container, - os.path.join( - context.irods_home(), - 'scripts', - 'core_tests_list.json') - ) + + return json_utils.get_json_from_file( + container, os.path.join(context.irods_home(), "scripts", "core_tests_list.json") + ) diff --git a/run_core_tests.py b/run_core_tests.py index 8e8907e..889bed3 100644 --- a/run_core_tests.py +++ b/run_core_tests.py @@ -19,7 +19,9 @@ import cli from irods_testing_environment import logs - parser = argparse.ArgumentParser(description='Run iRODS tests in a consistent environment.') + parser = argparse.ArgumentParser( + description="Run iRODS tests in a consistent environment." + ) cli.add_common_args(parser) cli.add_compose_args(parser) @@ -27,45 +29,53 @@ cli.add_irods_package_args(parser) cli.add_irods_test_args(parser) - parser.add_argument('--use-ssl', - dest='use_ssl', action='store_true', - help=textwrap.dedent('''\ + parser.add_argument( + "--use-ssl", + dest="use_ssl", + action="store_true", + help=textwrap.dedent( + """\ Indicates that SSL should be configured and enabled in the test \ - Zone.''')) + Zone.""" + ), + ) args = parser.parse_args() if not args.package_version and not args.install_packages: - print('--irods-package-version is required when using --use-static-image') + print("--irods-package-version is required when using --use-static-image") exit(1) if args.package_directory and args.package_version: - print('--irods-package-directory and --irods-package-version are incompatible') + print("--irods-package-directory and --irods-package-version are incompatible") exit(1) project_directory = os.path.abspath(args.project_directory or os.getcwd()) if not args.install_packages: - os.environ['dockerfile'] = 'release.Dockerfile' + os.environ["dockerfile"] = "release.Dockerfile" if args.package_version: - os.environ['irods_package_version'] = args.package_version + os.environ["irods_package_version"] = args.package_version - ctx = context.context(docker.from_env(use_ssh_client=True), - compose.cli.command.get_project( - project_dir=project_directory, - project_name=args.project_name)) + ctx = context.context( + docker.from_env(use_ssh_client=True), + compose.cli.command.get_project( + project_dir=project_directory, project_name=args.project_name + ), + ) if args.output_directory: dirname = args.output_directory else: import tempfile + dirname = tempfile.mkdtemp(prefix=ctx.compose_project.name) job_name = test_utils.job_name(ctx.compose_project.name, args.job_name) output_directory = test_utils.make_output_directory(dirname, job_name) - logs.configure(args.verbosity, os.path.join(output_directory, 'script_output.log')) + logs.configure(args.verbosity, os.path.join(output_directory, "script_output.log")) rc = 0 @@ -74,40 +84,50 @@ try: if args.do_setup: # Bring up the services - logging.debug('bringing up project [{}]'.format(ctx.compose_project.name)) + logging.debug("bringing up project [{}]".format(ctx.compose_project.name)) consumer_count = 0 - services.create_topologies(ctx, - zone_count=args.executor_count, - externals_directory=args.irods_externals_package_directory, - package_directory=args.package_directory, - package_version=args.package_version, - odbc_driver=args.odbc_driver, - consumer_count=consumer_count, - install_packages=args.install_packages) + services.create_topologies( + ctx, + zone_count=args.executor_count, + externals_directory=args.irods_externals_package_directory, + package_directory=args.package_directory, + package_version=args.package_version, + odbc_driver=args.odbc_driver, + consumer_count=consumer_count, + install_packages=args.install_packages, + ) # Configure the containers for running iRODS automated tests - logging.info('configuring iRODS containers for testing') + logging.info("configuring iRODS containers for testing") irods_config.configure_irods_testing(ctx.docker_client, ctx.compose_project) # Get the container on which the command is to be executed containers = [ ctx.docker_client.containers.get( - context.container_name(ctx.compose_project.name, - context.irods_catalog_provider_service(), - service_instance=i + 1) + context.container_name( + ctx.compose_project.name, + context.irods_catalog_provider_service(), + service_instance=i + 1, ) + ) for i in range(args.executor_count) ] - logging.debug('got containers to run on [{}]'.format(container.name for container in containers)) + logging.debug( + "got containers to run on [{}]".format( + container.name for container in containers + ) + ) - options = ['--xml_output'] + options = ["--xml_output"] if args.use_ssl: - options.append('--use_ssl') + options.append("--use_ssl") if args.do_setup: ssl_setup.configure_ssl_in_zone(ctx.docker_client, ctx.compose_project) - rc = test_utils.run_specific_tests(containers, args.tests, [options] * args.executor_count, args.fail_fast) + rc = test_utils.run_specific_tests( + containers, args.tests, [options] * args.executor_count, args.fail_fast + ) except Exception as e: logging.critical(e) @@ -121,25 +141,28 @@ if args.save_logs: try: - logging.error('collecting logs [{}]'.format(output_directory)) + logging.error("collecting logs [{}]".format(output_directory)) # collect the usual logs - logs.collect_logs(ctx.docker_client, ctx.irods_containers(), output_directory) + logs.collect_logs( + ctx.docker_client, ctx.irods_containers(), output_directory + ) # and then the test reports - archive.collect_files_from_containers(ctx.docker_client, - ctx.irods_containers(), - [os.path.join(context.irods_home(), 'test-reports')], - output_directory) + archive.collect_files_from_containers( + ctx.docker_client, + ctx.irods_containers(), + [os.path.join(context.irods_home(), "test-reports")], + output_directory, + ) except Exception as e: logging.error(e) - logging.error('failed to collect some log files') + logging.error("failed to collect some log files") if rc == 0: rc = 1 - if args.cleanup_containers: ctx.compose_project.down(include_volumes=True, remove_image_type=False) diff --git a/run_federation_tests.py b/run_federation_tests.py index a65f32f..6f6820e 100644 --- a/run_federation_tests.py +++ b/run_federation_tests.py @@ -22,7 +22,9 @@ import cli from irods_testing_environment import logs - parser = argparse.ArgumentParser(description='Run iRODS tests in a consistent environment.') + parser = argparse.ArgumentParser( + description="Run iRODS tests in a consistent environment." + ) cli.add_common_args(parser) cli.add_compose_args(parser) @@ -30,45 +32,53 @@ cli.add_irods_package_args(parser) cli.add_irods_test_args(parser) - parser.add_argument('--use-ssl', - dest='use_ssl', action='store_true', - help=textwrap.dedent('''\ + parser.add_argument( + "--use-ssl", + dest="use_ssl", + action="store_true", + help=textwrap.dedent( + """\ Indicates that SSL should be configured and enabled in each Zone.\ - ''')) + """ + ), + ) args = parser.parse_args() if not args.package_version and not args.install_packages: - print('--irods-package-version is required when using --use-static-image') + print("--irods-package-version is required when using --use-static-image") exit(1) if args.package_directory and args.package_version: - print('--irods-package-directory and --irods-package-version are incompatible') + print("--irods-package-directory and --irods-package-version are incompatible") exit(1) project_directory = os.path.abspath(args.project_directory or os.getcwd()) if not args.install_packages: - os.environ['dockerfile'] = 'release.Dockerfile' + os.environ["dockerfile"] = "release.Dockerfile" if args.package_version: - os.environ['irods_package_version'] = args.package_version + os.environ["irods_package_version"] = args.package_version - ctx = context.context(docker.from_env(use_ssh_client=True), - compose.cli.command.get_project( - project_dir=project_directory, - project_name=args.project_name)) + ctx = context.context( + docker.from_env(use_ssh_client=True), + compose.cli.command.get_project( + project_dir=project_directory, project_name=args.project_name + ), + ) if args.output_directory: dirname = args.output_directory else: import tempfile + dirname = tempfile.mkdtemp(prefix=ctx.compose_project.name) job_name = test_utils.job_name(ctx.compose_project.name, args.job_name) output_directory = test_utils.make_output_directory(dirname, job_name) - logs.configure(args.verbosity, os.path.join(output_directory, 'script_output.log')) + logs.configure(args.verbosity, os.path.join(output_directory, "script_output.log")) rc = 0 container = None @@ -76,16 +86,18 @@ try: if args.do_setup: # Bring up the services - logging.debug('bringing up project [{}]'.format(ctx.compose_project.name)) + logging.debug("bringing up project [{}]".format(ctx.compose_project.name)) ctx.compose_project.build() - containers = ctx.compose_project.up(scale_override={ - context.irods_catalog_database_service(): 2, - context.irods_catalog_provider_service(): 2, - context.irods_catalog_consumer_service(): 0 - }) + containers = ctx.compose_project.up( + scale_override={ + context.irods_catalog_database_service(): 2, + context.irods_catalog_provider_service(): 2, + context.irods_catalog_consumer_service(): 0, + } + ) # The catalog consumers are only determined after the containers are running - zone_info_list = irods_setup.get_info_for_zones(ctx, ['tempZone', 'otherZone']) + zone_info_list = irods_setup.get_info_for_zones(ctx, ["tempZone", "otherZone"]) if args.do_setup: if args.install_packages: @@ -93,59 +105,77 @@ ctx, externals_directory=args.irods_externals_package_directory, package_directory=args.package_directory, - package_version=args.package_version) + package_version=args.package_version, + ) for z in zone_info_list: - irods_setup.setup_irods_zone(ctx, - provider_service_instance=z.provider_service_instance, - database_service_instance=z.database_service_instance, - consumer_service_instances=z.consumer_service_instances, - odbc_driver=args.odbc_driver, - zone_name=z.zone_name, - zone_key=z.zone_key, - negotiation_key=z.negotiation_key) + irods_setup.setup_irods_zone( + ctx, + provider_service_instance=z.provider_service_instance, + database_service_instance=z.database_service_instance, + consumer_service_instances=z.consumer_service_instances, + odbc_driver=args.odbc_driver, + zone_name=z.zone_name, + zone_key=z.zone_key, + negotiation_key=z.negotiation_key, + ) federate.form_federation_clique(ctx, zone_info_list) # Configure the containers for running iRODS automated tests - logging.info('configuring iRODS containers for testing') + logging.info("configuring iRODS containers for testing") irods_config.configure_irods_testing(ctx.docker_client, ctx.compose_project) # Get the container on which the command is to be executed container = ctx.docker_client.containers.get( - context.container_name(ctx.compose_project.name, - context.irods_catalog_provider_service(), - service_instance=2) + context.container_name( + ctx.compose_project.name, + context.irods_catalog_provider_service(), + service_instance=2, + ) ) - logging.debug('got container to run on [{}]'.format(container.name)) + logging.debug("got container to run on [{}]".format(container.name)) remote_container = ctx.docker_client.containers.get( - context.container_name(ctx.compose_project.name, - context.irods_catalog_provider_service())) + context.container_name( + ctx.compose_project.name, context.irods_catalog_provider_service() + ) + ) version = irods_config.get_irods_version(remote_container) zone = irods_config.get_irods_zone_name(remote_container) host = context.project_hostnames(ctx.docker_client, ctx.compose_project)[ - context.irods_catalog_provider_container(ctx.compose_project.name)] + context.irods_catalog_provider_container(ctx.compose_project.name) + ] - options = ['--xml_output', '--federation', '.'.join(str(v) for v in version), zone, host] + options = [ + "--xml_output", + "--federation", + ".".join(str(v) for v in version), + zone, + host, + ] if args.use_ssl: - options.append('--use_ssl') + options.append("--use_ssl") if args.do_setup: ssl_setup.configure_ssl_in_zone(ctx.docker_client, ctx.compose_project) # configure federation for testing if args.do_setup: - irods_config.configure_irods_federation_testing(ctx, zone_info_list[0], zone_info_list[1]) - - execute.execute_command(container, 'iadmin lu', user='irods') - execute.execute_command(container, 'iadmin lz', user='irods') - - rc = test_utils.run_specific_tests([container], - args.tests or ['test_federation'], - [options] * args.executor_count, - args.fail_fast) + irods_config.configure_irods_federation_testing( + ctx, zone_info_list[0], zone_info_list[1] + ) + + execute.execute_command(container, "iadmin lu", user="irods") + execute.execute_command(container, "iadmin lz", user="irods") + + rc = test_utils.run_specific_tests( + [container], + args.tests or ["test_federation"], + [options] * args.executor_count, + args.fail_fast, + ) except Exception as e: logging.critical(e) @@ -159,20 +189,24 @@ if args.save_logs: try: - logging.error('collecting logs [{}]'.format(output_directory)) + logging.error("collecting logs [{}]".format(output_directory)) # collect the usual logs - logs.collect_logs(ctx.docker_client, ctx.irods_containers(), output_directory) + logs.collect_logs( + ctx.docker_client, ctx.irods_containers(), output_directory + ) # and then the test reports - archive.collect_files_from_containers(ctx.docker_client, - [container], - [os.path.join(context.irods_home(), 'test-reports')], - output_directory) + archive.collect_files_from_containers( + ctx.docker_client, + [container], + [os.path.join(context.irods_home(), "test-reports")], + output_directory, + ) except Exception as e: logging.error(e) - logging.error('failed to collect some log files') + logging.error("failed to collect some log files") if rc == 0: rc = 1 diff --git a/run_plugin_tests.py b/run_plugin_tests.py index d4a3428..a86c22a 100644 --- a/run_plugin_tests.py +++ b/run_plugin_tests.py @@ -15,7 +15,9 @@ import cli -parser = argparse.ArgumentParser(description='Run iRODS plugin test hooks in a consistent environment.') +parser = argparse.ArgumentParser( + description="Run iRODS plugin test hooks in a consistent environment." +) cli.add_common_args(parser) cli.add_compose_args(parser) @@ -24,48 +26,57 @@ cli.add_irods_test_args(parser) cli.add_irods_plugin_args(parser) -parser.add_argument('--test-hook-path', - metavar='PATH_TO_TEST_HOOK_FILE', - dest='test_hook', - help='Path to local test hook file to run.') - -parser.add_argument('--extra-logs-path', - nargs='?', default=None, const='/var/lib/irods/test-reports', - help='Path to an extra log file or directory to be copied out after tests.') +parser.add_argument( + "--test-hook-path", + metavar="PATH_TO_TEST_HOOK_FILE", + dest="test_hook", + help="Path to local test hook file to run.", +) + +parser.add_argument( + "--extra-logs-path", + nargs="?", + default=None, + const="/var/lib/irods/test-reports", + help="Path to an extra log file or directory to be copied out after tests.", +) args = parser.parse_args() if not args.package_version and not args.install_packages: - print('--irods-package-version is required when using --use-static-image') + print("--irods-package-version is required when using --use-static-image") exit(1) if args.package_directory and args.package_version: - print('--irods-package-directory and --irods-package-version are incompatible') + print("--irods-package-directory and --irods-package-version are incompatible") exit(1) project_directory = os.path.abspath(args.project_directory or os.getcwd()) if not args.install_packages: - os.environ['dockerfile'] = 'release.Dockerfile' + os.environ["dockerfile"] = "release.Dockerfile" if args.package_version: - os.environ['irods_package_version'] = args.package_version + os.environ["irods_package_version"] = args.package_version -ctx = context.context(docker.from_env(use_ssh_client=True), - compose.cli.command.get_project( - project_dir=project_directory, - project_name=args.project_name)) +ctx = context.context( + docker.from_env(use_ssh_client=True), + compose.cli.command.get_project( + project_dir=project_directory, project_name=args.project_name + ), +) if args.output_directory: dirname = args.output_directory else: import tempfile + dirname = tempfile.mkdtemp(prefix=ctx.compose_project.name) job_name = test_utils.job_name(ctx.compose_project.name, args.job_name) output_directory = test_utils.make_output_directory(dirname, job_name) -logs.configure(args.verbosity, os.path.join(output_directory, 'script_output.log')) +logs.configure(args.verbosity, os.path.join(output_directory, "script_output.log")) rc = 0 @@ -73,46 +84,55 @@ if args.do_setup: # Bring up the services consumer_count = 0 - services.create_topologies(ctx, - zone_count=args.executor_count, - externals_directory=args.irods_externals_package_directory, - package_directory=args.package_directory, - package_version=args.package_version, - odbc_driver=args.odbc_driver, - consumer_count=consumer_count, - install_packages=args.install_packages) + services.create_topologies( + ctx, + zone_count=args.executor_count, + externals_directory=args.irods_externals_package_directory, + package_directory=args.package_directory, + package_version=args.package_version, + odbc_driver=args.odbc_driver, + consumer_count=consumer_count, + install_packages=args.install_packages, + ) # Configure the containers for running iRODS automated tests - logging.info('configuring iRODS containers for testing') + logging.info("configuring iRODS containers for testing") irods_config.configure_irods_testing(ctx.docker_client, ctx.compose_project) # Get the container on which the command is to be executed containers = [ ctx.docker_client.containers.get( - context.container_name(ctx.compose_project.name, - context.irods_catalog_provider_service(), - service_instance=i + 1) + context.container_name( + ctx.compose_project.name, + context.irods_catalog_provider_service(), + service_instance=i + 1, ) + ) for i in range(args.executor_count) ] - logging.debug('got containers to run on [{}]'.format(container.name for container in containers)) + logging.debug( + "got containers to run on [{}]".format( + container.name for container in containers + ) + ) plugin_package_directory = os.path.abspath(args.plugin_package_directory) for c in containers: - archive.copy_archive_to_container(c, - archive.create_archive( - [plugin_package_directory], - args.plugin_name)) + archive.copy_archive_to_container( + c, archive.create_archive([plugin_package_directory], args.plugin_name) + ) - options = ['--built_packages_root_directory', plugin_package_directory] + options = ["--built_packages_root_directory", plugin_package_directory] - rc = test_utils.run_plugin_tests(containers, - args.plugin_name, - args.test_hook, - args.tests, - [options] * args.executor_count, - args.fail_fast) + rc = test_utils.run_plugin_tests( + containers, + args.plugin_name, + args.test_hook, + args.tests, + [options] * args.executor_count, + args.fail_fast, + ) except Exception as e: logging.critical(e) @@ -122,29 +142,42 @@ finally: if args.save_logs: try: - logging.warning('collecting logs [{}]'.format(output_directory)) + logging.warning("collecting logs [{}]".format(output_directory)) # collect the usual logs - logs.collect_logs(ctx.docker_client, ctx.irods_containers(), output_directory) + logs.collect_logs( + ctx.docker_client, ctx.irods_containers(), output_directory + ) # and then the test reports - archive.collect_files_from_containers(ctx.docker_client, - ctx.irods_containers(), - [os.path.join(context.irods_home(), 'test-reports')], - output_directory) + archive.collect_files_from_containers( + ctx.docker_client, + ctx.irods_containers(), + [os.path.join(context.irods_home(), "test-reports")], + output_directory, + ) except Exception as e: logging.error(e) - logging.error('failed to collect some log files') + logging.error("failed to collect some log files") if rc == 0: rc = 1 if args.extra_logs_path: try: - logs.collect_logs(ctx.docker_client, ctx.irods_containers(), output_directory, logfile_path = args.extra_logs_path) + logs.collect_logs( + ctx.docker_client, + ctx.irods_containers(), + output_directory, + logfile_path=args.extra_logs_path, + ) except docker.errors.NotFound: - logging.warning('Path in container not found for --extra-logs-path {!r}'.format(args.extra_logs_path)) + logging.warning( + "Path in container not found for --extra-logs-path {!r}".format( + args.extra_logs_path + ) + ) if args.cleanup_containers: ctx.compose_project.down(include_volumes=True, remove_image_type=False) diff --git a/run_topology_tests.py b/run_topology_tests.py index 8710d9e..54dedc9 100644 --- a/run_topology_tests.py +++ b/run_topology_tests.py @@ -19,7 +19,9 @@ from irods_testing_environment import logs import cli - parser = argparse.ArgumentParser(description='Run iRODS tests in a consistent environment.') + parser = argparse.ArgumentParser( + description="Run iRODS tests in a consistent environment." + ) cli.add_common_args(parser) cli.add_compose_args(parser) @@ -27,78 +29,92 @@ cli.add_irods_package_args(parser) cli.add_irods_test_args(parser) - parser.add_argument('run_on', - metavar='', - choices=['provider', 'consumer'], - help=textwrap.dedent('''\ + parser.add_argument( + "run_on", + metavar="", + choices=["provider", "consumer"], + help=textwrap.dedent( + """\ Indicates whether to run tests from provider or from consumer.\ - ''')) - - parser.add_argument('--use-ssl', - dest='use_ssl', action='store_true', - help=textwrap.dedent('''\ + """ + ), + ) + + parser.add_argument( + "--use-ssl", + dest="use_ssl", + action="store_true", + help=textwrap.dedent( + """\ Indicates that SSL should be configured and enabled in the test \ - Zone.''')) + Zone.""" + ), + ) args = parser.parse_args() if not args.package_version and not args.install_packages: - print('--irods-package-version is required when using --use-static-image') + print("--irods-package-version is required when using --use-static-image") exit(1) if args.package_directory and args.package_version: - print('--irods-package-directory and --irods-package-version are incompatible') + print("--irods-package-directory and --irods-package-version are incompatible") exit(1) project_directory = os.path.abspath(args.project_directory or os.getcwd()) if not args.install_packages: - os.environ['dockerfile'] = 'release.Dockerfile' + os.environ["dockerfile"] = "release.Dockerfile" if args.package_version: - os.environ['irods_package_version'] = args.package_version + os.environ["irods_package_version"] = args.package_version - ctx = context.context(docker.from_env(use_ssh_client=True), - compose.cli.command.get_project( - project_dir=project_directory, - project_name=args.project_name)) + ctx = context.context( + docker.from_env(use_ssh_client=True), + compose.cli.command.get_project( + project_dir=project_directory, project_name=args.project_name + ), + ) if args.output_directory: dirname = args.output_directory else: import tempfile + dirname = tempfile.mkdtemp(prefix=ctx.compose_project.name) job_name = test_utils.job_name(ctx.compose_project.name, args.job_name) output_directory = test_utils.make_output_directory(dirname, job_name) - logs.configure(args.verbosity, os.path.join(output_directory, 'script_output.log')) + logs.configure(args.verbosity, os.path.join(output_directory, "script_output.log")) rc = 0 containers = None try: # some constants - zone_name = 'tempZone' + zone_name = "tempZone" consumer_count = 3 if args.do_setup: # Bring up the services - logging.debug('bringing up project [{}]'.format(ctx.compose_project.name)) - services.create_topologies(ctx, - zone_count=args.executor_count, - externals_directory=args.irods_externals_package_directory, - package_directory=args.package_directory, - package_version=args.package_version, - odbc_driver=args.odbc_driver, - consumer_count=consumer_count, - install_packages=args.install_packages) + logging.debug("bringing up project [{}]".format(ctx.compose_project.name)) + services.create_topologies( + ctx, + zone_count=args.executor_count, + externals_directory=args.irods_externals_package_directory, + package_directory=args.package_directory, + package_version=args.package_version, + odbc_driver=args.odbc_driver, + consumer_count=consumer_count, + install_packages=args.install_packages, + ) # Configure the containers for running iRODS automated tests - logging.info('configuring iRODS containers for testing') + logging.info("configuring iRODS containers for testing") irods_config.configure_irods_testing(ctx.docker_client, ctx.compose_project) - run_on_consumer = args.run_on == 'consumer' + run_on_consumer = args.run_on == "consumer" if run_on_consumer: target_service_name = context.irods_catalog_consumer_service() @@ -115,32 +131,44 @@ containers.append( ctx.docker_client.containers.get( - context.container_name(ctx.compose_project.name, - target_service_name, - service_instance=service_instance) + context.container_name( + ctx.compose_project.name, + target_service_name, + service_instance=service_instance, ) + ) + ) + logging.debug( + "got containers to run on [{}]".format( + container.name for container in containers ) - logging.debug('got containers to run on [{}]'.format(container.name for container in containers)) + ) - options_base = ['--xml_output'] - options_base.append('--topology={}'.format('resource' if run_on_consumer else 'icat')) + options_base = ["--xml_output"] + options_base.append( + "--topology={}".format("resource" if run_on_consumer else "icat") + ) hostname_map = context.project_hostnames(ctx.docker_client, ctx.compose_project) if args.use_ssl: - options_base.append('--use_ssl') + options_base.append("--use_ssl") if args.do_setup: ssl_setup.configure_ssl_in_zone(ctx.docker_client, ctx.compose_project) options_list = list() for i in range(args.executor_count): - logging.debug('hostname_map:{}'.format(hostname_map)) + logging.debug("hostname_map:{}".format(hostname_map)) hostnames_option = [ - '--hostnames', + "--hostnames", hostname_map[ # The service instance number is 1-based, so we need to add 1. - context.container_name(ctx.compose_project.name, context.irods_catalog_provider_service(), i + 1) - ] + context.container_name( + ctx.compose_project.name, + context.irods_catalog_provider_service(), + i + 1, + ) + ], ] consumer_hostname_map = {} @@ -148,30 +176,42 @@ container_service_instance = context.service_instance(c.name) # The range of service instances for the catalog service consumers for a given zone will be a # consecutive range of size consumer_count starting with 1. - consumer_service_instances_for_this_zone = range(i * consumer_count + 1, (i + 1) * consumer_count + 1) - logging.debug('consumer_service_instances_for_this_zone:[{}]'.format([inst for inst in consumer_service_instances_for_this_zone])) - container_is_consumer_for_this_zone = \ - context.is_irods_catalog_consumer_container(c) and \ - container_service_instance in consumer_service_instances_for_this_zone + consumer_service_instances_for_this_zone = range( + i * consumer_count + 1, (i + 1) * consumer_count + 1 + ) + logging.debug( + "consumer_service_instances_for_this_zone:[{}]".format( + [inst for inst in consumer_service_instances_for_this_zone] + ) + ) + container_is_consumer_for_this_zone = ( + context.is_irods_catalog_consumer_container(c) + and container_service_instance + in consumer_service_instances_for_this_zone + ) if container_is_consumer_for_this_zone: consumer_hostname_map[c.name] = hostname_map[ context.container_name( ctx.compose_project.name, context.irods_catalog_consumer_service(), - context.service_instance(c.name) - ) - ] + context.service_instance(c.name), + ) + ] # The hostnames cannot be indiscriminately inserted into the hostnames_option because the consumer # containers will not necessarily be in order. Therefore, a map of the container names to hostnames is # built up and sorted by keys to ensure that the hostnames provided to run_tests.py is in order. - hostnames_option.extend([consumer_hostname_map[k] for k in sorted(consumer_hostname_map)]) + hostnames_option.extend( + [consumer_hostname_map[k] for k in sorted(consumer_hostname_map)] + ) options_list.append(options_base + hostnames_option) logging.info(options_list) - rc = test_utils.run_specific_tests(containers, args.tests, options_list, args.fail_fast) + rc = test_utils.run_specific_tests( + containers, args.tests, options_list, args.fail_fast + ) except Exception as e: logging.critical(e) @@ -185,25 +225,28 @@ if args.save_logs: try: - logging.error('collecting logs [{}]'.format(output_directory)) + logging.error("collecting logs [{}]".format(output_directory)) # collect the usual logs - logs.collect_logs(ctx.docker_client, ctx.irods_containers(), output_directory) + logs.collect_logs( + ctx.docker_client, ctx.irods_containers(), output_directory + ) # and then the test reports - archive.collect_files_from_containers(ctx.docker_client, - containers, - [os.path.join(context.irods_home(), 'test-reports')], - output_directory) + archive.collect_files_from_containers( + ctx.docker_client, + containers, + [os.path.join(context.irods_home(), "test-reports")], + output_directory, + ) except Exception as e: logging.error(e) - logging.error('failed to collect some log files') + logging.error("failed to collect some log files") if rc == 0: rc = 1 - if args.cleanup_containers: ctx.compose_project.down(include_volumes=True, remove_image_type=False) diff --git a/run_unit_tests.py b/run_unit_tests.py index 6e231f1..86a71ea 100644 --- a/run_unit_tests.py +++ b/run_unit_tests.py @@ -16,7 +16,9 @@ import cli from irods_testing_environment import logs - parser = argparse.ArgumentParser(description='Run iRODS tests in a consistent environment.') + parser = argparse.ArgumentParser( + description="Run iRODS tests in a consistent environment." + ) cli.add_common_args(parser) cli.add_compose_args(parser) @@ -27,36 +29,39 @@ args = parser.parse_args() if not args.package_version and not args.install_packages: - print('--irods-package-version is required when using --use-static-image') + print("--irods-package-version is required when using --use-static-image") exit(1) if args.package_directory and args.package_version: - print('--irods-package-directory and --irods-package-version are incompatible') + print("--irods-package-directory and --irods-package-version are incompatible") exit(1) project_directory = os.path.abspath(args.project_directory or os.getcwd()) if not args.install_packages: - os.environ['dockerfile'] = 'release.Dockerfile' + os.environ["dockerfile"] = "release.Dockerfile" if args.package_version: - os.environ['irods_package_version'] = args.package_version + os.environ["irods_package_version"] = args.package_version - ctx = context.context(docker.from_env(use_ssh_client=True), - compose.cli.command.get_project( - project_dir=project_directory, - project_name=args.project_name)) + ctx = context.context( + docker.from_env(use_ssh_client=True), + compose.cli.command.get_project( + project_dir=project_directory, project_name=args.project_name + ), + ) if args.output_directory: dirname = args.output_directory else: import tempfile + dirname = tempfile.mkdtemp(prefix=ctx.compose_project.name) job_name = test_utils.job_name(ctx.compose_project.name, args.job_name) output_directory = test_utils.make_output_directory(dirname, job_name) - logs.configure(args.verbosity, os.path.join(output_directory, 'script_output.log')) + logs.configure(args.verbosity, os.path.join(output_directory, "script_output.log")) rc = 0 containers = None @@ -64,28 +69,32 @@ try: if args.do_setup: # Bring up the services - logging.debug('bringing up project [{}]'.format(ctx.compose_project.name)) + logging.debug("bringing up project [{}]".format(ctx.compose_project.name)) consumer_count = 0 - services.create_topologies(ctx, - zone_count=args.executor_count, - externals_directory=args.irods_externals_package_directory, - package_directory=args.package_directory, - package_version=args.package_version, - odbc_driver=args.odbc_driver, - consumer_count=consumer_count, - install_packages=args.install_packages) + services.create_topologies( + ctx, + zone_count=args.executor_count, + externals_directory=args.irods_externals_package_directory, + package_directory=args.package_directory, + package_version=args.package_version, + odbc_driver=args.odbc_driver, + consumer_count=consumer_count, + install_packages=args.install_packages, + ) # Configure the containers for running iRODS automated tests - logging.info('configuring iRODS containers for testing') + logging.info("configuring iRODS containers for testing") irods_config.configure_irods_testing(ctx.docker_client, ctx.compose_project) # Get the container on which the command is to be executed containers = [ ctx.docker_client.containers.get( - context.container_name(ctx.compose_project.name, - context.irods_catalog_provider_service(), - service_instance=i + 1) + context.container_name( + ctx.compose_project.name, + context.irods_catalog_provider_service(), + service_instance=i + 1, ) + ) for i in range(args.executor_count) ] @@ -102,10 +111,12 @@ cli.log_irods_version_and_commit_id(containers[0]) if args.save_logs: - logging.warning('collecting logs [{}]'.format(output_directory)) + logging.warning("collecting logs [{}]".format(output_directory)) # collect the usual logs (unit test reports appear in /var/lib/irods/log for now) - logs.collect_logs(ctx.docker_client, ctx.irods_containers(), output_directory) + logs.collect_logs( + ctx.docker_client, ctx.irods_containers(), output_directory + ) if args.cleanup_containers: ctx.compose_project.down(include_volumes=True, remove_image_type=False) diff --git a/stand_it_up.py b/stand_it_up.py index 7cd2768..7f76445 100644 --- a/stand_it_up.py +++ b/stand_it_up.py @@ -16,60 +16,75 @@ import cli from irods_testing_environment import logs - parser = argparse.ArgumentParser(description='Stand up an iRODS zone.') + parser = argparse.ArgumentParser(description="Stand up an iRODS zone.") cli.add_common_args(parser) cli.add_compose_args(parser) cli.add_database_config_args(parser) cli.add_irods_package_args(parser) - parser.add_argument('--consumer-instance-count', - metavar='IRODS_CATALOG_SERVICE_CONSUMER_INSTANCE_COUNT', - dest='consumer_count', type=int, default=0, - help=textwrap.dedent('''\ - Number of iRODS Catalog Service Consumer service instances.''')) - - parser.add_argument('--use-ssl', - dest='use_ssl', action='store_true', - help=textwrap.dedent('''\ + parser.add_argument( + "--consumer-instance-count", + metavar="IRODS_CATALOG_SERVICE_CONSUMER_INSTANCE_COUNT", + dest="consumer_count", + type=int, + default=0, + help=textwrap.dedent( + """\ + Number of iRODS Catalog Service Consumer service instances.""" + ), + ) + + parser.add_argument( + "--use-ssl", + dest="use_ssl", + action="store_true", + help=textwrap.dedent( + """\ Indicates that SSL should be configured and enabled in the Zone.\ - ''')) + """ + ), + ) args = parser.parse_args() if not args.package_version and not args.install_packages: - print('--irods-package-version is required when using --use-static-image') + print("--irods-package-version is required when using --use-static-image") exit(1) if args.package_directory and args.package_version: - print('--irods-package-directory and --irods-package-version are incompatible') + print("--irods-package-directory and --irods-package-version are incompatible") exit(1) project_directory = os.path.abspath(args.project_directory or os.getcwd()) if not args.install_packages: - os.environ['dockerfile'] = 'release.Dockerfile' + os.environ["dockerfile"] = "release.Dockerfile" if args.package_version: - os.environ['irods_package_version'] = args.package_version + os.environ["irods_package_version"] = args.package_version - ctx = context.context(docker.from_env(use_ssh_client=True), - compose.cli.command.get_project( - project_dir=project_directory, - project_name=args.project_name)) + ctx = context.context( + docker.from_env(use_ssh_client=True), + compose.cli.command.get_project( + project_dir=project_directory, project_name=args.project_name + ), + ) logs.configure(args.verbosity) - logging.debug(f'environment variables:[{os.environ}]') + logging.debug(f"environment variables:[{os.environ}]") # Bring up the services - logging.debug('bringing up project [{}]'.format(ctx.compose_project.name)) - services.create_topology(ctx, - externals_directory=args.irods_externals_package_directory, - package_directory=args.package_directory, - package_version=args.package_version, - odbc_driver=args.odbc_driver, - consumer_count=args.consumer_count, - install_packages=args.install_packages) + logging.debug("bringing up project [{}]".format(ctx.compose_project.name)) + services.create_topology( + ctx, + externals_directory=args.irods_externals_package_directory, + package_directory=args.package_directory, + package_version=args.package_version, + odbc_driver=args.odbc_driver, + consumer_count=args.consumer_count, + install_packages=args.install_packages, + ) if args.use_ssl: ssl_setup.configure_ssl_in_zone(ctx.docker_client, ctx.compose_project)