diff --git a/cli/conf.py b/cli/conf.py index bbd8550015..10844f70da 100644 --- a/cli/conf.py +++ b/cli/conf.py @@ -1,9 +1,10 @@ import ConfigParser - -import clg import os import yaml +import clg +import yamlordereddictloader + from cli import exceptions from cli import utils @@ -43,38 +44,41 @@ class SpecManager(object): SPEC_EXTENSION = '.spec' - def __init__(self, config): - self.config = config + @classmethod + def parse_args(cls, module_name, config, args=None): + """ + Looks for all the specs for specified module + and parses the commandline input arguments accordingly. - def get_specs(self, module_name): + :param module_name: the module name: installer|provisioner|tester + """ + cmd = clg.CommandLine(cls._get_specs(module_name, config)) + return cmd.parse(args) + + @classmethod + def _get_specs(cls, module_name, config): """ Gets specs files as a dict from settings/ folder. :param module_name: the module name: installer|provisioner|tester """ res = {} - for spec_file in self.__get_all_specs(subfolder=module_name): - spec = yaml.load(open(spec_file)) + for spec_file in cls._get_all_specs(config, subfolder=module_name): + spec = yaml.load(open(spec_file), + Loader=yamlordereddictloader.Loader) utils.dict_merge(res, spec) return res - def parse_args(self, module_name, args=None): - """ - Looks for all the specs for specified module - and parses the commandline input arguments accordingly. - """ - cmd = clg.CommandLine(self.get_specs(module_name)) - return cmd.parse(args) - - def __get_all_specs(self, subfolder=None): + @classmethod + def _get_all_specs(cls, config, subfolder=None): root_dir = utils.validate_settings_dir( - self.config.get('defaults', 'settings')) + config.get('defaults', 'settings')) if subfolder: root_dir = os.path.join(root_dir, subfolder) res = [] for dirpath, _, filenames in os.walk(root_dir): for filename in [f for f in filenames - if f.endswith(self.SPEC_EXTENSION)]: + if f.endswith(cls.SPEC_EXTENSION)]: res.append(os.path.join(dirpath, filename)) return res diff --git a/cli/exceptions.py b/cli/exceptions.py index 724006d389..b0e4b1a24b 100644 --- a/cli/exceptions.py +++ b/cli/exceptions.py @@ -35,9 +35,10 @@ def __init__(self, env_var): class IRPlaybookFailedException(IRException): - def __init__(self, playbook): - super(self.__class__, self).__init__( - 'Playbook "%s" failed!' % playbook) + def __init__(self, playbook, message=""): + msg = 'Playbook "%s" failed!' % playbook + msg = msg + message if message else msg + super(self.__class__, self).__init__(msg) class IRYAMLConstructorError(IRException): @@ -57,3 +58,14 @@ def __init__(self, trace_message): class IRNotImplemented(IRException): pass + + +class IRUnknownApplicationException(IRException): + """ + This exceptions is raised when unknown application is + started by user. + """ + def __init__(self, app_name): + self.app_name = app_name + super(IRUnknownApplicationException, self).__init__( + "Application is unknown: '{}'".format(app_name)) diff --git a/cli/execute.py b/cli/execute.py index 35c6ad7c51..2e083c1fce 100644 --- a/cli/execute.py +++ b/cli/execute.py @@ -136,9 +136,11 @@ def execute_ansible(playbook, args): print "" if len(failed_hosts) > 0: - raise Exception(2) + raise exceptions.IRPlaybookFailedException( + playbook, "Failed hosts: %s" % failed_hosts) if len(unreachable_hosts) > 0: - raise Exception(3) + raise exceptions.IRPlaybookFailedException( + playbook, "Unreachable hosts: %s" % unreachable_hosts) def ansible_wrapper(args): diff --git a/cli/install.py b/cli/install.py index 8b1371e4e2..a621f64de0 100755 --- a/cli/install.py +++ b/cli/install.py @@ -38,9 +38,8 @@ def get_args(entry_point, args=None): """ :return args: return loaded arguments from CLI """ - - spec_manager = conf.SpecManager(CONF) - args = spec_manager.parse_args(entry_point, args=args) + # todo(obaranov) remove one-line method. + args = conf.SpecManager.parse_args(entry_point, CONF, args=args) return args diff --git a/cli/provision.py b/cli/provision.py index ae40d58b4b..3b6aa1bef4 100755 --- a/cli/provision.py +++ b/cli/provision.py @@ -1,92 +1,231 @@ #!/usr/bin/env python import os +from logging import WARNING, INFO, DEBUG +from cli.install import set_network_template as set_network import yaml from cli import logger # logger creation is first thing to be done from cli import conf from cli import utils -from cli.install import set_network_template as set_network -from cli.install import get_args, get_settings_dir, set_logger_verbosity +from cli import exceptions import cli.yamls import cli.execute LOG = logger.LOG CONF = conf.config -ENTRY_POINT = 'provisioner' +NON_SETTINGS_OPTIONS = ['command0', 'verbose', 'extra-vars', 'output-file', + 'input-files', 'dry-run', 'cleanup', 'inventory'] + + +class IRFactory(object): + """ + Creates and configures the IR applications. + """ + + @classmethod + def create(cls, app_name, config): + """ + Create the application object + by module name and provided configuration. + """ + if app_name in ["provisioner", ]: + args = conf.SpecManager.parse_args(app_name, config) + cls.configure_environment(args) + setting_dir = utils.validate_settings_dir( + CONF.get('defaults', 'settings')) + app_instance = IRApplication(app_name, setting_dir, args) + + else: + raise exceptions.IRUnknownApplicationException( + "Application is not supported: '{}'".format(app_name)) + return app_instance + + @classmethod + def configure_environment(cls, args): + """ + Performs the environment configuration. + :param args: + :return: + """ + LOG.setLevel((WARNING, INFO)[args.verbose] + if args.verbose < 2 else DEBUG) + + +class IRSubCommand(object): + """ + Represents a command (virsh, ospd, etc) + for the application + """ + + def __init__(self, name, args, settings_dir): + self.name = name + self.args = args + self.settings_dir = settings_dir + + @classmethod + def create(cls, app, settings_dir, args): + """ + Constructs the sub-command. + :param app: tha application name + :param settings_dir: the default application settings dir + :param args: the application args + :return: the IRSubCommand instance. + """ + if args: + settings_dir = os.path.join(settings_dir, + app) + if hasattr(args, "command0"): + settings_dir = os.path.join(settings_dir, + args['command0']) + return cls(args['command0'], args, settings_dir) + + def get_settings_files(self): + """ + Collects all the settings files for given application and sub-command + """ + settings_files = [] + + # first take all the files from the input-files args + for input_file in self.args['input-files'] or []: + settings_files.append(utils.normalize_file(input_file)) + + # get the sub-command yml file + settings_files.append(os.path.join( + self.settings_dir, + self.name + '.yml')) + + settings_files.extend(self._load_yaml_files()) + + LOG.debug("All settings files to be loaded:\n%s" % settings_files) + return settings_files + + def _load_yaml_files(self): + # load directly from args + settings_files = [] + for key, val in vars(self.args).iteritems(): + if val is not None and key not in NON_SETTINGS_OPTIONS: + settings_file = os.path.join( + self.settings_dir, key, val + '.yml') + LOG.debug('Searching settings file for the "%s" key...' % key) + if not os.path.isfile(settings_file): + settings_file = utils.normalize_file(val) + settings_files.append(settings_file) + LOG.debug('"%s" was added to settings ' + 'files list as an argument ' + 'for "%s" key' % (settings_file, key)) + return settings_files + + def get_settings_dict(self): + return {} + + +class VirshCommand(IRSubCommand): + + def get_settings_dict(self): + + # todo(obaranov) this is virsh specific + # rework that and make this part of lookup or something. + image = dict( + name=self.args['image-file'], + base_url=self.args['image-server'] + ) + host = dict( + ssh_host=self.args['host'], + ssh_user=self.args['ssh-user'], + ssh_key_file=self.args['ssh-key'] + ) + + settings_dict = utils.dict_merge( + {'provisioner': {'image': image}}, + {'provisioner': {'hosts': {'host1': host}}}) + + # load network and image settings + for arg_dir in ('network', 'topology'): + with open(set_network(self.args[arg_dir], os.path.join( + self.settings_dir, arg_dir))) as settings_file: + settings = yaml.load(settings_file) + utils.dict_merge(settings_dict, settings) + + return settings_dict + + def _load_yaml_files(self): + # do not load additional yaml files. + return [] + + +class IRApplication(object): + """ + Hold the default application workflow logic. + """ + def __init__(self, name, settings_dir, args): + self.name = name + self.args = args + self.settings_dir = settings_dir + + # todo(obaranov) replace with subcommand factory + self.sub_command = VirshCommand.create(name, settings_dir, args) + + def run(self): + """ + :return: Runs the application + """ + settings = self.collect_settings() + self.dump_settings(settings) + self.execute(settings) + + def collect_settings(self): + settings_files = self.sub_command.get_settings_files() + settings_dict = self.sub_command.get_settings_dict() + + return self.lookup(settings_files, settings_dict) + + def lookup(self, settings_files, settings_dict): + """ + Replaces a setting values with !lookup + in the setting files by values from + other settings files. + """ + cli.yamls.Lookup.settings = utils.generate_settings( + settings_files, + self.args['extra-vars']) + + cli.yamls.Lookup.settings.merge(settings_dict) + cli.yamls.Lookup.in_string_lookup() + + return cli.yamls.Lookup.settings + + def dump_settings(self, settings): + LOG.debug("Dumping settings...") + output = yaml.safe_dump(settings, + default_flow_style=False) + dump_file = self.args['output-file'] + if dump_file: + LOG.debug("Dump file: {}".format(dump_file)) + with open(dump_file, 'w') as output_file: + output_file.write(output) + else: + print output + + def execute(self, settings): + """ + Executes a playbook. + """ + if not self.args['dry-run']: + vars(self.args)['settings'] = yaml.load(yaml.safe_dump( + settings, + default_flow_style=False)) + if not self.args['cleanup']: + vars(self.args)['provision'] = True + + cli.execute.ansible_wrapper(self.args) def main(): - args = get_args(ENTRY_POINT) - - settings_files = [] - - set_logger_verbosity(args.verbose) - - for input_file in args['input-files'] or []: - settings_files.append(utils.normalize_file(input_file)) - - settings_files.append(os.path.join(get_settings_dir(ENTRY_POINT, args), - args["command0"] + '.yml')) - - # todo(aopincar): virsh specific - settings_dict = set_image_source(args) - utils.dict_merge(settings_dict, set_host(args)) - - for arg_dir in ('network', 'topology'): - with open(set_network(args[arg_dir], os.path.join( - get_settings_dir(ENTRY_POINT, args), arg_dir))) as \ - settings_file: - settings = yaml.load(settings_file) - utils.dict_merge(settings_dict, settings) - - LOG.debug("All settings files to be loaded:\n%s" % settings_files) - cli.yamls.Lookup.settings = utils.generate_settings(settings_files, - args['extra-vars']) - # todo(aopincar): virsh specific - cli.yamls.Lookup.settings = cli.yamls.Lookup.settings.merge(settings_dict) - - cli.yamls.Lookup.in_string_lookup() - - LOG.debug("Dumping settings...") - - output = yaml.safe_dump(cli.yamls.Lookup.settings, - default_flow_style=False) - - if args['output-file']: - with open(args['output-file'], 'w') as output_file: - output_file.write(output) - else: - print output - - # playbook execution stage - if not args['dry-run']: - vars(args)['settings'] = yaml.load(yaml.safe_dump( - cli.yamls.Lookup.settings, default_flow_style=False)) - if not args['cleanup']: - vars(args)['provision'] = True - - cli.execute.ansible_wrapper(args) - - -def set_image_source(args): - image = dict( - name=args['image-file'], - base_url=args['image-server'] - ) - return {'provisioner': {'image': image}} - - -def set_host(args): - host = dict( - ssh_host=args['host'], - ssh_user=args['ssh-user'], - ssh_key_file=args['ssh-key'] - ) - return {'provisioner': {'hosts': {'host1': host}}} - + app = IRFactory.create('provisioner', CONF) + app.run() if __name__ == '__main__': main() diff --git a/requirements.txt b/requirements.txt index 6d9c751d64..02bbf3ee6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ PyYAML>=3.11 configure>=0.5 colorlog>=2.6.1 clg>=2.0.0 +yamlordereddictloader>=0.1.1