diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..0bbb03cc --- /dev/null +++ b/.gitignore @@ -0,0 +1,122 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a packager +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.kitchen +.kitchen.local.yml +kitchen.local.yml +junit-*.xml + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# Bundler +Gemfile.lock + +# copied `.md` files used for conversion to `.rst` using `m2r` +docs/*.md + +# Vim +*.sw? + +## Collected when centralising formulas (check and sort) +# `collectd-formula` +.pytest_cache/ +/.idea/ +Dockerfile.*_* +ignore/ +tmp/ diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 00000000..bdae9aa9 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +--- +# General overrides used across formulas in the org +Metrics/LineLength: + # Increase from default of `80` + # Based on https://github.com/PyCQA/flake8-bugbear#opinionated-warnings (`B950`) + Max: 88 + +# Any offenses that should be fixed, e.g. collected via. `rubocop --auto-gen-config` diff --git a/.salt-lint b/.salt-lint new file mode 100644 index 00000000..a539954b --- /dev/null +++ b/.salt-lint @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +--- +exclude_paths: [] +skip_list: + # Using `salt-lint` for linting other files as well, such as Jinja macros/templates + - 205 # Use ".sls" as a Salt State file extension + # Skipping `207` and `208` because `210` is sufficient, at least for the time-being + # I.e. Allows 3-digit unquoted codes to still be used, such as `644` and `755` + - 207 # File modes should always be encapsulated in quotation marks + - 208 # File modes should always contain a leading zero +tags: [] +verbosity: 1 diff --git a/.travis.yml b/.travis.yml index c4b0f18b..7ed3d818 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,87 +1,90 @@ # -*- coding: utf-8 -*- # vim: ft=yaml --- +## Machine config dist: bionic -stages: - - test - - lint - - name: release - if: branch = master AND type != pull_request - sudo: required -cache: bundler -language: ruby - services: - docker -# Make sure the instances listed below match up with -# the `platforms` defined in `kitchen.yml` -# NOTE: Please try to select up to six instances that add some meaningful -# testing of the formula's behaviour. If possible, try to refrain from -# the classical "chosing all the instances because I want to test on -# another/all distro/s" trap: it will just add time to the testing (see -# the discussion on #121). As an example, the set chosen below covers -# the most used distros families, systemd and non-systemd and the latest -# three supported Saltstack versions with python2 and 3. -# As for `kitchen.yml`, that should still contain all of the platforms, -# to allow for comprehensive local testing -# Ref: https://github.com/saltstack-formulas/template-formula/issues/118 -# Ref: https://github.com/saltstack-formulas/template-formula/issues/121 -env: - matrix: - - INSTANCE: debian-10-develop-py3 - # - INSTANCE: default-ubuntu-1804-develop-py3 - - INSTANCE: centos-7-develop-py3 - # INSTANCE: default-fedora-30-develop-py3 - # - INSTANCE: default-opensuse-leap-15-develop-py3 - # - INSTANCE: default-amazonlinux-2-develop-py2 - # - INSTANCE: default-arch-base-latest-develop-py2 - # - INSTANCE: default-debian-9-2019-2-py3 - - INSTANCE: ubuntu-1804-2019-2-py3 - - INSTANCE: centos-7-2019-2-py3 - # - INSTANCE: default-fedora-30-2019-2-py3 - - INSTANCE: opensuse-leap-15-2019-2-py3 - # INSTANCE: default-amazonlinux-2-2019-2-py2 - - INSTANCE: arch-base-latest-2019-2-py2 - # - INSTANCE: default-debian-9-2018-3-py2 - # - INSTANCE: default-ubuntu-1604-2018-3-py2 - # - INSTANCE: default-centos-7-2018-3-py2 - # INSTANCE: fedora-29-2018-3-py2 - # INSTANCE: default-opensuse-leap-15-2018-3-py2 - - INSTANCE: amazonlinux-2-2018-3-py2 - # - INSTANCE: default-arch-base-latest-2018-3-py2 - # - INSTANCE: default-debian-8-2017-7-py2 - # - INSTANCE: default-ubuntu-1604-2017-7-py2 - # INSTANCE: centos6-centos-6-2017-7-py2 - # - INSTANCE: default-fedora-29-2017-7-py2 - # - INSTANCE: default-opensuse-leap-15-2017-7-py2 - # - INSTANCE: default-amazonlinux-2-2017-7-py2 - # - INSTANCE: default-arch-base-latest-2017-7-py2 +## Language and cache config +language: ruby +cache: bundler +## Script to run for the test stage script: - - bin/kitchen verify ${INSTANCE} + - bin/kitchen verify "${INSTANCE}" +## Stages and jobs matrix +stages: + - test + - name: release + if: branch = master AND type != pull_request jobs: include: - # Define the `lint` stage (runs `yamllint` and `commitlint`) - - stage: lint - language: node_js + ## Define the test stage that runs the linters (and testing matrix, if applicable) + + # Run all of the linters in a single job + - language: node_js node_js: lts/* + env: Lint + name: 'Lint: salt-lint, yamllint, rubocop & commitlint' before_install: skip script: + # Install and run `salt-lint` + - pip install --user salt-lint + - git ls-files | grep '\.sls$\|\.jinja$\|\.j2$\|\.tmpl$' + | xargs -I {} salt-lint {} # Install and run `yamllint` # Need at least `v1.17.0` for the `yaml-files` setting - pip install --user yamllint>=1.17.0 - yamllint -s . + # Install and run `rubocop` + - gem install rubocop + - rubocop -d # Install and run `commitlint` - npm install @commitlint/config-conventional -D - npm install @commitlint/travis-cli -D - commitlint-travis - # Define the release stage that runs `semantic-release` + + ## Define the rest of the matrix based on Kitchen testing + # Make sure the instances listed below match up with + # the `platforms` defined in `kitchen.yml` + - env: INSTANCE=default-debian-10-develop-py3 + # - env: INSTANCE=default-ubuntu-1804-develop-py3 + - env: INSTANCE=centarch-centos-7-develop-py3 + # - env: INSTANCE=default-fedora-30-develop-py3 + # - env: INSTANCE=default-opensuse-leap-15-develop-py3 + # - env: INSTANCE=default-amazonlinux-2-develop-py2 + # - env: INSTANCE=centarch-arch-base-latest-develop-py2 + # - env: INSTANCE=default-debian-9-2019-2-py3 + - env: INSTANCE=default-ubuntu-1804-2019-2-py3 + - env: INSTANCE=centarch-centos-7-2019-2-py3 + # - env: INSTANCE=default-fedora-30-2019-2-py3 + - env: INSTANCE=default-opensuse-leap-15-2019-2-py3 + # - env: INSTANCE=default-amazonlinux-2-2019-2-py2 + - env: INSTANCE=centarch-arch-base-latest-2019-2-py2 + # - env: INSTANCE=default-debian-9-2018-3-py2 + # - env: INSTANCE=default-ubuntu-1604-2018-3-py2 + # - env: INSTANCE=centarch-centos-7-2018-3-py2 + # - env: INSTANCE=default-fedora-29-2018-3-py2 + # - env: INSTANCE=default-opensuse-leap-15-2018-3-py2 + - env: INSTANCE=default-amazonlinux-2-2018-3-py2 + # - env: INSTANCE=centarch-arch-base-latest-2018-3-py2 + # - env: INSTANCE=default-debian-8-2017-7-py2 + # - env: INSTANCE=default-ubuntu-1604-2017-7-py2 + # - env: INSTANCE=centarch-centos-6-2017-7-py2 + # - env: INSTANCE=default-fedora-29-2017-7-py2 + # - env: INSTANCE=default-opensuse-leap-15-2017-7-py2 + # - env: INSTANCE=default-amazonlinux-2-2017-7-py2 + # - env: INSTANCE=centarch-arch-base-latest-2017-7-py2 + + ## Define the release stage that runs `semantic-release` - stage: release language: node_js node_js: lts/* + env: Release + name: 'Run semantic-release inc. file updates to AUTHORS, CHANGELOG & FORMULA' before_install: skip script: # Update `AUTHORS.md` diff --git a/.yamllint b/.yamllint index e825af60..ded251dc 100644 --- a/.yamllint +++ b/.yamllint @@ -7,20 +7,24 @@ extends: default # Files to ignore completely # 1. All YAML files under directory `node_modules/`, introduced during the Travis run # 2. Any SLS files under directory `test/`, which are actually state files +# 3. Any YAML files under directory `.kitchen/`, introduced during local testing ignore: | node_modules/ test/**/states/**/*.sls + .kitchen/ iscsi/oscodename.yaml yaml-files: # Default settings - '*.yaml' - '*.yml' + - .salt-lint - .yamllint # SaltStack Formulas additional settings - '*.example' - - '*.arch' - test/**/*.sls + # Formula-specific additional settings + - '*.arch' rules: empty-values: @@ -30,3 +34,6 @@ rules: # Increase from default of `80` # Based on https://github.com/PyCQA/flake8-bugbear#opinionated-warnings (`B950`) max: 88 + octal-values: + forbid-implicit-octal: true + forbid-explicit-octal: true diff --git a/Gemfile b/Gemfile index 3b36de32..5a232b61 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,7 @@ -source "https://rubygems.org" +# frozen_string_literal: true + +source 'https://rubygems.org' gem 'kitchen-docker', '>= 2.9' -gem 'kitchen-salt', '>= 0.6.0' gem 'kitchen-inspec', '>= 1.1' - +gem 'kitchen-salt', '>= 0.6.0' diff --git a/bin/kitchen b/bin/kitchen index 1cd44f3a..dcfdb4ca 100755 --- a/bin/kitchen +++ b/bin/kitchen @@ -8,22 +8,25 @@ # this file is here to facilitate running it. # -require "pathname" -ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", - Pathname.new(__FILE__).realpath) +require 'pathname' +ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', + Pathname.new(__FILE__).realpath) -bundle_binstub = File.expand_path("../bundle", __FILE__) +bundle_binstub = File.expand_path('bundle', __dir__) if File.file?(bundle_binstub) if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ load(bundle_binstub) else - abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. -Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") + abort( + 'Your `bin/bundle` was not generated by Bundler, '\ + 'so this binstub cannot run. Replace `bin/bundle` by running '\ + '`bundle binstubs bundler --force`, then run this command again.' + ) end end -require "rubygems" -require "bundler/setup" +require 'rubygems' +require 'bundler/setup' -load Gem.bin_path("test-kitchen", "kitchen") +load Gem.bin_path('test-kitchen', 'kitchen') diff --git a/docs/CONTRIBUTING.rst b/docs/CONTRIBUTING.rst index 0e447189..b7da8f49 100644 --- a/docs/CONTRIBUTING.rst +++ b/docs/CONTRIBUTING.rst @@ -154,56 +154,6 @@ An example of that: ... BREAKING CHANGE: With the removal of all of the `.sls` files under - `iscsi package`, this formula no longer supports the installation of + `template package`, this formula no longer supports the installation of packages. - -Semantic release formulas -------------------------- - -These formulas are already compatible with semantic-release: - -* `apt-formula `_ -* `bind-formula `_ -* `cert-formula `_ -* `chrony-formula `_ -* `collectd-formula `_ -* `cron-formula `_ -* `deepsea-formula `_ -* `dhcpd-formula `_ -* `fail2ban-formula `_ -* `golang-formula `_ -* `grafana-formula `_ -* `influxdb-formula `_ -* `iptables-formula `_ -* `iscsi-formula `_ -* `keepalived-formula `_ -* `libvirt-formula `_ -* `locale-formula `_ -* `logrotate-formula `_ -* `mysql-formula `_ -* `nginx-formula `_ -* `openvpn-formula `_ -* [`WIP `_] `packages-formula `_ -* `php-formula `_ -* `postfix-formula `_ -* `postgres-formula `_ -* `prometheus-formula `_ -* `rkhunter-formula `_ -* `salt-formula `_ -* `sudoers-formula `_ -* `sysctl-formula `_ -* `syslog-ng-formula `_ -* `sysstat-formula `_ -* `systemd-formula `_ -* `timezone-formula `_ -* `ufw-formula `_ -* `users-formula `_ -* `vault-formula `_ -* `vim-formula `_ -* `vsftpd-formula `_ - -Documentation -------------- - -`Documentation contributing guidelines `_ diff --git a/docs/TOFS_pattern.rst b/docs/TOFS_pattern.rst new file mode 100644 index 00000000..4fea5dda --- /dev/null +++ b/docs/TOFS_pattern.rst @@ -0,0 +1,518 @@ +.. _tofs_pattern: + +TOFS: A pattern for using SaltStack +=================================== + +.. list-table:: + :name: tofs-authors + :header-rows: 1 + :stub-columns: 1 + :widths: 2,2,3,2 + + * - + - Person + - Contact + - Date + * - Authored by + - Roberto Moreda + - moreda@allenta.com + - 29/12/2014 + * - Modified by + - Daniel Dehennin + - daniel.dehennin@baby-gnu.org + - 07/02/2019 + * - Modified by + - Imran Iqbal + - https://github.com/myii + - 23/02/2019 + +All that follows is a proposal based on my experience with `SaltStack `_. The good thing of a piece of software like this is that you can "bend it" to suit your needs in many possible ways, and this is one of them. All the recommendations and thoughts are given "as it is" with no warranty of any type. + +.. contents:: **Table of Contents** + +Usage of values in pillar vs templates in ``file_roots`` +-------------------------------------------------------- + +Among other functions, the *master* (or *salt-master*) serves files to the *minions* (or *salt-minions*). The `file_roots `_ is the list of directories used in sequence to find a file when a minion requires it: the first match is served to the minion. Those files could be `state files `_ or configuration templates, among others. + +Using SaltStack is a simple and effective way to implement configuration management, but even in a `non-multitenant `_ scenario, it is not a good idea to generally access some data (e.g. the database password in our `Zabbix `_ server configuration file or the private key of our `Nginx `_ TLS certificate). + +To avoid this situation we can use the `pillar mechanism `_, which is designed to provide controlled access to data from the minions based on some selection rules. As pillar data could be easily integrated in the `Jinja `_ templates, it is a good mechanism to store values to be used in the final rendering of state files and templates. + +There are a variety of approaches on the usage of pillar and templates as seen in the `saltstack-formulas `_' repositories. `Some `_ `developments `_ stress the initial purpose of pillar data into a storage for most of the possible variables for a determined system configuration. This, in my opinion, is shifting too much load from the original template files approach. Adding up some `non-trivial Jinja `_ code as essential part of composing the state file definitely makes SaltStack state files (hence formulas) more difficult to read. The extreme of this approach is that we could end up with a new render mechanism, implemented in Jinja, storing everything needed in pillar data to compose configurations. Additionally, we are establishing a strong dependency with the Jinja renderer. + +In opposition to the *put the code in file_roots and the data in pillars* approach, there is the *pillar as a store for a set of key-values* approach. A full-blown configuration file abstracted in pillar and jinja is complicated to develop, understand and maintain. I think a better and simpler approach is to keep a configuration file templated using just a basic (non-extensive but extensible) set of pillar values. + +On the reusability of SaltStack state files +------------------------------------------- + +There is a brilliant initiative of the SaltStack community called `salt-formulas `_. Their goal is to provide state files, pillar examples and configuration templates ready to be used for provisioning. I am a contributor for two small ones: `zabbix-formula `_ and `varnish-formula `_. + +The `design guidelines `_ for formulas are clear in many aspects and it is a recommended reading for anyone willing to write state files, even non-formulaic ones. + +In the next section, I am going to describe my proposal to extend further the reusability of formulas, suggesting some patterns of usage. + +The Template Override and Files Switch (TOFS) pattern +----------------------------------------------------- + +I understand a formula as a **complete, independent set of SaltStack state and configuration template files sufficient to configure a system**. A system could be something as simple as an NTP server or some other much more complex service that requires many state and configuration template files. + +The customization of a formula should be done mainly by providing pillar data used later to render either the state or the configuration template files. + +Example: NTP before applying TOFS +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Let's work with the NTP example. A basic formula that follows the `design guidelines `_ has the following files and directories tree: + +.. code-block:: + + /srv/saltstack/salt-formulas/ntp-saltstack-formula/ + ntp/ + map.jinja + init.sls + conf.sls + files/ + default/ + etc/ + ntp.conf.jinja + +In order to use it, let's assume a `masterless configuration `_ and this relevant section of ``/etc/salt/minion``: + +.. code-block:: yaml + + pillar_roots: + base: + - /srv/saltstack/pillar + file_client: local + file_roots: + base: + - /srv/saltstack/salt + - /srv/saltstack/salt-formulas/ntp-saltstack-formula + +.. code-block:: jinja + + {#- /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/map.jinja #} + {%- set ntp = salt['grains.filter_by']({ + 'default': { + 'pkg': 'ntp', + 'service': 'ntp', + 'config': '/etc/ntp.conf', + }, + }, merge=salt['pillar.get']('ntp:lookup')) %} + +In ``init.sls`` we have the minimal states required to have NTP configured. In many cases ``init.sls`` is almost equivalent to an ``apt-get install`` or a ``yum install`` of the package. + +.. code-block:: sls + + ## /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/init.sls + {%- from 'ntp/map.jinja' import ntp with context %} + + Install NTP: + pkg.installed: + - name: {{ ntp.pkg }} + + Enable and start NTP: + service.running: + - name: {{ ntp.service }} + - enabled: True + - require: + - pkg: Install NTP package + +In ``conf.sls`` we have the configuration states. In most cases, that is just managing configuration file templates and making them to be watched by the service. + +.. code-block:: sls + + ## /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/conf.sls + include: + - ntp + + {%- from 'ntp/map.jinja' import ntp with context %} + + Configure NTP: + file.managed: + - name: {{ ntp.config }} + - template: jinja + - source: salt://ntp/files/default/etc/ntp.conf.jinja + - watch_in: + - service: Enable and start NTP service + - require: + - pkg: Install NTP package + +Under ``files/default``, there is a structure that mimics the one in the minion in order to avoid clashes and confusion on where to put the needed templates. There you can find a mostly standard template for the configuration file. + +.. code-block:: jinja + + {#- /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/files/default/etc/ntp.conf.jinja #} + {#- Managed by saltstack #} + {#- Edit pillars or override this template in saltstack if you need customization #} + {%- set settings = salt['pillar.get']('ntp', {}) %} + {%- set default_servers = ['0.ubuntu.pool.ntp.org', + '1.ubuntu.pool.ntp.org', + '2.ubuntu.pool.ntp.org', + '3.ubuntu.pool.ntp.org'] %} + + driftfile /var/lib/ntp/ntp.drift + statistics loopstats peerstats clockstats + filegen loopstats file loopstats type day enable + filegen peerstats file peerstats type day enable + filegen clockstats file clockstats type day enable + + {%- for server in settings.get('servers', default_servers) %} + server {{ server }} + {%- endfor %} + + restrict -4 default kod notrap nomodify nopeer noquery + restrict -6 default kod notrap nomodify nopeer noquery + + restrict 127.0.0.1 + restrict ::1 + +With all this, it is easy to install and configure a simple NTP server by just running ``salt-call state.sls ntp.conf``: the package will be installed, the service will be running and the configuration should be correct for most of cases, even without pillar data. + +Alternatively, you can define a highstate in ``/srv/saltstack/salt/top.sls`` and run ``salt-call state.highstate``. + +.. code-block:: sls + + ## /srv/saltstack/salt/top.sls + base: + '*': + - ntp.conf + +**Customizing the formula just with pillar data**, we have the option to define the NTP servers. + +.. code-block:: sls + + ## /srv/saltstack/pillar/top.sls + base: + '*': + - ntp + +.. code-block:: sls + + ## /srv/saltstack/pillar/ntp.sls + ntp: + servers: + - 0.ch.pool.ntp.org + - 1.ch.pool.ntp.org + - 2.ch.pool.ntp.org + - 3.ch.pool.ntp.org + +Template Override +^^^^^^^^^^^^^^^^^ + +If the customization based on pillar data is not enough, we can override the template by creating a new one in ``/srv/saltstack/salt/ntp/files/default/etc/ntp.conf.jinja`` + +.. code-block:: jinja + + {#- /srv/saltstack/salt/ntp/files/default/etc/ntp.conf.jinja #} + {#- Managed by saltstack #} + {#- Edit pillars or override this template in saltstack if you need customization #} + + {#- Some bizarre configurations here #} + {#- ... #} + + {%- for server in settings.get('servers', default_servers) %} + server {{ server }} + {%- endfor %} + +This way we are locally **overriding the template files** offered by the formula in order to make a more complex adaptation. Of course, this could be applied as well to any of the files, including the state files. + +Files Switch +^^^^^^^^^^^^ + +To bring some order into the set of template files included in a formula, as we commented, we suggest having a similar structure to a normal final file system under ``files/default``. + +We can make different templates coexist for different minions, classified by any `grain `_ value, by simply creating new directories under ``files``. This mechanism is based on **using values of some grains as a switch for the directories under** ``files/``. + +If we decide that we want ``os_family`` as switch, then we could provide the formula template variants for both the ``RedHat`` and ``Debian`` families. + +.. code-block:: + + /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/files/ + default/ + etc/ + ntp.conf.jinja + RedHat/ + etc/ + ntp.conf.jinja + Debian/ + etc/ + ntp.conf.jinja + +To make this work we need a ``conf.sls`` state file that takes a list of possible files as the configuration template. + +.. code-block:: sls + + ## /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/conf.sls + include: + - ntp + + {%- from 'ntp/map.jinja' import ntp with context %} + + Configure NTP: + file.managed: + - name: {{ ntp.config }} + - template: jinja + - source: + - salt://ntp/files/{{ grains.get('os_family', 'default') }}/etc/ntp.conf.jinja + - salt://ntp/files/default/etc/ntp.conf.jinja + - watch_in: + - service: Enable and start NTP service + - require: + - pkg: Install NTP package + +If we want to cover the possibility of a special template for a minion identified by ``node01`` then we could have a specific template in ``/srv/saltstack/salt/ntp/files/node01/etc/ntp.conf.jinja``. + +.. code-block:: jinja + + {#- /srv/saltstack/salt/ntp/files/node01/etc/ntp.conf.jinja #} + {#- Managed by saltstack #} + {#- Edit pillars or override this template in saltstack if you need customization #} + + {#- Some crazy configurations here for node01 #} + {#- ... #} + +To make this work we could write a specially crafted ``conf.sls``. + +.. code-block:: sls + + ## /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/conf.sls + include: + - ntp + + {%- from 'ntp/map.jinja' import ntp with context %} + + Configure NTP: + file.managed: + - name: {{ ntp.config }} + - template: jinja + - source: + - salt://ntp/files/{{ grains.get('id') }}/etc/ntp.conf.jinja + - salt://ntp/files/{{ grains.get('os_family') }}/etc/ntp.conf.jinja + - salt://ntp/files/default/etc/ntp.conf.jinja + - watch_in: + - service: Enable and start NTP service + - require: + - pkg: Install NTP package + +Using the ``files_switch`` macro +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We can simplify the ``conf.sls`` with the new ``files_switch`` macro to use in the ``source`` parameter for the ``file.managed`` state. + +.. code-block:: sls + + ## /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/conf.sls + include: + - ntp + + {%- set tplroot = tpldir.split('/')[0] %} + {%- from 'ntp/map.jinja' import ntp with context %} + {%- from 'ntp/libtofs.jinja' import files_switch %} + + Configure NTP: + file.managed: + - name: {{ ntp.config }} + - template: jinja + - source: {{ files_switch(['/etc/ntp.conf.jinja'], + lookup='Configure NTP' + ) + }} + - watch_in: + - service: Enable and start NTP service + - require: + - pkg: Install NTP package + + +* This uses ``config.get``, searching for ``ntp:tofs:source_files:Configure NTP`` to determine the list of template files to use. +* If this returns a result, the default of ``['/etc/ntp.conf.jinja']`` will be appended to it. +* If this does not yield any results, the default of ``['/etc/ntp.conf.jinja']`` will be used. + +In ``libtofs.jinja``, we define this new macro ``files_switch``. + +.. literalinclude:: ../template/libtofs.jinja + :caption: /srv/saltstack/salt-formulas/ntp-saltstack-formula/ntp/libtofs.jinja + :language: jinja + +How to customise the ``source`` further +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The examples below are based on an ``Ubuntu`` minion called ``theminion`` being configured via. pillar. + +Using the default settings of the ``files_switch`` macro above, +the ``source`` will be: + +.. code-block:: sls + + - source: + - salt://ntp/files/theminion/etc/ntp.conf.jinja + - salt://ntp/files/Debian/etc/ntp.conf.jinja + - salt://ntp/files/default/etc/ntp.conf.jinja + +Customise ``files`` +~~~~~~~~~~~~~~~~~~~ + +The ``files`` portion can be customised: + +.. code-block:: sls + + ntp: + tofs: + dirs: + files: files_alt + +Resulting in: + +.. code-block:: sls + + - source: + - salt://ntp/files_alt/theminion/etc/ntp.conf.jinja + - salt://ntp/files_alt/Debian/etc/ntp.conf.jinja + - salt://ntp/files_alt/default/etc/ntp.conf.jinja + +Customise the use of grains +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Grains can be customised and even arbitrary paths can be supplied: + +.. code-block:: sls + + ntp: + tofs: + files_switch: + - any/path/can/be/used/here + - id + - os + - os_family + +Resulting in: + +.. code-block:: sls + + - source: + - salt://ntp/files/any/path/can/be/used/here/etc/ntp.conf.jinja + - salt://ntp/files/theminion/etc/ntp.conf.jinja + - salt://ntp/files/Ubuntu/etc/ntp.conf.jinja + - salt://ntp/files/Debian/etc/ntp.conf.jinja + - salt://ntp/files/default/etc/ntp.conf.jinja + +Customise the ``default`` path +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``default`` portion of the path can be customised: + +.. code-block:: sls + + ntp: + tofs: + dirs: + default: default_alt + +Resulting in: + +.. code-block:: sls + + - source: + ... + - salt://ntp/files/default_alt/etc/ntp.conf.jinja + +Customise the list of ``source_files`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The list of ``source_files`` can be given: + +.. code-block:: sls + + ntp: + tofs: + source_files: + Configure NTP: + - '/etc/ntp.conf_alt.jinja' + +Resulting in: + +.. code-block:: sls + + - source: + - salt://ntp/files/theminion/etc/ntp.conf_alt.jinja + - salt://ntp/files/theminion/etc/ntp.conf.jinja + - salt://ntp/files/Debian/etc/ntp.conf_alt.jinja + - salt://ntp/files/Debian/etc/ntp.conf.jinja + - salt://ntp/files/default/etc/ntp.conf_alt.jinja + - salt://ntp/files/default/etc/ntp.conf.jinja + +Note: This does *not* override the default value. +Rather, the value from the pillar/config is prepended to the default. + +Using sub-directories for ``components`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If your formula is composed of several components, you may prefer to provides files under sub-directories, like in the `systemd-formula `_. + +.. code-block:: + + /srv/saltstack/systemd-formula/ + systemd/ + init.sls + libtofs.jinja + map.jinja + networkd/ + init.sls + files/ + default/ + network/ + 99-default.link + resolved/ + init.sls + files/ + default/ + resolved.conf + timesyncd/ + init.sls + files/ + Arch/ + resolved.conf + Debian/ + resolved.conf + default/ + resolved.conf + Ubuntu/ + resolved.conf + +For example, the following ``formula.component.config`` SLS: + +.. code-block:: sls + + {%- from "formula/libtofs.jinja" import files_switch with context %} + + formula configuration file: + file.managed: + - name: /etc/formula.conf + - user: root + - group: root + - mode: 644 + - template: jinja + - source: {{ files_switch(['formula.conf'], + lookup='formula', + use_subpath=True + ) + }} + +will be rendered on a ``Debian`` minion named ``salt-formula.ci.local`` as: + +.. code-block:: sls + + formula configuration file: + file.managed: + - name: /etc/formula.conf + - user: root + - group: root + - mode: 644 + - template: jinja + - source: + - salt://formula/component/files/salt-formula.ci.local/formula.conf + - salt://formula/component/files/Debian/formula.conf + - salt://formula/component/files/default/formula.conf + - salt://formula/files/salt-formula.ci.local/formula.conf + - salt://formula/files/Debian/formula.conf + - salt://formula/files/default/formula.conf diff --git a/iscsi/initiator/clean.sls b/iscsi/initiator/clean.sls index b4fc9a4f..a2e3c11d 100644 --- a/iscsi/initiator/clean.sls +++ b/iscsi/initiator/clean.sls @@ -6,4 +6,4 @@ include: - .config.clean - .kernel.clean - .make.clean - - .package.clean + - .package.clean diff --git a/iscsi/initiator/config/files/default/iscsi.tmpl b/iscsi/initiator/config/files/default/iscsi.tmpl index d3a68f5c..cfdafe79 100644 --- a/iscsi/initiator/config/files/default/iscsi.tmpl +++ b/iscsi/initiator/config/files/default/iscsi.tmpl @@ -14,11 +14,11 @@ {%- macro iscsi_conf(key, value, spaces=0, last=False) -%} {%- set shift = spaces * ' ' -%} {%- if value is mapping %} -{{shift}}{{ key }} { # nickname +{{ shift }}{{ key }} { # nickname {{ readconf(value, spaces|int+4) }} } {%- elif value is string or value is number %} -{{shift}}{{ key }} = {{"'" if value is not string else ''}}{{ value }}{{"'" if value is not string else ''}} +{{ shift }}{{ key }} = {{ "'" if value is not string else '' }}{{ value }}{{ "'" if value is not string else '' }} {% endif %} {%- endmacro -%} diff --git a/iscsi/initiator/config/files/default/open-iscsi.tmpl b/iscsi/initiator/config/files/default/open-iscsi.tmpl index e09a4dbb..abceecb2 100644 --- a/iscsi/initiator/config/files/default/open-iscsi.tmpl +++ b/iscsi/initiator/config/files/default/open-iscsi.tmpl @@ -13,7 +13,7 @@ {%- macro openiscsi(key, value, spaces=0) -%} {%- set shift = spaces * ' ' -%} -{{shift}}{{ key }} = {{ value ~ '\n' if value is string else '"' ~ value ~ '"\n' }} +{{ shift }}{{ key }} = {{ value ~ '\n' if value is string else '"' ~ value ~ '"\n' }} {%- endmacro %} {{ readconf(data, 0) }} diff --git a/iscsi/isns/clean.sls b/iscsi/isns/clean.sls index b1a0a502..dd17e741 100644 --- a/iscsi/isns/clean.sls +++ b/iscsi/isns/clean.sls @@ -5,4 +5,4 @@ include: - .service.clean - .config.clean - .make.clean - - .package.clean + - .package.clean diff --git a/iscsi/isns/config/files/default/isns.tmpl b/iscsi/isns/config/files/default/isns.tmpl index 0f985f0d..231df000 100644 --- a/iscsi/isns/config/files/default/isns.tmpl +++ b/iscsi/isns/config/files/default/isns.tmpl @@ -13,7 +13,7 @@ {%- macro isns(key, value, spaces=0, last=False) -%} {%- set shift = spaces * ' ' -%} -{{shift}}{{ key }} = {{ value }} +{{ shift }}{{ key }} = {{ value }} {%- endmacro -%} {{ readconf(data, 0) }} diff --git a/iscsi/target/clean.sls b/iscsi/target/clean.sls index b4fc9a4f..a2e3c11d 100644 --- a/iscsi/target/clean.sls +++ b/iscsi/target/clean.sls @@ -6,4 +6,4 @@ include: - .config.clean - .kernel.clean - .make.clean - - .package.clean + - .package.clean diff --git a/iscsi/target/config/files/default/ctld.tmpl b/iscsi/target/config/files/default/ctld.tmpl index 12353e4d..fbabd3c7 100644 --- a/iscsi/target/config/files/default/ctld.tmpl +++ b/iscsi/target/config/files/default/ctld.tmpl @@ -14,11 +14,11 @@ {%- macro ctld(key, value, spaces=0, last=False) -%} {%- set shift = spaces * ' ' -%} {%- if value is mapping %} -{{shift}}{{ key }} { +{{ shift }}{{ key }} { {{ readconf(value, spaces|int+4) }} -{{shift}}} +{{ shift }} } {%- elif value is string or value is number %} -{{shift}}{{ key }} {{ value }} +{{ shift }}{{ key }} {{ value }} {%- endif %} {%- endmacro -%} diff --git a/iscsi/target/config/files/default/ietd.tmpl b/iscsi/target/config/files/default/ietd.tmpl index 55096d5b..802fa233 100644 --- a/iscsi/target/config/files/default/ietd.tmpl +++ b/iscsi/target/config/files/default/ietd.tmpl @@ -14,10 +14,10 @@ {%- macro ietd(key, value, spaces=0, last=False) -%} {%- set shift = spaces * ' ' -%} {%- if value is mapping %} -{{shift}}{{ key }} +{{ shift }}{{ key }} {{ readconf(value, spaces|int+4) }} {%- elif value is string or value is number %} -{{shift}}{{ key }} {{ "'" if value is not string else ''}}{{ value }}{{"'" if value is not string else ''}} +{{ shift }}{{ key }} {{ "'" if value is not string else '' }}{{ value }}{{ "'" if value is not string else '' }} {%- endif %} {%- endmacro -%} diff --git a/iscsi/target/config/files/default/lio.tmpl b/iscsi/target/config/files/default/lio.tmpl index 4bdbc9a8..e03dfbb2 100644 --- a/iscsi/target/config/files/default/lio.tmpl +++ b/iscsi/target/config/files/default/lio.tmpl @@ -1,10 +1,10 @@ -{#######################################################} +{# ################################################### #} {# File managed by Salt at: #} -{# salt://iscsi/target/config/files/default/lio.tmpl #} -{# Your changes may get overwritten. #} -{#######################################################} +{# salt://iscsi/target/config/files/default/lio.tmpl #} +{# Your changes may get overwritten. #} +{# ################################################### #} -{% set arrays = ('fabric_modules', 'storage_objects', 'alua_tpgs', 'targets', 'node_acls', 'luns', 'mapped_luns', 'portals')%} +{% set arrays = ('fabric_modules', 'storage_objects', 'alua_tpgs', 'targets', 'node_acls', 'luns', 'mapped_luns', 'portals') %} {%- if data and component -%} @@ -29,7 +29,7 @@ {{ shift ~ ' }' if last else shift ~ ' },\n' }} {%- elif value is string or value is number %} {{ shift }}"{{ key }}": {{ '' if value is number else '"' }}{{ value }}{{ '' if value is number else '"' }}{{ '' if last else ',' -}} -{{ '\n' ~ shift ~ '}' if last and parent|lower in ('tpgs',) else '' -}} +{{ '\n' ~ shift ~ '}' if last and parent|lower in ('tpgs',) else '' -}} {%- endif %} {%- endmacro -%} diff --git a/iscsi/target/config/files/default/tgtd.tmpl b/iscsi/target/config/files/default/tgtd.tmpl index 67ced108..1be4bf32 100644 --- a/iscsi/target/config/files/default/tgtd.tmpl +++ b/iscsi/target/config/files/default/tgtd.tmpl @@ -14,12 +14,12 @@ {%- macro tgtd(key, value, spaces=0, last=False) -%} {%- set shift = spaces * ' ' -%} {%- if value is mapping %} -{{shift}}{{ '<' ~ key ~ '>' }} +{{ shift }}{{ '<' ~ key ~ '>' }} {{ readconf(value, spaces|int+4) }} -{{shift}}{{ '' }} +{{ shift }}{{ '' }} {% elif value is string or value is number %} -{{shift}}{{ key }} {{ value }} +{{ shift }}{{ key }} {{ value }} {%- endif %} {%- endmacro -%} diff --git a/kitchen.yml b/kitchen.yml index 26c494d4..500d0fa6 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -195,14 +195,15 @@ verifier: suites: - name: default excludes: + - centos-7-develop-py3 + - arch-base-latest-develop-py2 - centos-7-2019-2-py3 - arch-base-latest-2019-2-py2 - - centos-7-develop-py3 + - centos-7-2018-3-py2 + - arch-base-latest-2018-3-py2 + - centos-6-2017-7-py2 + - arch-base-latest-2017-7-py2 provisioner: - dependencies: - - name: lvm - repo: git - source: https://github.com/saltstack-formulas/lvm-formula.git state_top: base: '*': @@ -215,28 +216,29 @@ suites: top.sls: base: '*': - - iscsi - lvm + - iscsi pillars_from_files: - iscsi.sls: test/salt/pillar/pillar.example lvm.sls: test/salt/pillar/lvm.example + iscsi.sls: test/salt/pillar/pillar.example + dependencies: + - name: lvm + repo: git + source: https://github.com/saltstack-formulas/lvm-formula.git verifier: inspec_tests: - path: test/integration/default - - - name: centarch # skip 'target' service on travis + - name: centarch includes: - - arch-base-latest-2019-2-py2 - - centos-7-2019-2-py3 - centos-7-develop-py3 + - arch-base-latest-develop-py2 + - centos-7-2019-2-py3 + - arch-base-latest-2019-2-py2 + - centos-7-2018-3-py2 + - arch-base-latest-2018-3-py2 + - centos-6-2017-7-py2 + - arch-base-latest-2017-7-py2 provisioner: - dependencies: - - name: lvm - repo: git - source: https://github.com/saltstack-formulas/lvm-formula.git - - name: users # for archlinux - repo: git - source: https://github.com/saltstack-formulas/users-formula.git state_top: base: '*': @@ -251,12 +253,19 @@ suites: base: '*': - users - - iscsi - lvm + - iscsi pillars_from_files: users.sls: test/salt/pillar/users.arch - iscsi.sls: test/salt/pillar/pillar.travis lvm.sls: test/salt/pillar/lvm.example + iscsi.sls: test/salt/pillar/pillar.travis + dependencies: + - name: users + repo: git + source: https://github.com/saltstack-formulas/users-formula.git + - name: lvm + repo: git + source: https://github.com/saltstack-formulas/lvm-formula.git verifier: inspec_tests: - path: test/integration/default diff --git a/test/integration/default/controls/config_spec.rb b/test/integration/default/controls/config_spec.rb index 5701bb22..7f58ee7e 100644 --- a/test/integration/default/controls/config_spec.rb +++ b/test/integration/default/controls/config_spec.rb @@ -1,4 +1,5 @@ +# frozen_string_literal: true + control 'iscsi configuration' do title 'should match desired lines' - end diff --git a/test/integration/default/controls/packages_spec.rb b/test/integration/default/controls/packages_spec.rb index 4884bbdf..0be067aa 100644 --- a/test/integration/default/controls/packages_spec.rb +++ b/test/integration/default/controls/packages_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Overide by OS control 'template package' do title 'should be installed' diff --git a/test/integration/default/controls/services_spec.rb b/test/integration/default/controls/services_spec.rb index 6db06ca2..1c268031 100644 --- a/test/integration/default/controls/services_spec.rb +++ b/test/integration/default/controls/services_spec.rb @@ -1,6 +1,7 @@ +# frozen_string_literal: true + # Overide by OS control 'iscsi service' do impact 0.5 title 'should be running and enabled' - end diff --git a/test/integration/default/controls/subcomponent_config_spec.rb b/test/integration/default/controls/subcomponent_config_spec.rb index fb1535d7..9228b0e4 100644 --- a/test/integration/default/controls/subcomponent_config_spec.rb +++ b/test/integration/default/controls/subcomponent_config_spec.rb @@ -1,4 +1,5 @@ +# frozen_string_literal: true + control 'iscsi subcomponent configuration' do title 'should match desired lines' - end