Testing cleanup

.travis.yml

@@ -3,6 +3,7 @@
 ---
 dist: bionic
 stages:
+  - test upgrades
   - test
   - lint
   - name: release
@@ -19,7 +20,7 @@ services:
 # the `platforms` defined in `kitchen.yml`
 env:
   matrix:
-    - INSTANCE: debian-10-develop-py3
+    - INSTANCE: \(dev\|prod\)-server-debian-10-develop-py3
     # - INSTANCE: ubuntu-1804-develop-py3
     # - INSTANCE: centos-7-develop-py3
     # - INSTANCE: fedora-30-develop-py3
@@ -53,6 +54,19 @@ script:
 
 jobs:
   include:
+    # Test upgrading the Vault binary by setting the pillar value to
+    # an earlier version, performing a converge, restoring the later
+    # pillar value, performing another converge then finally verifying
+    # the install binary is the correct version
+    - stage: test upgrades
+      env:
+        - INSTANCE: install-binary-debian-9-2019-2-py3
+      script:
+        - sed --in-place=.bak 's/1.1.0/1.0.3/' test/salt/pillar/install_binary.sls
+        - bin/kitchen converge ${INSTANCE}
+        - git restore test/salt/pillar/install_binary.sls
+        - bin/kitchen converge ${INSTANCE}
+        - bin/kitchen verify ${INSTANCE}
     # Define the `lint` stage (runs `yamllint` and `commitlint`)
     - stage: lint
       language: node_js

Gemfile

@@ -3,5 +3,3 @@ source "https://rubygems.org"
 gem 'kitchen-docker', '>= 2.9'
 gem 'kitchen-salt', '>= 0.6.0'
 gem 'kitchen-inspec', '>= 1.1'
-gem 'inspec',      '~> 4.16.0'
-

kitchen.yml

@@ -11,14 +11,14 @@ driver:
 # Make sure the platforms listed below match up with
 # the `env.matrix` instances defined in `.travis.yml`
 platforms:
-  - name: amazonlinux
+  - name: amazonlinux-1-2019-2-py2
     driver:
       image: amazonlinux:1
       platform: rhel
       run_command: /sbin/init
       provision_command:
         # install latest stable Salt
-        - curl -L https://bootstrap.saltstack.com | sh -s -- -X
+        - curl -L https://bootstrap.saltstack.com | sh -s -- -X stable 2019.2.0
 
   ## SALT `develop`
   - name: debian-10-develop-py3

test/integration/dev_server/controls/vault_spec.rb

@@ -1,49 +1,19 @@
-describe command('/usr/local/bin/vault -version') do
-  its(:exit_status) { should eq 0 }
-  its(:stderr) { should be_empty }
-  its(:stdout) { should match(/^Vault v[0-9\.]+ \('[0-9a-f]+'\)/) }
+if os[:name] == 'amazon' && os[:release] =~ /^20\d\d/
+  log_command = command('cat /var/log/vault.log')
+else
+  log_command = command('journalctl -u vault')
 end
 
-describe.one do
-  describe file('/etc/systemd/system/vault.service') do
-    it { should be_a_file }
-    its(:content) { should_not match(/syslog/) }
-  end
-
-  describe file('/etc/init/vault.conf') do
-    it { should be_a_file }
-  end
-end
-
-describe service('vault') do
-  it { should be_enabled }
-  it { should be_running }
-end
+include_controls 'vault-server-baseline'
 
 describe file("/etc/vault/conf.d/config.json") do
   it { should_not be_a_file }
 end
 
-describe.one do
-  describe command('journalctl -u vault') do
-    its(:exit_status) { should eq 0 }
-    its(:stderr) { should be_empty }
-    its(:stdout) { should match(/WARNING! dev mode is enabled!/) }
-  end
-
-  describe file('/var/log/vault.log') do
-    it { should be_a_file }
-    its(:content) { should match(/WARNING! dev mode is enabled!/) }
-  end
-end
-
-describe port(8200) do
-  it { should be_listening }
-  its('processes') { should include 'vault' }
-end
-
-describe http('http://127.0.0.1:8200/v1/sys/seal-status') do
-  its('status') { should cmp 200 }
+describe log_command do
+  its(:exit_status) { should eq 0 }
+  its(:stderr) { should be_empty }
+  its(:stdout) { should match(/WARNING! dev mode is enabled!/) }
 end
 
 describe json(content: http('http://127.0.0.1:8200/v1/sys/seal-status').body) do

test/integration/dev_server/inspec.yml

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 # vim: ft=yaml
 ---
-name: dev_server
-title: vault formula
+name: vault-dev-server
+title: Vault Formula Development Server
 maintainer: SaltStack Formulas
 license: Apache-2.0
-summary: Verify that the vault development server is setup and configured correctly
+summary: Verify that the Vault Server (Development) is setup and configured correctly
+version: 0.1
 supports:
   - platform-name: debian
   - platform-name: ubuntu
@@ -16,3 +17,6 @@ supports:
   - platform-name: freebsd
   - platform-name: amazon
   - platform-name: arch
+depends:
+  - name: vault-server-baseline
+    path: test/integration/server-baseline

test/integration/install_binary/inspec.yml

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 # vim: ft=yaml
 ---
-name: install_binary
-title: vault formula
+name: vault-install-binary
+title: Vault Formula Binary Installation
 maintainer: SaltStack Formulas
 license: Apache-2.0
-summary: Verify that the vault binary is installed correctly
+summary: Verify that the Vault binary is installed correctly
+version: 0.1
 supports:
   - platform-name: debian
   - platform-name: ubuntu

test/integration/prod_server/controls/vault_spec.rb

@@ -1,9 +1,11 @@
-describe command('/usr/local/bin/vault -version') do
-  its(:exit_status) { should eq 0 }
-  its(:stderr) { should be_empty }
-  its(:stdout) { should match(/^Vault v[0-9\.]+ \('[0-9a-f]+'\)/) }
+if os[:name] == 'amazon' && os[:release] =~ /^20\d\d/
+  log_command = command('cat /var/log/vault.log')
+else
+  log_command = command('journalctl -u vault')
 end
 
+include_controls 'vault-server-baseline'
+
 describe command('getcap $(readlink -f /usr/local/bin/vault)') do
   its(:exit_status) { should eq 0 }
   its(:stderr) { should be_empty }
@@ -22,42 +24,10 @@
   its('mode') { should cmp '0640' }
 end
 
-describe.one do
-  describe file('/etc/systemd/system/vault.service') do
-    it { should be_a_file }
-    its(:content) { should_not match(/syslog/) }
-  end
-
-  describe file('/etc/init/vault.conf') do
-    it { should be_a_file }
-  end
-end
-
-describe service('vault') do
-  it { should be_enabled }
-  it { should be_running }
-end
-
-describe.one do
-  describe command('journalctl -u vault') do
-    its(:exit_status) { should eq 0 }
-    its(:stderr) { should be_empty }
-    its(:stdout) { should match(/Vault server started/) }
-  end
-
-  describe file('/var/log/vault.log') do
-    it { should be_a_file }
-    its(:content) { should match(/Vault server started/) }
-  end
-end
-
-describe port(8200) do
-  it { should be_listening }
-  its('processes') { should include 'vault' }
-end
-
-describe http('http://127.0.0.1:8200/v1/sys/seal-status') do
-  its('status') { should cmp 200 }
+describe log_command do
+  its(:exit_status) { should eq 0 }
+  its(:stderr) { should be_empty }
+  its(:stdout) { should match(/Vault server started/) }
 end
 
 describe json(content: http('http://127.0.0.1:8200/v1/sys/seal-status').body) do

test/integration/prod_server/inspec.yml

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 # vim: ft=yaml
 ---
-name: prod_server
-title: vault formula
+name: vault-prod-server
+title: Vault Formula Production Server
 maintainer: SaltStack Formulas
 license: Apache-2.0
-summary: Verify that the vault production server is setup and configured correctly
+summary: Verify that the Vault Server (Production) is setup and configured correctly
+version: 0.1
 supports:
   - platform-name: debian
   - platform-name: ubuntu
@@ -16,3 +17,6 @@ supports:
   - platform-name: freebsd
   - platform-name: amazon
   - platform-name: arch
+depends:
+  - name: vault-server-baseline
+    path: test/integration/server-baseline

test/integration/server-baseline/README.md

@@ -0,0 +1,50 @@
+# InSpec Profile: `vault-server-baseline`
+
+This shows the implementation of the `vault-server-baseline` InSpec [profile](https://github.com/inspec/inspec/blob/master/docs/profiles.md).
+
+## Verify a profile
+
+InSpec ships with built-in features to verify a profile structure.
+
+```bash
+$ inspec check vault-server-baseline
+Summary
+-------
+Location: vault-server-baseline
+Profile: profile
+Controls: 4
+Timestamp: 2019-06-24T23:09:01+00:00
+Valid: true
+
+Errors
+------
+
+Warnings
+--------
+```
+
+## Execute a profile
+
+To run all **supported** controls on a local machine use `inspec exec /path/to/profile`.
+
+```bash
+$ inspec exec vault-server-baseline
+..
+
+Finished in 0.0025 seconds (files took 0.12449 seconds to load)
+8 examples, 0 failures
+```
+
+## Execute a specific control from a profile
+
+To run one control from the profile use `inspec exec /path/to/profile --controls name`.
+
+```bash
+$ inspec exec vault-server-baseline --controls package
+.
+
+Finished in 0.0025 seconds (files took 0.12449 seconds to load)
+1 examples, 0 failures
+```
+
+See an [example control here](https://github.com/inspec/inspec/blob/master/examples/profile/controls/example.rb).

test/integration/server-baseline/controls/baseline.rb

@@ -0,0 +1,31 @@
+if os[:name] == 'amazon' && os[:release] =~ /^20\d\d/
+  init_file = '/etc/init/vault.conf'
+  log_command = command('cat /var/log/vault.log')
+else
+  init_file = '/etc/systemd/system/vault.service'
+  log_command = command('journalctl -u vault')
+end
+
+describe command('/usr/local/bin/vault -version') do
+  its(:exit_status) { should eq 0 }
+  its(:stderr) { should be_empty }
+  its(:stdout) { should match(/^Vault v[0-9\.]+ \('[0-9a-f]+'\)/) }
+end
+
+describe file(init_file) do
+  it { should be_a_file }
+end
+
+describe service('vault') do
+  it { should be_enabled }
+  it { should be_running }
+end
+
+describe port(8200) do
+  it { should be_listening }
+  its('processes') { should include 'vault' }
+end
+
+describe http('http://127.0.0.1:8200/v1/sys/seal-status') do
+  its('status') { should cmp 200 }
+end

test/integration/server-baseline/inspec.yml

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+name: vault-server-baseline
+title: Vault Formula Server Baseline
+maintainer: SaltStack Formulas
+license: Apache-2.0
+summary: Verify that the Vault server has a baseline setup
+version: 0.1
+supports:
+  - platform-name: debian
+  - platform-name: ubuntu
+  - platform-name: centos
+  - platform-name: fedora
+  - platform-name: opensuse
+  - platform-name: suse
+  - platform-name: freebsd
+  - platform-name: amazon
+  - platform-name: arch

test/salt/pillar/install_binary.sls

@@ -4,6 +4,5 @@
 vault:
   # test upgrades by doing a double-converge, changing the version pillar
   # between each one
-  # version: 1.0.3
   version: 1.1.0
   verify_download: false