diff --git a/Gemfile b/Gemfile index 2db1fb1..372140e 100644 --- a/Gemfile +++ b/Gemfile @@ -12,13 +12,13 @@ group :test do gem 'rake' gem 'puppet-lint' - gem 'rspec-puppet', :git => 'https://github.com/rodjek/rspec-puppet.git' + gem 'rspec-puppet' gem 'puppet-syntax' gem 'puppetlabs_spec_helper' gem 'simplecov' gem 'simplecov-console' gem 'metadata-json-lint' - gem 'vault' + gem 'vault', '>= 0.13.0' gem 'debouncer' end @@ -29,4 +29,6 @@ group :development do gem 'github_changelog_generator' gem 'activesupport', '< 5' gem 'pdk' + gem 'pry' + gem 'rb-readline' end diff --git a/Gemfile.lock b/Gemfile.lock index baf4062..b5fef77 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,10 +1,3 @@ -GIT - remote: https://github.com/rodjek/rspec-puppet.git - revision: 491b7c8d6fe2c7fea615ed81aa90d96044b88a1d - specs: - rspec-puppet (2.7.5) - rspec - GEM remote: http://rubygems.org/ specs: @@ -13,24 +6,27 @@ GEM minitest (~> 5.1) thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) - addressable (2.6.0) - public_suffix (>= 2.0.2, < 4.0) + addressable (2.7.0) + public_suffix (>= 2.0.2, < 5.0) ansi (1.5.0) ast (2.4.0) - aws-sigv4 (1.0.3) + aws-eventstream (1.0.3) + aws-sigv4 (1.1.0) + aws-eventstream (~> 1.0, >= 1.0.2) childprocess (0.7.1) ffi (~> 1.0, >= 1.0.11) + coderay (1.1.2) concurrent-ruby (1.1.5) - cri (2.15.7) + cri (2.15.9) debouncer (0.2.2) deep_merge (1.2.1) diff-lcs (1.3) - docile (1.3.1) - domain_name (0.5.20180417) + docile (1.3.2) + domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - equatable (0.5.0) - facter (2.5.1) - faraday (0.15.4) + equatable (0.6.1) + facter (2.5.6) + faraday (0.17.1) multipart-post (>= 1.2, < 3) faraday-http-cache (2.0.0) faraday (~> 0.8) @@ -39,91 +35,96 @@ GEM gettext (3.2.9) locale (>= 2.0.5) text (>= 1.3.0) - gettext-setup (0.30) + gettext-setup (0.31) fast_gettext (~> 1.1.0) gettext (>= 3.0.2) locale - github_changelog_generator (1.14.3) + github_changelog_generator (1.15.0) activesupport faraday-http-cache multi_json octokit (~> 4.6) - rainbow (>= 2.1) + rainbow (>= 2.2.1) rake (>= 10.0) - retriable (~> 2.1) - hiera (3.5.0) - hirb (0.7.3) + retriable (~> 3.0) + hiera (3.6.0) hitimes (1.3.0) - hocon (1.2.5) + hocon (1.3.0) + http-accept (1.7.0) http-cookie (1.0.3) domain_name (~> 0.5) httpclient (2.8.3) i18n (0.9.5) concurrent-ruby (~> 1.0) - json (2.2.0) + jaro_winkler (1.5.4) + json (2.3.0) json-schema (2.8.0) addressable (>= 2.4) json_pure (2.1.0) locale (2.1.2) - metaclass (0.0.4) metadata-json-lint (2.2.0) json-schema (~> 2.8) spdx-licenses (~> 1.0) - mime-types (3.2.2) + method_source (0.9.2) + mime-types (3.3) mime-types-data (~> 3.2015) - mime-types-data (3.2019.0331) - minitar (0.6.1) - minitest (5.11.3) - mocha (1.1.0) - metaclass (~> 0.0.1) - multi_json (1.13.1) + mime-types-data (3.2019.1009) + minitar (0.9) + minitest (5.13.0) + mocha (1.10.2) + multi_json (1.14.1) multipart-post (2.1.1) - necromancer (0.4.0) - net-ssh (4.2.0) + necromancer (0.5.1) netrc (0.11.0) octokit (4.14.0) sawyer (~> 0.8.0, >= 0.5.3) - parallel (1.17.0) - parser (2.5.1.2) + parallel (1.19.1) + parser (2.6.5.0) ast (~> 2.4.0) - pastel (0.7.2) - equatable (~> 0.5.0) - tty-color (~> 0.4.0) + pastel (0.7.3) + equatable (~> 0.6) + tty-color (~> 0.5) pathspec (0.2.1) - pdk (1.10.0) + pdk (1.15.0) bundler (>= 1.15.0, < 3.0.0) childprocess (~> 0.7.1) - cri (>= 2.10.1, < 2.16.0) + concurrent-ruby (~> 1.1.5) + cri (~> 2.10) deep_merge (~> 1.1) diff-lcs (= 1.3) + facter (~> 2.5.1) ffi (~> 1.9.0) gettext-setup (~> 0.24) hitimes (= 1.3.0) + httpclient (~> 2.8.3) json-schema (= 2.8.0) json_pure (~> 2.1.0) - minitar (~> 0.6.1) - net-ssh (~> 4.2.0) + minitar (~> 0.6) pathspec (~> 0.2.1) - tty-prompt (= 0.13.1) - tty-spinner (= 0.5.0) - tty-which (= 0.3.0) - powerpack (0.1.2) - public_suffix (3.1.0) - puppet (6.4.2) + tty-prompt (~> 0.13) + tty-spinner (~> 0.5) + tty-which (~> 0.3) + pry (0.12.2) + coderay (~> 1.1.0) + method_source (~> 0.9.0) + public_suffix (4.0.1) + puppet (6.11.1) + concurrent-ruby (~> 1.0) + deep_merge (~> 1.0) facter (> 2.0.1, < 4) - fast_gettext (~> 1.1.2) + fast_gettext (~> 1.1) hiera (>= 3.2.1, < 4) httpclient (~> 2.8) locale (~> 2.1) multi_json (~> 1.10) puppet-resource_api (~> 1.5) semantic_puppet (~> 1.0) - puppet-blacksmith (4.1.2) + puppet-blacksmith (5.0.0) rest-client (~> 2.0) - puppet-lint (2.3.6) - puppet-resource_api (1.8.1) + puppet-lint (2.4.2) + puppet-resource_api (1.8.7) hocon (>= 1.0) - puppet-syntax (2.4.3) + puppet-syntax (2.6.0) rake puppetlabs_spec_helper (2.14.1) mocha (~> 1.0) @@ -131,76 +132,82 @@ GEM puppet-lint (~> 2.0) puppet-syntax (~> 2.0) rspec-puppet (~> 2.0) - rainbow (2.2.2) - rake - rake (12.3.2) - rest-client (2.0.2) + rainbow (3.0.0) + rake (13.0.1) + rb-readline (0.5.5) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - retriable (2.1.0) - rspec (3.8.0) - rspec-core (~> 3.8.0) - rspec-expectations (~> 3.8.0) - rspec-mocks (~> 3.8.0) - rspec-core (3.8.0) - rspec-support (~> 3.8.0) - rspec-expectations (3.8.3) + retriable (3.1.2) + rspec (3.9.0) + rspec-core (~> 3.9.0) + rspec-expectations (~> 3.9.0) + rspec-mocks (~> 3.9.0) + rspec-core (3.9.0) + rspec-support (~> 3.9.0) + rspec-expectations (3.9.0) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.8.0) - rspec-mocks (3.8.0) + rspec-support (~> 3.9.0) + rspec-mocks (3.9.0) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.8.0) - rspec-support (3.8.0) - rubocop (0.49.1) + rspec-support (~> 3.9.0) + rspec-puppet (2.7.8) + rspec + rspec-support (3.9.0) + rubocop (0.77.0) + jaro_winkler (~> 1.5.1) parallel (~> 1.10) - parser (>= 2.3.3.1, < 3.0) - powerpack (~> 0.1) - rainbow (>= 1.99.1, < 3.0) + parser (>= 2.6) + rainbow (>= 2.2.2, < 4.0) ruby-progressbar (~> 1.7) - unicode-display_width (~> 1.0, >= 1.0.1) - rubocop-rspec (1.16.0) - rubocop (>= 0.49.0) + unicode-display_width (>= 1.4.0, < 1.7) + rubocop-rspec (1.37.0) + rubocop (>= 0.68.1) ruby-progressbar (1.10.1) safe_yaml (1.0.5) sawyer (0.8.2) addressable (>= 2.3.5) faraday (> 0.8, < 2.0) semantic_puppet (1.0.2) - simplecov (0.16.1) + simplecov (0.17.1) docile (~> 1.1) json (>= 1.8, < 3) simplecov-html (~> 0.10.0) - simplecov-console (0.4.2) + simplecov-console (0.6.0) ansi - hirb simplecov + terminal-table simplecov-html (0.10.2) spdx-licenses (1.2.0) + terminal-table (1.8.0) + unicode-display_width (~> 1.1, >= 1.1.1) text (1.3.1) thread_safe (0.3.6) - timers (4.1.2) - hitimes - tty-color (0.4.3) - tty-cursor (0.5.0) - tty-prompt (0.13.1) - necromancer (~> 0.4.0) + tty-color (0.5.0) + tty-cursor (0.7.0) + tty-prompt (0.20.0) + necromancer (~> 0.5.0) pastel (~> 0.7.0) - timers (~> 4.1.2) - tty-cursor (~> 0.5.0) + tty-reader (~> 0.7.0) + tty-reader (0.7.0) + tty-cursor (~> 0.7) + tty-screen (~> 0.7) wisper (~> 2.0.0) - tty-spinner (0.5.0) - tty-cursor (>= 0.5.0) - tty-which (0.3.0) + tty-screen (0.7.0) + tty-spinner (0.9.2) + tty-cursor (~> 0.7) + tty-which (0.4.1) tzinfo (1.2.5) thread_safe (~> 0.1) unf (0.1.4) unf_ext unf_ext (0.0.7.6) unicode-display_width (1.6.0) - vault (0.12.0) + vault (0.13.0) aws-sigv4 - wisper (2.0.0) + wisper (2.0.1) PLATFORMS ruby @@ -212,19 +219,21 @@ DEPENDENCIES json_pure metadata-json-lint pdk + pry puppet (~> 6) puppet-blacksmith puppet-lint puppet-syntax puppetlabs_spec_helper rake - rspec-puppet! + rb-readline + rspec-puppet rubocop rubocop-rspec safe_yaml simplecov simplecov-console - vault + vault (>= 0.13.0) BUNDLED WITH 2.0.1 diff --git a/lib/puppet/functions/hiera_vault.rb b/lib/puppet/functions/hiera_vault.rb index f101c9b..d419e75 100644 --- a/lib/puppet/functions/hiera_vault.rb +++ b/lib/puppet/functions/hiera_vault.rb @@ -131,9 +131,9 @@ def vault_get(key, options, context) begin - secret = get_kv_v1(secretpath, key) + secret = get_kv_v1(secretpath, key, context) if secret.nil? - secret = get_kv_v2(mount, path, key) + secret = get_kv_v2(mount, path, key, context) end rescue Vault::HTTPConnectionError @@ -180,7 +180,8 @@ def vault_get(key, options, context) return answer end - def get_kv_v1(secretpath, key) + def get_kv_v1(secretpath, key, context) + context.explain { "[hiera-vault] Checking path: #{secretpath}" } res = $vault.logical.read(File.join(secretpath, key)) if ! res.nil? res=res.data @@ -188,21 +189,23 @@ def get_kv_v1(secretpath, key) return res end - def get_kv_v2(mount, path, key) - begin - # secretengine -> mount+path / secret -> key / key -> default_field - secretpath = File.join(mount, path, 'data', key).chomp('/') - res = $vault.logical.read(secretpath).data[:data][:value] - rescue - begin - # secretengine -> mount / secret -> path / key -> key - secretpath = File.join(mount, 'data', path, key).chomp('/') - res = $vault.logical.read(secretpath).data[:data][:value] - rescue + def get_kv_v2(mount, path, key, context) + # secretengine -> mount+path / secret -> key / key -> default_field + secretpath_mount_path_data_key = File.join(mount, path, 'data', key).chomp('/') + context.explain { "[hiera-vault] Checking path: #{secretpath_mount_path_data_key}" } + result = $vault.logical.read(secretpath_mount_path_data_key) + if result.respond_to?('data') + return result.data[:data] + else + secretpath_mount_data_path_key = File.join(mount, 'data', path, key).chomp('/') + context.explain { "[hiera-vault] Checking path: #{secretpath_mount_data_path_key}" } + result = $vault.logical.read(secretpath_mount_data_path_key) + if result.respond_to?('data') + return result.data[:data] + else return nil end end - return { :value => res } end # Stringify key:values so user sees expected results and nested objects diff --git a/spec/functions/hiera_vault_happy_path_spec.rb b/spec/functions/hiera_vault_happy_path_spec.rb index f02dd1b..982e23d 100644 --- a/spec/functions/hiera_vault_happy_path_spec.rb +++ b/spec/functions/hiera_vault_happy_path_spec.rb @@ -96,13 +96,6 @@ def vault_test_client to output(/Read secret: test_key/).to_stdout end - it 'should show error when file token is not valid' do - vault_token_tmpfile = Tempfile.open('w') - vault_token_tmpfile.puts('not-valid-token') - vault_token_tmpfile.close - expect { function.lookup_key('test_key', vault_options.merge({'token' => vault_token_tmpfile.path}), context) }. - to output(/Could not read secret puppet\/common\/test_key: permission denied/).to_stdout - end end context 'reading secrets' do diff --git a/spec/functions/hiera_vault_happy_path_v2_spec.rb b/spec/functions/hiera_vault_happy_path_v2_spec.rb new file mode 100644 index 0000000..3736a65 --- /dev/null +++ b/spec/functions/hiera_vault_happy_path_v2_spec.rb @@ -0,0 +1,198 @@ +require 'spec_helper' +require 'support/vault_server' +require 'puppet/functions/hiera_vault' + +describe FakeFunction do + let :function do + described_class.new + end + + let :context do + ctx = instance_double('Puppet::LookupContext') + allow(ctx).to receive(:cache_has_key).and_return(false) + if ENV['DEBUG'] + allow(ctx).to receive(:explain) { |&block| puts(block.call) } + else + allow(ctx).to receive(:explain).and_return(:nil) + end + allow(ctx).to receive(:not_found) + allow(ctx).to receive(:cache).with(String, anything) do |_, val| + val + end + allow(ctx).to receive(:interpolate).with(anything) do |val| + val + end + ctx + end + + let :vault_options do + { + 'address' => RSpec::VaultServer.address, + 'token' => RSpec::VaultServer.token, + 'mounts' => { + 'puppetv2' => [ + 'common' + ] + } + } + end + + def vault_test_client + Vault::Client.new( + address: RSpec::VaultServer.address, + token: RSpec::VaultServer.token + ) + end + + describe '#lookup_key' do + context 'accessing vault' do + + context 'when vault is unsealed' do + before(:context) do + vault_test_client.sys.mount('puppetv2', 'kv', 'puppet secrets v2', { "options" => {"version": "2" }}) + vault_test_client.logical.write('puppetv2/data/common/test_key', { "data" => { value: 'default'} } ) + vault_test_client.logical.write('puppetv2/data/common/array_key', { "data" => { value: '["a", "b", "c"]'} } ) + vault_test_client.logical.write('puppetv2/data/common/hash_key', { "data" => { value: '{"a": 1, "b": 2, "c": 3}'} } ) + vault_test_client.logical.write('puppetv2/data/common/multiple_values_key', { "data" => { a: 1, b: 2, c: 3} } ) + vault_test_client.logical.write('puppetv2/data/common/values_key', { "data" => { value: 123, a: 1, b: 2, c: 3} } ) + vault_test_client.logical.write('puppetv2/data/common/broken_json_key', { "data" => { value: '[,'} } ) + vault_test_client.logical.write('puppetv2/data/common/confined_vault_key', { "data" => { value: 'find_me'} } ) + vault_test_client.logical.write('puppetv2/data/common/stripped_key', { "data" => { value: 'regexed_key'} } ) + end + + context 'configuring vault' do + let :context do + ctx = instance_double('Puppet::LookupContext') + allow(ctx).to receive(:cache_has_key).and_return(false) + allow(ctx).to receive(:explain) { |&block| puts(block.call) } + allow(ctx).to receive(:not_found) + allow(ctx).to receive(:cache).with(String, anything) do |_, val| + val + end + allow(ctx).to receive(:interpolate).with(anything) do |val| + val + end + ctx + end + + it 'should exit early if ENV VAULT_TOKEN is set to IGNORE-VAULT' do + ENV['VAULT_TOKEN'] = 'IGNORE-VAULT' + expect(context).to receive(:not_found) + expect { function.lookup_key('test_key', vault_options.merge({'token' => nil}), context) }. + to output(/token set to IGNORE-VAULT - Quitting early/).to_stdout + end + + it 'should exit early if token is set to IGNORE-VAULT' do + expect(context).to receive(:not_found) + expect { function.lookup_key('test_key', vault_options.merge({'token' => 'IGNORE-VAULT'}), context) }. + to output(/token set to IGNORE-VAULT - Quitting early/).to_stdout + end + + it 'should allow the configuring of a vault token from a file' do + vault_token_tmpfile = Tempfile.open('w') + vault_token_tmpfile.puts(RSpec::VaultServer.token) + vault_token_tmpfile.close + expect { function.lookup_key('test_key', vault_options.merge({'token' => vault_token_tmpfile.path}), context) }. + to output(/Read secret: test_key/).to_stdout + end + + end + + context 'reading secrets' do + it 'should return the full key if no default_field specified' do + expect(function.lookup_key('test_key', vault_options, context)) + .to include('value' => 'default') + end + + it 'should return the key if regex matches confine_to_keys' do + expect(function.lookup_key('confined_vault_key', vault_options.merge('confine_to_keys' => ['^vault.*$']), context)) + .to include('value' => 'find_me') + end + + it 'should not return the key if regex does not match confine_to_keys' do + expect(context).to receive(:not_found) + expect(function.lookup_key('puppet/data/test_key', vault_options.merge('confine_to_keys' => ['^vault.*$']), context)) + .to be_nil + end + + it 'should return nil on non-existant key' do + expect(context).to receive(:not_found) + expect(function.lookup_key('doesnt_exist', vault_options, context)).to be_nil + end + + it 'should return the default_field value if present' do + expect(function.lookup_key('test_key', { 'default_field' => 'value' }.merge(vault_options), context)) + .to eq('default') + end + + it 'should return a hash lacking a default field' do + expect(function.lookup_key('multiple_values_key', vault_options, context)) + .to include('a' => 1, 'b' => 2, 'c' => 3) + end + + it 'should return an array parsed from json' do + expect(function.lookup_key('array_key', { + 'default_field' => 'value', + 'default_field_parse' => 'json' + }.merge(vault_options), context)) + .to contain_exactly('a', 'b', 'c') + end + + it 'should return a hash parsed from json' do + expect(function.lookup_key('hash_key', { + 'default_field' => 'value', + 'default_field_parse' => 'json' + }.merge(vault_options), context)) + .to include('a' => 1, 'b' => 2, 'c' => 3) + end + + it 'should return the string value on broken json content' do + expect(function.lookup_key('broken_json_key', { + 'default_field' => 'value', + 'default_field_parse' => 'json' + }.merge(vault_options), context)) + .to eq('[,') + end + + it 'should return *only* the default_field value if present' do + expect(function.lookup_key('test_key', { + 'default_field' => 'value', + 'default_field_behavior' => 'only' + }.merge(vault_options), context)) + .to eq('default') + + expect(function.lookup_key('values_key', { + 'default_field' => 'value', + 'default_field_behavior' => 'only' + }.merge(vault_options), context)) + .to include('a' => 1, 'b' => 2, 'c' => 3, 'value' => 123) + expect(function.lookup_key('values_key', { + 'default_field' => 'value' + }.merge(vault_options), context)) + .to eq(123) + end + + it 'should return nil with value field not existing and default_field_behavior set to ignore' do + expect(function.lookup_key('multiple_values_key', { + 'default_field' => 'value', + 'default_field_behavior' => 'ignore' + }.merge(vault_options), context)) + .to be_nil + end + + it 'should return nil but continue to look if continue_if_not_found is present' do + expect(context).to receive(:not_found) + expect(function.lookup_key('doesnt_exist', vault_options.merge('continue_if_not_found' => true), context)) + .to be_nil + end + + it 'should gsub the path string if strip_from_keys is present' do + expect(function.lookup_key('stripped_key12345', vault_options.merge('strip_from_keys' => [/[0-9]*/], 'default_field' => 'value'), context)) + .to eql('regexed_key') + end + + end + end + end + end +end diff --git a/spec/functions/hiera_vault_sad_path_spec.rb b/spec/functions/hiera_vault_sad_path_spec.rb index 4f99b9d..39a9c22 100644 --- a/spec/functions/hiera_vault_sad_path_spec.rb +++ b/spec/functions/hiera_vault_sad_path_spec.rb @@ -105,7 +105,7 @@ def vault_test_client vault_token_tmpfile.puts('not-valid-token') vault_token_tmpfile.close expect { function.lookup_key('test_key', vault_options.merge({'token' => vault_token_tmpfile.path}), context) }. - to output(/Could not read secret puppet\/common\/test_key: permission denied/).to_stdout + to output(/Could not read secret .+ permission denied/).to_stdout end end