Merge pull request #284 from myii/feat/use-top-level_values_for-map.j…

…inja-dump-files

feat(_mapdata): use top-level `values` for `map.jinja` dump files

ssf/defaults.yaml

@@ -62,8 +62,8 @@ ssf_node_anchors:
             #       An alternative method could be to use:
             #       `git describe --abbrev=0 --tags`
             # yamllint disable rule:line-length rule:quoted-strings
-            title: "test(map): verify '`'map.jinja'`' dump using '`'_mapdata'`' state"
-            body: '* Automated using https://github.com/myii/ssf-formula/pull/283'
+            title: "refactor(map): use top-level '`'values:'`' key in '`'map.jinja'`' dumps"
+            body: '* Semi-automated using https://github.com/myii/ssf-formula/pull/284'
             # yamllint enable rule:line-length rule:quoted-strings
           github:
             owner: 'saltstack-formulas'
@@ -134,6 +134,7 @@ ssf_node_anchors:
         map_jinja:
           filename: 'map.jinja'
           verification: {}
+          version: 3
         platforms:
           # Could use `opensuse-leap` throughout since `/` never used at this end
           # Would have to modify the `if` in the `kitchen.yml` template(s), though

ssf/files/default/formula/_mapdata/_mapdata.jinja

@@ -1,9 +1,9 @@
 # yamllint disable rule:indentation rule:line-length
-# {{ grains.get('osfinger', grains.os) }}
+# {{ grains.get("osfinger", grains.os) }}
 ---
 {#- use salt.slsutil.serialize to avoid encoding errors on some platforms #}
-{{ salt['slsutil.serialize'](
-     'yaml',
+{{ salt["slsutil.serialize"](
+     "yaml",
      map,
      default_flow_style=False,
      allow_unicode=True,

ssf/files/default/formula/_mapdata/init.sls

@@ -1,22 +1,34 @@
-${ '# -*- coding: utf-8 -*-' }
-${ '# vim: ft=sls' }
+${ "# -*- coding: utf-8 -*-" }
+${ "# vim: ft=sls" }
 ---
 {#- Get the `tplroot` from `tpldir` #}
-{%- set tplroot = tpldir.split('/')[0] %}
-<% filename = map_jinja['filename'] %>\
-<% import_var = map_jinja['verification']['import'] %>\
-<% import_var_as = '' if import_var == 'mapdata' else ' as mapdata' %>\
-{%- from tplroot ~ "/${ filename }" import ${ import_var }${ import_var_as } with context %}
+{%- set tplroot = tpldir.split("/")[0] %}
+<% filename = map_jinja["filename"] %>\
+<% import_vars = map_jinja["verification"]["import"] %>\
+% for import_var in import_vars:
+{%- from tplroot ~ "/${ filename }" import ${ import_var } with context %}
+% endfor
 
-{%- do salt['log.debug']('### MAP.JINJA DUMP ###\n' ~ mapdata | yaml(False)) %}
+{%- set _mapdata = {
+% if map_jinja["version"] > 3:
+      "values": mapdata,
+% else:
+      "values": {
+        % for import_var in import_vars:
+        "${ import_var }": ${ import_var },
+        % endfor
+      }
+% endif
+    } %}
+{%- do salt["log.debug"]("### MAP.JINJA DUMP ###\n" ~ _mapdata | yaml(False)) %}
 
-{%- set output_dir = '/temp' if grains.os_family == 'Windows' else '/tmp' %}
-{%- set output_file = output_dir ~ '/salt_mapdata_dump.yaml' %}
+{%- set output_dir = "/temp" if grains.os_family == "Windows" else "/tmp" %}
+{%- set output_file = output_dir ~ "/salt_mapdata_dump.yaml" %}
 
 {{ tplroot }}-mapdata-dump:
   file.managed:
     - name: {{ output_file }}
     - source: salt://{{ tplroot }}/_mapdata/_mapdata.jinja
     - template: jinja
     - context:
-        map: {{ mapdata | yaml }}
+        map: {{ _mapdata | yaml }}

ssf/files/default/inspec/controls/_mapdata_spec.rb

@@ -5,19 +5,43 @@
 control '`map.jinja` YAML dump' do
   title 'should match the comparison file'
 
-  # Strip the `platform[:finger]` version number down to the "OS major release"
-  mapdata_file = "_mapdata/#{system.platform[:finger].split('.').first}.yaml"
+  ### Method
+  # The steps below for each file appear convoluted but they are both required
+  # and similar in nature:
+  # 1. The earliest method was to simply compare the files textually but this often
+  #    led to false positives due to inconsistencies (e.g. spacing, ordering)
+  # 2. The next method was to load the files back into YAML structures and then
+  #    compare but InSpec provided block diffs this way, unusable by end users
+  # 3. The final step was to dump the YAML structures back into a string to use
+  #    for the comparison; this both worked and provided human-friendly diffs
 
-  # Load the mapdata from profile https://docs.chef.io/inspec/profiles/#profile-files
-  mapdata_dump = YAML.safe_load(inspec.profile.file(mapdata_file))
+  ### Comparison file for the specific platform
+  ### Static, adjusted as part of code contributions, as map data is changed
+  # Strip the `platform[:finger]` version number down to the "OS major release"
+  platform_finger = system.platform[:finger].split('.').first.to_s
+  # Use that to set the path to the file (relative to the InSpec suite directory)
+  mapdata_file_path = "_mapdata/#{platform_finger}.yaml"
+  # Load the mapdata from profile, into a YAML structure
+  # https://docs.chef.io/inspec/profiles/#profile-files
+  mapdata_file_yaml = YAML.safe_load(inspec.profile.file(mapdata_file_path))
+  # Dump the YAML back into a string for comparison
+  mapdata_file_dump = YAML.dump(mapdata_file_yaml)
 
-  # Derive the location of the dumped mapdata
+  ### Output file produced by running the `_mapdata` state
+  ### Dynamic, generated during Kitchen's `converge` phase
+  # Derive the location of the dumped mapdata (differs for Windows)
   output_dir = platform[:family] == 'windows' ? '/temp' : '/tmp'
-  output_file = "#{output_dir}/salt_mapdata_dump.yaml"
+  # Use that to set the path to the file (absolute path, i.e. within the container)
+  output_file_path = "#{output_dir}/salt_mapdata_dump.yaml"
+  # Load the output into a YAML structure using InSpec's `yaml` resource
+  # https://github.com/inspec/inspec/blob/49b7d10/lib/inspec/resources/yaml.rb#L29
+  output_file_yaml = yaml(output_file_path).params
+  # Dump the YAML back into a string for comparison
+  output_file_dump = YAML.dump(output_file_yaml)
 
   describe 'File content' do
     it 'should match profile map data exactly' do
-      expect(yaml(output_file).params).to eq(mapdata_dump)
+      expect(output_file_dump).to eq(mapdata_file_dump)
     end
   end
 end

ssf/formulas.yaml

@@ -3345,7 +3345,7 @@ ssf:
           1: *inspec_suites_kitchen__share_suite
         map_jinja:
           verification:
-            import: 'openntpd'
+            import: ['openntpd']
         platforms: *platforms_new
         platforms_matrix:
           # [os           , os_ver, salt_ver, py_ver,    inspec_suite]
@@ -3385,7 +3385,8 @@ ssf:
           1: *inspec_suites_kitchen__share_suite
         map_jinja:
           verification:
-            import: 'mapdata'
+            import: ['mapdata']
+          version: 5
         platforms: *platforms_new
         platforms_matrix: *platforms_matrix_new_mainly_master_images
         use_libsaltcli: true
@@ -3417,7 +3418,7 @@ ssf:
           1: *inspec_suites_kitchen__share_suite
         map_jinja:
           verification:
-            import: 'map'
+            import: ['map']
         platforms: *platforms_new_saltimages
         platforms_matrix:
           # [os           , os_ver, salt_ver, py_ver,    inspec_suite]
@@ -3635,15 +3636,14 @@ ssf:
           - suse
         map_jinja:
           verification:
-            import: 'php'
+            import: ['php']
         platforms_matrix:
           - [debian       ,   10   ,   master,      3,       debian]
           - [ubuntu       ,   18.04,   2019.2,      3,       ubuntu]
           - [amazonlinux  ,    2   ,   2019.2,      3,       redhat]
           - [fedora       ,   30   ,   2018.3,      3,       redhat]
           - [opensuse/leap,   15.1 ,   2018.3,      2,         suse]
           # # - [centos       ,    6   ,   2017.7,      2,       redhat]
-        travis: *travis_do_not_use_single_job_for_linters
         use_tofs: true
       semrel_files: *semrel_files_inc_map_jinja_verifier
     postfix:
@@ -3966,7 +3966,7 @@ ssf:
         map_jinja:
           filename: 'package-map.jinja'
           verification:
-            import: 'pkgs'
+            import: ['pkgs']
         platforms_matrix:
           # [os           , os_ver, salt_ver, py_ver,    inspec_suite]
           - [debian       ,   10   ,   master,      3,          latest]
@@ -4569,7 +4569,7 @@ ssf:
           1: *inspec_suites_kitchen__share_suite
         map_jinja:
           verification:
-            import: 'sudoers'
+            import: ['sudoers']
         platforms: *platforms_new
         platforms_matrix: *platforms_matrix_new_mainly_master_images
       semrel_files: *semrel_files_inc_map_jinja_verifier
@@ -4825,7 +4825,7 @@ ssf:
           - gentoo
         map_jinja:
           verification:
-            import: 'TEMPLATE'
+            import: ['TEMPLATE']
         platforms:
           ### `tiamat-py3`
           - [debian       ,   10   ,   tiamat,      3]