Add lenient yamllint, fix example docs

Configured to only care about things like duplicate keys,
not whitespace issues.

Author: jeremyh

.travis/environment.yaml

@@ -45,3 +45,4 @@ dependencies:
   - pytest-logging
   - pytest-faulthandler
   - SharedArray
+  - yamllint # testing

.yamllint

@@ -0,0 +1,34 @@
+extends: default
+
+rules:
+    # We could enable these if we find a decent yaml autoformatter.
+    # (yamlfmt in pip is too buggy at the moment for our files)
+    #
+    # The effort of fixing them by hand, especially auto-generated yamls, is not
+    # currently worth it.
+
+    # Many tools (eg. generated secure keys) don't output wrapped lines.
+    line-length: disable
+
+    # Pedantry & existing docs don't have it.
+    document-start: disable
+
+    # Warning that truthy values are not quoted.
+    # Many documents currently use "True" without quotes, so this would be a
+    # larger change across almost every doc.
+    truthy: disable
+
+    # Whitespace issues: hundreds in existing docs.
+    indentation: disable
+    trailing-spaces: disable
+    commas: disable
+    new-line-at-end-of-file: disable
+    brackets: disable
+    hyphens: disable
+    colons: disable
+    comments: disable
+    comments-indentation: disable
+
+
+
+

check-code.sh

@@ -12,3 +12,12 @@ pylint -j 2 --reports no datacube datacube_apps
 # Users can specify extra folders as arguments.
 pytest -r sx --cov datacube --doctest-ignore-import-errors --durations=5 datacube tests datacube_apps $@
 
+set +x
+
+# Optinally validate example yaml docs.
+if which yamllint;
+then
+    set -x
+    yamllint $(find . \( -iname '*.yaml' -o -iname '*.yml' \) )
+    set +x
+fi

docs/config_samples/dataset_types/dem1sv1_0.yaml

@@ -30,4 +30,3 @@ measurements:
         dtype: float32
         nodata: -3.4028235e+38
         units: 'metre'
-

docs/config_samples/dataset_types/dsm1sv10.yaml

@@ -22,4 +22,3 @@ measurements:
         dtype: float32
         nodata: .nan
         units: 'metre'
-

docs/config_samples/dataset_types/h8_granules.yaml

@@ -242,8 +242,6 @@ measurements:
 
 ---
 
-
-
 name: ls7_ledaps_scene
 description: Landsat 7 LEDAPS 25 metre
 metadata_type: eo

docs/config_samples/dataset_types/ls7_scenes.yaml

@@ -335,7 +335,6 @@ measurements:
   units: '1'
   dtype: uint16
   nodata: 0
-  units: '1'
   flags_definition: 
     sca:
         bits: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
@@ -351,4 +350,4 @@ measurements:
             7: cloud high probability
             8: thin cirrus
             9: snow   
-            10: defective or saturated
+            10: defective or saturated

docs/config_samples/dataset_types/s2_granules.yaml

@@ -1,4 +1,3 @@
-
 name: ls5_usgs_ard
 description: Landsat 5 USGS ARD 30 metre tile
 metadata_type: eo
@@ -176,4 +175,3 @@ measurements:
       spectral_definition:
           wavelength: [2000, 2001, 2003, 2005, 2007, 2009, 2011, 2013, 2015, 2017, 2018, 2020, 2022, 2024, 2026, 2028, 2030, 2032, 2034, 2035, 2037, 2039, 2041, 2043, 2045, 2047, 2049, 2051, 2052, 2054, 2056, 2058, 2060, 2062, 2064, 2066, 2067, 2069, 2071, 2073, 2075, 2077, 2079, 2081, 2083, 2085, 2086, 2088, 2090, 2092, 2094, 2096, 2099, 2100, 2102, 2104, 2106, 2108, 2110, 2112, 2114, 2116, 2117, 2119, 2121, 2123, 2125, 2127, 2129, 2131, 2133, 2135, 2136, 2138, 2140, 2142, 2144, 2146, 2148, 2150, 2151, 2153, 2155, 2157, 2159, 2161, 2163, 2165, 2166, 2168, 2170, 2172, 2174, 2176, 2178, 2180, 2182, 2183, 2185, 2187, 2189, 2191, 2193, 2195, 2197, 2199, 2201, 2203, 2205, 2207, 2209, 2210, 2212, 2214, 2216, 2218, 2220, 2222, 2223, 2226, 2227, 2229, 2231, 2233, 2235, 2237, 2239, 2241, 2242, 2244, 2246, 2248, 2250, 2252, 2254, 2256, 2258, 2259, 2261, 2263, 2265, 2267, 2269, 2271, 2273, 2274, 2276, 2278, 2280, 2282, 2284, 2286, 2288, 2290, 2292, 2293, 2295, 2297, 2299, 2301, 2303, 2305, 2307, 2309, 2310, 2312, 2314, 2316, 2318, 2320, 2322, 2323, 2325, 2327, 2329, 2331, 2333, 2335, 2337, 2339, 2340, 2342, 2344, 2346, 2348, 2350, 2352, 2354, 2355, 2357, 2359, 2361, 2363, 2365, 2367, 2369, 2371, 2373, 2374, 2376, 2378, 2380, 2382, 2384, 2386, 2390, 2395, 2400]
           response: [0.0, 0.003, -0.002, 0.003, -0.001, 0.0, 0.004, -0.004, 0.002, 0.002, 0.002, 0.012, 0.009, 0.007, 0.011, 0.02, 0.017, 0.03, 0.035, 0.037, 0.044, 0.051, 0.065, 0.08, 0.088, 0.102, 0.133, 0.165, 0.188, 0.22, 0.264, 0.316, 0.367, 0.421, 0.484, 0.554, 0.59, 0.67, 0.683, 0.73, 0.756, 0.767, 0.794, 0.774, 0.776, 0.789, 0.775, 0.784, 0.778, 0.768, 0.762, 0.761, 0.775, 0.775, 0.764, 0.784, 0.792, 0.814, 0.794, 0.825, 0.817, 0.806, 0.819, 0.821, 0.852, 0.832, 0.836, 0.85, 0.855, 0.862, 0.853, 0.871, 0.848, 0.882, 0.875, 0.86, 0.856, 0.887, 0.85, 0.872, 0.879, 0.857, 0.865, 0.867, 0.871, 0.882, 0.87, 0.869, 0.873, 0.877, 0.868, 0.88, 0.877, 0.87, 0.878, 0.88, 0.868, 0.881, 0.87, 0.856, 0.863, 0.863, 0.857, 0.844, 0.859, 0.857, 0.852, 0.866, 0.868, 0.856, 0.856, 0.847, 0.861, 0.862, 0.84, 0.856, 0.838, 0.856, 0.837, 0.842, 0.826, 0.844, 0.827, 0.842, 0.822, 0.843, 0.823, 0.854, 0.839, 0.853, 0.854, 0.865, 0.873, 0.869, 0.865, 0.893, 0.89, 0.89, 0.906, 0.924, 0.92, 0.922, 0.939, 0.916, 0.94, 0.93, 0.942, 0.957, 0.954, 0.951, 0.954, 0.966, 0.975, 0.985, 0.971, 0.973, 0.97, 0.993, 0.996, 0.983, 0.972, 1.0, 0.998, 0.971, 0.968, 0.967, 0.962, 0.949, 0.923, 0.929, 0.917, 0.934, 0.903, 0.926, 0.916, 0.942, 0.924, 0.92, 0.863, 0.824, 0.775, 0.684, 0.583, 0.48, 0.378, 0.275, 0.233, 0.171, 0.131, 0.111, 0.081, 0.069, 0.046, 0.029, 0.038, -0.002, 0.029, 0.016, 0.009, 0.017, 0.003, 0.015, 0.0, -0.009, 0.007, 0.0, 0.0, 0.0]
-

docs/config_samples/dataset_types/usgs_ard.yaml

@@ -529,38 +529,6 @@ lineage:
                         label: Middle Infrared 2
                         number: '7'
                         cell_size: 25.0
-            lineage:
-                algorithm:
-                    name: LPGS
-                    version: 12.7.0
-                    parameters: {}
-                machine:
-                    hostname: r2200
-                    runtime_id: d052fcb0-1ccb-11e6-b546-a0000100fe80
-                    software_versions:
-                        eodatasets:
-                            repo_url: https://github.com/GeoscienceAustralia/eo-datasets.git
-                            version: '0.4'
-                        galpgs:
-                            repo_url: https://github.com/GeoscienceAustralia/galpgs.git
-                            version: 1.1.1
-                        gqa:
-                            repo_url: https://github.com/GeoscienceAustralia/gqa.git
-                            version: '0.6'
-                        pinkmatter: 4.1.4084
-                    uname: 'Linux r2200 2.6.32-573.22.1.el6.x86_64 #1 SMP Wed Mar
-                        23 03:35:39 UTC 2016 x86_64'
-                ancillary_quality: DEFINITIVE
-                ancillary:
-                    cpf:
-                        name: L5CPF20111001_20111231.08
-                        uri: /g/data/v10/eoancillarydata/sensor-specific/LANDSAT5/CalibrationParameterFile/L5CPF20111001_20111231.08
-                        access_dt: 2016-05-18 18:30:03
-                        modification_dt: 2014-11-26 13:03:33
-                        checksum_sha1: 1743fad95ac392239f92d9cb85fd30f3228f680c
-                        label: Middle Infrared 2
-                        number: '7'
-                        cell_size: 25.0
             lineage:
                 algorithm:
                     name: LPGS