feat[redis cache plugin]: Add Redis connection parameters full contro…

…l support

plugins/cache/redis.py

@@ -15,30 +15,34 @@
     requirements:
       - redis>=2.4.5 (python lib)
     options:
-      _uri:
-        description:
-          - Redis connection string, 3 possible formats.
-          - The simple format consist in a colon separated string of connection information for Redis.
-          - This first format is V(host:port:db:password), for example V(localhost:6379:0:changeme).
-          - To use encryption in transit, prefix the connection with V(tls://), as in V(tls://localhost:6379:0:changeme).
-          - To use redis sentinel, use second format with separator V(;), for example V(localhost:26379;localhost:26379;0:changeme). Requires redis>=2.9.0.
-          - To specify advanced connection parameters (keep alive, etc ...), use the 3rd format.
-          - This last format consists in a '&' separated string of redis connection parameters as <name>=<value> pairs.
-          - For example V(host=localhost&port=6379&db=0&password=changeme&socket_keepalive=true).
-          - See https://redis-py2.readthedocs.io/en/latest/ for Redis connection parameters name and accepted values.
-        required: true
+      _decode_responses:
+        description: If set to `true`, returned values from Redis commands get decoded automatically using the client’s charset value.
+        default: false
         env:
-          - name: ANSIBLE_CACHE_PLUGIN_CONNECTION
+          - name: ANSIBLE_CACHE_REDIS_DECODE_RESPONSES
         ini:
-          - key: fact_caching_connection
+          - key: fact_caching_redis_decode_responses
             section: defaults
-      _prefix:
-        description: User defined prefix to use when creating the DB entries
-        default: ansible_facts
+        type: bool
+      _encoding:
+        description: Set the charset to use for facts encoding.
+        default: utf-8
         env:
-          - name: ANSIBLE_CACHE_PLUGIN_PREFIX
+          - name: ANSIBLE_CACHE_REDIS_ENCODING
         ini:
-          - key: fact_caching_prefix
+          - key: fact_caching_redis_encoding
+            section: defaults
+      _encoding_errors:
+        description:
+          - The error handling scheme to use for encoding errors.
+          - The default is `strict` meaning that encoding errors raise a `UnicodeEncodeError`.
+          - See https://docs.python.org/fr/3/library/stdtypes.html#str.encode for more details.
+        default: strict
+        choices: [ backslashreplace, ignore, replace, strict, xmlcharrefreplace ]
+        env:
+          - name: ANSIBLE_CACHE_REDIS_ENCODING_ERRORS
+        ini:
+          - key: fact_caching_redis_encoding_errors
             section: defaults
       _keyset_name:
         description: User defined name for cache keyset name.
@@ -49,6 +53,26 @@
           - key: fact_caching_redis_keyset_name
             section: defaults
         version_added: 1.3.0
+      _prefix:
+        description: User defined prefix to use when creating the DB entries
+        default: ansible_facts
+        env:
+          - name: ANSIBLE_CACHE_PLUGIN_PREFIX
+        ini:
+          - key: fact_caching_prefix
+            section: defaults
+      _retry_on_timeout:
+        description:
+          - Controls how socket.timeout errors are handled.
+          - When set to `false` a TimeoutError will be raised anytime a socket.timeout is encountered.
+          - When set to `true`, it enable retries like other `socket.error`s.
+        default: false
+        env:
+          - name: ANSIBLE_CACHE_REDIS_RETRY_ON_TIMEOUT
+        ini:
+          - key: fact_caching_redis_retry_on_timeout
+            section: defaults
+        type: bool
       _sentinel_service_name:
         description: The redis sentinel service name (or referenced as cluster name).
         env:
@@ -57,6 +81,79 @@
           - key: fact_caching_redis_sentinel
             section: defaults
         version_added: 1.3.0
+      _socket_connect_timeout:
+        description:
+          - Timeout value, in seconds, for Redis socket connection.
+          - If not set, connection timeout is disabled.
+        env:
+          - name: ANSIBLE_CACHE_REDIS_SOCKET_CONNECT_TIMEOUT
+        ini:
+          - key: fact_caching_redis_socket_connect_timeout
+            section: defaults
+        type: integer
+      _socket_keepalive:
+        description:
+          - Specifies whether to enable keepalive for Redis socket connection.
+        default: false
+        env:
+          - name: ANSIBLE_CACHE_REDIS_SOCKET_KEEPALIVE
+        ini:
+          - key: fact_caching_redis_socket_keepalive
+            section: defaults
+        type: bool
+      _socket_keepalive_options:
+        description:
+          - Finer grain control keepalive options when `_socket_keepalive` is set to `true`.
+          - A comma separated socket options string of <key>:<value> pairs, for example V(TCP_KEEPIDLE:600,TCP_KEEPCNT=10,TCP_KEEPINTVL:300).
+          - Accepted keys are `TCP_KEEPIDLE`, `TCP_KEEPCNT`, and `TCP_KEEPINTVL`.
+          - Integers are expected for values.
+        env:
+          - name: ANSIBLE_CACHE_REDIS_SOCKET_KEEPALIVE_OPTIONS
+        ini:
+          - key: fact_caching_redis_socket_keepalive_options
+            section: defaults
+      _socket_timeout:
+        description:
+          - Timeout value, in seconds, for Redis socket connection.
+          - If not set, timeout is disabled.
+        env:
+          - name: ANSIBLE_CACHE_REDIS_SOCKET_TIMEOUT
+        ini:
+          - key: fact_caching_redis_socket_timeout
+            section: defaults
+        type: integer
+      _ssl_ca_certs_file:
+        description: When using SSL on Redis connection, specifies the SSL CA file path.
+        env:
+          - name: ANSIBLE_CACHE_REDIS_SSL_CA_CERTS_FILE
+        ini:
+          - key: fact_caching_redis_ssl_ca_certs_file
+            section: defaults
+      _ssl_cert_file:
+        description: When using SSL on Redis connection, specifies the SSL certificate file path.
+        env:
+          - name: ANSIBLE_CACHE_REDIS_SSL_CERT_FILE
+        ini:
+          - key: fact_caching_redis_ssl_cert_file
+            section: defaults
+      _ssl_cert_reqs:
+        default: none
+        description:
+          - When using SSL on Redis connection, specifies the security mode to use.
+          - See https://docs.python.org/3/library/ssl.html#ssl.SSLContext.verify_mode for more details.
+        env:
+          - name: ANSIBLE_CACHE_REDIS_SSL_CERT_REQ
+        ini:
+          - key: fact_caching_redis_ssl_cert_req
+            section: defaults
+        choices: [ none, optionnal, required ]
+      _ssl_key_file:
+        description: When using SSL on Redis connection, specifies the SSL key file path.
+        env:
+          - name: ANSIBLE_CACHE_REDIS_SSL_KEY_FILE
+        ini:
+          - key: fact_caching_redis_ssl_key_file
+            section: defaults
       _timeout:
         default: 86400
         description: Expiration timeout in seconds for the cache plugin data. Set to 0 to never expire
@@ -66,6 +163,18 @@
           - key: fact_caching_timeout
             section: defaults
         type: integer
+      _uri:
+        description:
+          - A colon separated string of connection information for Redis.
+          - The format is V(host:port:db:password), for example V(localhost:6379:0:changeme).
+          - To use encryption in transit, prefix the connection with V(tls://), as in V(tls://localhost:6379:0:changeme).
+          - To use redis sentinel, use separator V(;), for example V(localhost:26379;localhost:26379;0:changeme). Requires redis>=2.9.0.
+        required: true
+        env:
+          - name: ANSIBLE_CACHE_PLUGIN_CONNECTION
+        ini:
+          - key: fact_caching_connection
+            section: defaults
 '''
 
 import re
@@ -97,9 +206,11 @@ class CacheModule(BaseCacheModule):
     performance.
     """
     _sentinel_service_name = None
+    _encoding_errors_choices = ['backslashreplace', 'ignore', 'replace', 'strict', 'xmlcharrefreplace']
+    _socket_keepalive_available_opts = ['TCP_KEEPIDLE', 'TCP_KEEPCNT', 'TCP_KEEPINTVL']
     re_url_conn = re.compile(r'^([^:]+|\[[^]]+\]):(\d+):(\d+)(?::(.*))?$')
     re_sent_conn = re.compile(r'^(.*):(\d+)$')
-    re_full_desc_url_conn = re.compile(r'^(?:([a-z_]+=[^&]+)&)(?:([a-z_]+=[^&]+)&)*([a-z_]+=[^&]+)$')
+    re_socket_keepalive_opts = re.compile(r'^(\w+:\d+)(?:,(\w+:\d+))+$')
 
     def __init__(self, *args, **kwargs):
         uri = ''
@@ -111,39 +222,82 @@ def __init__(self, *args, **kwargs):
         self._prefix = self.get_option('_prefix')
         self._keys_set = self.get_option('_keyset_name')
         self._sentinel_service_name = self.get_option('_sentinel_service_name')
+        self._decode_responses = bool(self.get_option('_decode_responses'))
+        self._encoding = self.get_option('_encoding')
+        self._encoding_errors = self.get_option('_encoding_errors')
+        self._retry_on_timeout = bool(self.get_option('_retry_on_timeout'))
+        self._socket_keepalive = self.get_option('_socket_keepalive')
+        self._socket_connect_timeout = int(self.get_option('_socket_connect_timeout')) \
+            if self._socket_keepalive and self.get_option('_socket_connect_timeout') else None
+        self._socket_keepalive_options = self._parse_socket_options(self.get_option('_socket_keepalive_options')) \
+            if self._socket_keepalive and self.get_option('_socket_keepalive_options') else None
+        self._socket_timeout = int(self.get_option('_socket_timeout')) \
+            if self._socket_keepalive and self.get_option('_socket_timeout') else None
+        self._ssl_ca_certs_file = self.get_option('_ssl_ca_certs_file')
+        self._ssl_cert_file = self.get_option('_ssl_cert_file')
+        self._ssl_cert_reqs = self.get_option('_ssl_cert_reqs')
+        self._ssl_key_file = self.get_option('_ssl_key_file')
 
         if not HAS_REDIS:
             raise AnsibleError("The 'redis' python module (version 2.4.5 or newer) is required for the redis fact cache, 'pip install redis'")
 
-        self._cache = {}
-        kw = {}
-
-        if self.re_full_desc_url_conn.match(uri):
-            conn_params = uri.split('&')
+        if not self._encoding_errors in self._encoding_errors_choices:
+            raise AnsibleError("Unsupported value '%s' for Redis cache plugin parameter '_encoding_errors'" % (self._encoding_errors))
 
-            connection = {}
-
-            for param in conn_params:
-                p_key, p_value = param.split('=')
-                connection[p_key] = p_value
-
-            self._db = StrictRedis(**connection)
+        self._cache = {}
+        kw = {
+            'decode_responses': self._decode_responses,
+            'encoding_errors': self._encoding_errors,
+            'encoding': self._encoding,
+            'retry_on_timeout': self._retry_on_timeout,
+            'socket_connect_timeout': self._socket_connect_timeout,
+            'socket_keepalive_options': self._socket_keepalive_options,
+            'socket_keepalive': self._socket_keepalive,
+            'socket_timeout': self._socket_timeout,
+        }
+
+        # tls connection
+        tlsprefix = 'tls://'
+        if uri.startswith(tlsprefix):
+            from os import access, R_OK
+            from os.path import isfile
+            import ssl
+
+            if not self._ssl_cert_reqs.upper() in list(map(lambda x: x.name.split('_')[1], ssl.VerifyMode)):
+                raise AnsibleError("Unsupported value '%s' for Redis cache plugin parameter '_ssl_cert_reqs'" % (self._ssl_cert_reqs))
+
+            if self._ssl_ca_certs_file:
+                if not isfile(self._ssl_ca_certs_file) and not access(self._ssl_ca_certs_file, R_OK):
+                    raise AnsibleError("File %s doesn't exist or isn't readable for Redis cache plugin parameter '_ssl_ca_certs_file'" % self._ssl_ca_certs_file)
+
+            if self._ssl_cert_file:
+                if not isfile(self._ssl_cert_file) and not access(self._ssl_cert_file, R_OK):
+                    raise AnsibleError("File %s doesn't exist or isn't readable for Redis cache plugin parameter '_ssl_cert_file'" % self._ssl_cert_file)
+
+            if self._ssl_key_file:
+                if not isfile(self._ssl_key_file) and not access(self._ssl_key_file, R_OK):
+                    raise AnsibleError("File %s doesn't exist or isn't readable for Redis cache plugin parameter '_ssl_key_file'" % self._ssl_key_file)
+
+            kw.update({
+                'ssl': True,
+                'ssl_keyfile': self._ssl_key_file,
+                'ssl_certfile': self._ssl_cert_file,
+                'ssl_cert_reqs': self._ssl_cert_reqs,
+                'ssl_ca_certs': self._ssl_ca_certs_file
+            })
+
+            uri = uri[len(tlsprefix):]
+
+        # redis sentinel connection
+        if self._sentinel_service_name:
+            self._db = self._get_sentinel_connection(uri, kw)
+        # normal connection
         else:
-            # tls connection
-            tlsprefix = 'tls://'
-            if uri.startswith(tlsprefix):
-                kw['ssl'] = True
-                uri = uri[len(tlsprefix):]
-
-            # redis sentinel connection
-            if self._sentinel_service_name:
-                self._db = self._get_sentinel_connection(uri, kw)
-            # normal connection
-            else:
-                connection = self._parse_connection(self.re_url_conn, uri)
-                self._db = StrictRedis(*connection, **kw)
+            connection = self._parse_connection(self.re_url_conn, uri)
+            self._db = StrictRedis(*connection, **kw)
 
         display.vv('Redis connection: %s' % self._db)
+        display.vvv("Redis connection kwargs: %s" % (self._db.get_connection_kwargs()))
 
     @staticmethod
     def _parse_connection(re_patt, uri):
@@ -152,6 +306,18 @@ def _parse_connection(re_patt, uri):
             raise AnsibleError("Unable to parse connection string")
         return match.groups()
 
+    def _parse_socket_options(self, options):
+        if not self.re_socket_keepalive_opts.match(options):
+            raise AnsibleError("Unable to parse Redis cache socket keepalive options string")
+        import socket;
+        opts = {}
+        for opt in options.split(','):
+            key, value = opt.split(':')
+            if not key in self._socket_keepalive_available_opts:
+                raise AnsibleError("Option '%s' is not available for parameter '_socket_keepalive_options' for Redis cache plugin" % (key))
+            opts[ getattr(socket, key) ] = int(value)
+        return opts
+
     def _get_sentinel_connection(self, uri, kw):
         """
         get sentinel connection details from _uri