Merge pull request #234 from mirceaulinic/develop

Release 2021.6.0

.github/workflows/docker-publish.yml

@@ -0,0 +1,78 @@
+name: Docker
+
+on:
+  push:
+    # Publish `develop` as Docker `latest` image.
+    branches:
+      - develop
+
+    # Publish `v1.2.3` tags as releases.
+    tags:
+      - '*'
+
+  # Run tests for any PRs.
+  pull_request:
+
+env:
+  IMAGE_NAME: salt-sproxy
+
+jobs:
+  # Run tests.
+  # See also https://docs.docker.com/docker-hub/builds/automated-testing/
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Run tests
+        run: |
+          if [ -f docker-compose.test.yml ]; then
+            docker-compose --file docker-compose.test.yml build
+            docker-compose --file docker-compose.test.yml run sut
+          else
+            docker build . --file Dockerfile
+          fi
+
+  # Push image to GitHub Packages.
+  # See also https://docs.docker.com/docker-hub/builds/
+  push:
+    # Ensure test job passes before pushing image.
+    needs: test
+
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build image
+        run: docker build . --file Dockerfile --tag $IMAGE_NAME
+
+      - name: Log into GitHub Container Registry
+        run: echo "${{ secrets.CR_PAT }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Push image to GitHub Container Registry
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+
+          # Change all uppercase to lowercase
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+          # Strip git ref prefix from version
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+          # Strip "v" prefix from tag name
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+          # Publish develop under the develop tag
+          [ "$VERSION" == "develop" ] && VERSION=develop
+          
+          # Publish master under the latest tag
+          [ "$VERSION" == "master" ] && VERSION=latest
+
+          echo IMAGE_ID=$IMAGE_ID
+          echo VERSION=$VERSION
+
+          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION

docs/events.rst

@@ -192,8 +192,8 @@ function:
   shutdown_interface:
     runner.proxy.execute:
       - tgt: {{ data.host }}
-      - function: net.load_template
       - kwarg:
+          salt_function: net.load_template
           template_name: salt://templates/shut_interface.jinja
           interface_name: {{ data.yang_message.interfaces.interface.keys()[0] }}
 

docs/releases/2019.6.0.rst

@@ -0,0 +1,8 @@
+.. _release-2019.6.0:
+
+================
+Release 2019.6.0
+================
+
+This is the initial release. Please explore the documentation and usage 
+examples to get familiar with *salt-sproxy*.

docs/releases/2020.10.0.rst

@@ -1,8 +1,8 @@
 .. _release-2020.10.0:
 
-=========
-2020.10.0
-=========
+=================
+Release 2020.10.0
+=================
 
 New Features
 ------------

docs/releases/2021.6.0.rst

@@ -0,0 +1,35 @@
+.. _release-2021.6.0:
+
+================
+Release 2021.6.0
+================
+
+While this release doesn't bring any particularly new features, it provides 
+several performance improvements which are easily noticeable, from an user 
+perspective (when transitioning from a previous *salt-sproxy* version to 
+2021.6.0):
+
+- Batch execution is no longer the default behaviour when targeting against 
+  running Proxy Minions (i.e., when using the ``--use-existing-proxy`` CLI 
+  option, or setting the configuration to ``use_existing_proxy: true`` into the 
+  Master config file). That is, when executing *salt-sproxy* in an environment 
+  where you have a Salt Master running and managing running Proxy or regular
+  Minions, and your targeting expression matches some or all of those Minions,
+  the execution is spread out to all the Minions at once. If you want to 
+  execute in batches, you will need to explicitly use the ``-b`` (or 
+  ``--batch-size`` CLI argument). Note however, that for devices that don't 
+  have a (Proxy) Minion running, the batch continues to default to the number 
+  of CPUs available on the local machine. As always, for a quicker execution 
+  (yet more resource intensive) you may want to set a higher batch size value.
+
+- Whenever targeting against running (Proxy) Minions, in case you want to 
+  perform a survey against your Minions and determine which ones are alive, you 
+  can use the ``--test-ping`` CLI option (or set ``test_ping: true`` in the 
+  Master configuration file). Before this release, this option was the default 
+  behaviour. While this allows you to know in advance which Minions are going 
+  to reply, and which are unresponsive, it is more resource consuming, as it 
+  spreads an additional set of events, by executing a ``test.ping`` across your 
+  fleet.
+
+Among a few bug fixes, it's worth noting that extension modules you may have in 
+your own environment should now be working well when :ref:`ssh`.

docs/releases/index.rst

@@ -10,15 +10,17 @@ Latest Release
 .. toctree::
    :maxdepth: 1
 
-   2020.10.0
+   2021.6.0
 
 Previous Releases
 ^^^^^^^^^^^^^^^^^
 
 .. toctree::
    :maxdepth: 1
 
+   2020.10.0
    2020.7.0
    2020.3.0
    2020.2.0
    2019.10.0
+   2019.6.0

docs/runner.rst

@@ -7,9 +7,9 @@ The :ref:`proxy-runner` is the core functionality of ``salt-sproxy`` and can be
 used to trigger jobs as :ref:`events-reactions`, or invoked when
 :ref:`salt-api`.
 
-In both cases mentioned above you are going to need to have a Salt Master 
-running, that allows you to set up the Reactors and the Salt API; that means, 
-the ``proxy`` Runner needs to be available on your Master. To do so, you have 
+In both cases mentioned above you are going to need to have a Salt Master
+running, that allows you to set up the Reactors and the Salt API; that means,
+the ``proxy`` Runner needs to be available on your Master. To do so, you have
 two options:
 
 1. Reference it from the salt-sproxy installation
@@ -19,7 +19,7 @@ After installing salt-sproxy, you can execute the following command:
 
 .. code-block:: bash
 
-    $ salt-sproxy --file-roots
+    $ salt-sproxy --display-file-roots
     salt-sproxy is installed at: /home/mircea/venvs/salt-sproxy/lib/python3.6/site-packages/salt_sproxy
 
     You can configure the file_roots on the Master, e.g.,
@@ -36,17 +36,17 @@ After installing salt-sproxy, you can execute the following command:
 1.a. Update the ``file_roots`` and / or ``runner_dirs`` manually
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-As suggested in the output, you can directly reference the salt-sproxy 
-installation path to start using the ``proxy`` Runner (and other extension 
+As suggested in the output, you can directly reference the salt-sproxy
+installation path to start using the ``proxy`` Runner (and other extension
 modules included in the package).
 
-After updating the master configuration file, make sure to execute ``salt-run 
+After updating the master configuration file, make sure to execute ``salt-run
 saltutil.sync_all`` or ``salt-run saltutil.sync_runners``.
 
 1.b. Use the ``--save-file-roots`` CLI argument to update the master config
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-A simpler alternative is executing with ``--save-file-roots`` which adds the 
+A simpler alternative is executing with ``--save-file-roots`` which adds the
 path for you, and synchronizes the extension modules provided together with e.g.,
 
 .. code-block:: bash
@@ -64,14 +64,14 @@ path for you, and synchronizes the extension modules provided together with e.g.
 
 .. note::
 
-    While this option will preserve the configuration you have (but appending 
-    another path to ``file_roots`` and / or ``runner_dirs``), it may re-arrange 
+    While this option will preserve the configuration you have (but appending
+    another path to ``file_roots`` and / or ``runner_dirs``), it may re-arrange
     (visually) the contents - however without any side effects.
 
 2. Copy the source file
 -----------------------
 
-You can either download it from 
+You can either download it from
 https://github.com/mirceaulinic/salt-sproxy/blob/master/salt_sproxy/_runners/proxy.py,
 e.g., if your ``file_roots`` configuration on the Master looks like:
 
@@ -81,7 +81,7 @@ e.g., if your ``file_roots`` configuration on the Master looks like:
       base:
         - /srv/salt
 
-You are going to need to create a directory under ``/srv/salt/_runners``, then 
+You are going to need to create a directory under ``/srv/salt/_runners``, then
 download the ``proxy`` Runner there:
 
 .. code-block:: bash
@@ -92,10 +92,10 @@ download the ``proxy`` Runner there:
 
 .. note::
 
-    In the above I've used the *raw* like from GitHub to ensure the source code 
+    In the above I've used the *raw* like from GitHub to ensure the source code
     is preserved.
 
-Alternatively, you can also put it under an arbitrary path, e.g., 
+Alternatively, you can also put it under an arbitrary path, e.g.,
 (configuration on the Master)
 
 .. code-block:: yaml

examples/pillar_roster/README.rst

@@ -25,7 +25,7 @@ devices you're aiming to manage:
 
     $ salt-run pillar.show_pillar
     devices:
-      - name: device1
+      - id: device1
       ...
 
 It really doesn't matter where is Salt pulling this data from.
@@ -47,7 +47,7 @@ e.g., using ``curl``:
 .. code-block:: bash
 
     $ curl http://example.com/devices
-    {"devices": [{"name": "router1"}, {"name": "router2"}, {"name": "switch1"}]}
+    {"devices": [{"id": "router1"}, {"id": "router2"}, {"id": "switch1"}]}
 
 That being available, we can configure the ``http_json`` External Pillar:
 
@@ -67,9 +67,9 @@ Now, let's verify that the data is pulled properly into the Pillar:
 
   $ salt-run pillar.show_pillar
   devices:
-    - name: router1
-    - name: router2
-    - name: switch1
+    - id: router1
+    - id: router2
+    - id: switch1
 
 That being validated, salt-sproxy is now aware of all the devices to be 
 managed:
@@ -91,7 +91,7 @@ As well as other target types such as ``list`` or ``PCRE``:
   - router1
   - router2
 
-  # target all devices with the name starting with "router",
+  # target all devices with the Minion ID starting with "router",
   # followed by one or more numbers:
   
   $ salt-sproxy -E 'router\d+' --preview-target

requirements-dev.txt

@@ -1,5 +1,5 @@
 -r requirements.txt
-tox==3.20.1
+tox==3.21.4
 black==19.10b0
 pylint==2.6.0
 SaltPylint==2020.9.28

requirements.txt

@@ -1,3 +1,2 @@
-msgpack<=0.6.2
 salt
 progressbar2

salt_sproxy/_proxy/ssh.py

@@ -140,8 +140,9 @@ def _prep_conn(opts, fun, *args, **kwargs):
     for cli_opt in ('identities_only', 'priv', 'priv_passwd'):
         if opts.get(cli_opt) and not opts['proxy'].get(cli_opt):
             opts['proxy'][cli_opt] = opts[cli_opt]
+    ext_mods = salt.client.ssh.mod_data(fsclient)
     conn = salt.client.ssh.Single(
-        opts, argv, opts['id'], fsclient=fsclient, **opts['proxy']
+        opts, argv, opts['id'], fsclient=fsclient, mods=ext_mods, **opts['proxy']
     )
     conn.args = args
     conn.kwargs = kwargs

salt_sproxy/_roster/pillar.py

@@ -67,7 +67,10 @@ def targets(tgt, tgt_type='glob', **kwargs):
     pillar_devices = pillar[pillar_key]
     log.debug('Compiled the following list of devices from the Pillar')
     log.debug(pillar_devices)
-    pool = {device['name']: {'minion_opts': device} for device in pillar_devices}
+    pool = {
+        device.pop('id', device.pop('name')): {'minion_opts': device}
+        for device in pillar_devices
+    }
     pool = salt_sproxy._roster.load_cache(
         pool, __runner__, __opts__, tgt, tgt_type=tgt_type
     )