Merge pull request #355 from delphix/release

Release 3.1.0

Author: mothslaw

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.0.0
+current_version = 3.1.0
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<dev>\d+))?

.github/workflows/publish-docs.yml

@@ -9,6 +9,7 @@ on:
       - 'docs/**'
     paths:
       - 'docs/**'
+      - '.github/workflows/publish-docs.yml'
 
 jobs:
   publish:
@@ -17,6 +18,7 @@ jobs:
       matrix:
         python-version: [3.7]
         package: [docs]
+        repository: ['delphix/virtualization-sdk']
 
     steps:
       - uses: actions/checkout@v2
@@ -69,7 +71,18 @@ jobs:
         working-directory: ${{ matrix.package }}
         run: |
           pipenv run mkdocs build --clean
-      - name: Deploy the contents of docs/site to gh-pages 🚀
+      - name: Deploy the contents of docs/site to gh-pages (developer.delphix.com) 🚀
+        if: ${{ github.repository == matrix.repository }}
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: docs/site
+          commit_message: Deploy to gh-pages 🚀
+          user_name: "github-actions[bot]"
+          user_email: "github-actions[bot]@users.noreply.github.com"
+          cname: developer.delphix.com
+      - name: Deploy the contents of docs/site to personal gh-pages 🚀
+        if: ${{ github.repository != matrix.repository }}
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/publish-python-packages.yml

@@ -6,9 +6,10 @@ on:
     branches:
       - master
       - develop
-      - 'release/**'
+      - release
     paths:
       - 'dvp/src/main/python/dlpx/virtualization/VERSION'
+      - '.github/workflows/publish-python-packages.yml'
 
 jobs:
   publish:
@@ -39,4 +40,3 @@ jobs:
         run: |
           python setup.py sdist bdist_wheel
           twine upload dist/*
-

common/requirements.txt

@@ -1,5 +1,6 @@
 bump2version==0.5.11
 contextlib2==0.6.0.post1 ; python_version < '3'
+enum34==1.1.6
 funcsigs==1.0.2 ; python_version < '3.0'
 importlib-metadata==0.23 ; python_version < '3.8'
 more-itertools==5.0.0 ; python_version <= '2.7'

common/setup.py

@@ -4,7 +4,7 @@
 PYTHON_SRC = 'src/main/python'
 
 install_requires = [
-  "dvp-api == 1.4.0",
+  "dvp-api == 1.5.0",
 ]
 
 with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file:

common/src/main/python/dlpx/virtualization/common/VERSION

@@ -1 +1 @@
-3.0.0
+3.1.0

common/src/main/python/dlpx/virtualization/common/_common_classes.py

@@ -2,8 +2,10 @@
 # Copyright (c) 2019 by Delphix. All rights reserved.
 #
 
-from dlpx.virtualization.api import common_pb2
+from abc import ABCMeta
+from dlpx.virtualization.api import common_pb2, libs_pb2
 from dlpx.virtualization.common.exceptions import IncorrectTypeError
+from enum import IntEnum
 
 """Classes used for Plugin Operations
 
@@ -17,7 +19,10 @@
     "RemoteConnection",
     "RemoteEnvironment",
     "RemoteHost",
-    "RemoteUser"]
+    "RemoteUser",
+    "Credentials",
+    "PasswordCredentials",
+    "KeyPairCredentials"]
 
 
 class RemoteConnection(object):
@@ -293,3 +298,121 @@ def from_proto(user):
                 common_pb2.RemoteUser)
         remote_user = RemoteUser(name=user.name, reference=user.reference)
         return remote_user
+
+
+class Credentials(object):
+    """Plugin base class for CredentialsResult to be used for plugin operations
+    and library functions.
+
+    Plugin authors should use this instead of the corresponding protobuf generated
+    class.
+
+    Args:
+        username: User name.
+    """
+    def __init__(self, username):
+        if not isinstance(username, basestring):
+            raise IncorrectTypeError(
+                Credentials,
+                'username',
+                type(username),
+                basestring)
+        self.__username = username
+
+    __metaclass__ = ABCMeta
+
+    @property
+    def username(self):
+        return self.__username
+
+
+class PasswordCredentials(Credentials):
+    """Plugin class for CredentialsResult with password to be used for plugin operations
+    and library functions.
+
+    Plugin authors should use this instead of the corresponding protobuf generated
+    class.
+
+    Args:
+        username: User name.
+        password: Password.
+    """
+    def __init__(self, username, password):
+        super(PasswordCredentials, self).__init__(username)
+        if not isinstance(password, basestring):
+            raise IncorrectTypeError(
+                PasswordCredentials,
+                'password',
+                type(password),
+                basestring)
+        self.__password = password
+
+    @property
+    def password(self):
+        return self.__password
+
+    @staticmethod
+    def from_proto(credentials_result):
+        """Converts protobuf class libs_pb2.CredentialsResult to plugin class PasswordCredentials
+        """
+        if not isinstance(credentials_result, libs_pb2.CredentialsResult):
+            raise IncorrectTypeError(
+                PasswordCredentials,
+                'credentials_result',
+                type(credentials_result),
+                libs_pb2.CredentialsResult)
+        return PasswordCredentials(
+            username=credentials_result.username, password=credentials_result.pasword)
+
+
+class KeyPairCredentials(Credentials):
+    """Plugin class for CredentialsResult with key pair to be used for plugin operations
+    and library functions.
+
+    Plugin authors should use this instead of the corresponding protobuf generated
+    class.
+
+    Args:
+        username (str): User name.
+        private_key (str): Private key.
+        public_key (str): Public key corresponding to private key. Empty string if not present.
+    """
+    def __init__(self, username, private_key, public_key):
+        super(KeyPairCredentials, self).__init__(username)
+        if not isinstance(private_key, basestring):
+            raise IncorrectTypeError(
+                KeyPairCredentials,
+                'private_key',
+                type(private_key),
+                basestring)
+        self.__private_key = private_key
+        if not isinstance(public_key, basestring):
+            raise IncorrectTypeError(
+                KeyPairCredentials,
+                'public_key',
+                type(public_key),
+                basestring)
+        self.__public_key = public_key
+
+    @property
+    def private_key(self):
+        return self.__private_key
+
+    @property
+    def public_key(self):
+        return self.__public_key
+
+    @staticmethod
+    def from_proto(credentials_result):
+        """Converts protobuf class libs_pb2.CredentialsResult to plugin class KeyPairCredentials
+        """
+        if not isinstance(credentials_result, libs_pb2.CredentialsResult):
+            raise IncorrectTypeError(
+                KeyPairCredentials,
+                'credentials_result',
+                type(credentials_result),
+                libs_pb2.CredentialsResult)
+        return KeyPairCredentials(
+            username=credentials_result.username,
+            private_key=credentials_result.key_pair.private_key,
+            public_key=credentials_result.key_pair.public_key)

docs/docs/Best_Practices/Sensitive_Data.md

@@ -10,7 +10,7 @@ Plugins must be careful to handle sensitive data appropriately. Three tips for h
 
 Each of these tips are explained below.
 
-# Marking Your Data As Sensitive
+## Marking Your Data As Sensitive
 
 Because the Delphix Engine manages the storing and retrieving of plugin-defined data, it needs to know which pieces of data are sensitive. The plugin does this in its [schemas](/References/Glossary.md#schema), by using the special [`password`](/References/Schemas.md#password) keyword.
 
@@ -37,11 +37,45 @@ This tells the Delphix Engine to take special precautions with this password pro
 !!! note
     Removing a previously added password property from a field and running a [Data Migration](/References/Glossary.md#data-migration) will expose the password in plaintext. If this is intentional, write a migration to ensure that the new property conforms to the new schema.
 
-# Using Environment Variables For Remote Data Passing
+## Protecting Sensitive Data with Password Vaults
+
+Plugins can also leverage the password vaults configured in the Delphix engine to avoid storing sensitive data in the engine itself. In addition, vaults can rotate secrets seamlessly behind the scenes without requiring Delphix users to update those secrets in the engine. To give users the option to choose between directly entering a secret, such as a password or private key, or retrieving it from a vault, Delphix provides [pre-defined credential types](/References/Schemas.md#delphix-specific-pre-defined-types).
+
+When using these special types, the example above becomes:
+
+```json
+{
+    "type": "object",
+    "properties": {
+        "db_connectionPort": {"type": "string"},
+        "db_credentials_supplier": {
+          "$ref": "https://delphix.com/platform/api#/definitions/passwordCredentialsSupplier"
+        }
+    }
+}
+```
+
+For details on how the user can provide the information required for a property such as `db_credentials_supplier`, see the [section on pre-defined types](/References/Schemas.md#delphix-specific-pre-defined-types).
+
+At runtime, the plugin code must convert the credentials information provided by the user into an actual set of credentials that the plugin can use. To do this, the plugin must call the library function [`retrieve_credentials`](/References/Platform_Libraries.md#retrieve_credentials). For example:
+
+```python
+from dlpx.virtualization import libs
+from dlpx.virtualization.common import PasswordCredentials
+...
+@plugin.virtual.stop()
+def my_virtual_stop(virtual_source, repository, source_config):
+  credentials = libs.retrieve_credentials(virtual_source.parameters.db_credentials_supplier)
+  assert isinstance(credentials, PasswordCredentials)
+  connect_to_dbms(credentials.username, credentials.password)
+```
+
+
+## Using Environment Variables For Remote Data Passing
 
 Sometimes, a plugin will need to pass sensitive data to a remote environment. For example, perhaps a database command needs to be run on a [staging environment](/References/Glossary.md#staging-environment), and that database command will need to use a password.
 
-## Example
+### Example
 Let us take a look at a very simple example where we need to shutdown a database called "inventory" on a target environment by using the `db_cmd shutdown inventory` command. This command will ask for a password on `stdin`, and for our example our password is "hunter2".
 
 If we were running this command by hand, it might look like this:
@@ -59,7 +93,7 @@ Since a plugin cannot type in the password by hand, it will do something like th
 $ echo "hunter2" | db_cmd shutdown inventory
 ```
 
-## Don't Do This
+### Don't Do This
 
 First, let us take a look at how **not** to do this! Here is a bit of plugin python code that will run the above command.
 
@@ -80,7 +114,7 @@ This constructs a Python string containing exactly the desired command from abov
 
 The problem here is that there is a cleartext password in the Python string. But, this Python string is not treated as sensitive by the Virtualization Platform. For example, suppose the Virtualization Platform cannot make a connection to the target environment. In which case, it will raise an error containing the Python string, so that people will know what command failed. But, in our example, that would result in the password being part of the cleartext error message.
 
-## Using Environment Variables
+### Using Environment Variables
 
 The Delphix Engine provides a better way to pass sensitive data to remote bash (or powershell) calls: environment variables. Let us look at a different way to run the same command as above.
 
@@ -106,7 +140,7 @@ Once the command runs on the target environment, Bash will substitute in the pas
 
 Unlike with the command string, the Virtualization Platform **does** treat environment variables as sensitive information, and will not include them in error messages or internal logs, etc.
 
-# Don't Write Out Sensitive Data
+## Don't Write Out Sensitive Data
 
 Plugin writers are strongly advised to never write out unencrypted sensitive data. This is common-sense general advice that applies to all areas of programming, not just for plugins. However, there are a couple of special concerns for plugins.
 

docs/docs/Building_Your_First_Plugin/Data_Ingestion.md

@@ -21,7 +21,7 @@ quite limiting.
 
 For our first plugin, we will be using the more flexible [staging](/References/Glossary.md#staged-linkingsyncing) strategy. With this strategy, the Delphix Engine uses NFS for Unix environments (or iSCSI on Windows environments) to mount storage onto a [staging environment](/References/Glossary.md#staging-environment). Our plugin will then be in full control of how to get data from the source environment onto this storage mount.
 
-With the staging strategy, there are two types of syncs: sync and resync. A `sync` is used to ingest incremental changes while a `resync` is used to re-ingest all the data for the dSource. For databases, this could mean re-ingesting from a full database backup to reset the dSource. A `sync` and a `resync` will execute the same plugin operations. To differentiate a `sync` from a `resync`, simply add a boolean property (i.e. `resync`) in the plugin's [snapshot parameters definition](References/Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema). Once `sync` or `resync` is selected, the property will be passed into [linked.pre_snapshot](/References/Plugin_Operations.md#staged-linked-source-pre-snapshot) and [linked.post_snapshot](/References/Plugin_Operations.md#staged-linked-source-post-snapshot) as a [snapshot parameter](/References/Glossary.md#snapshot-parameters).
+With the staging strategy, there are two types of syncs: sync and resync. A `sync` is used to ingest incremental changes while a `resync` is used to re-ingest all the data for the dSource. For databases, this could mean re-ingesting from a full database backup to reset the dSource. A `sync` and a `resync` will execute the same plugin operations. To differentiate a `sync` from a `resync`, simply add a boolean property (i.e. `resync`) in the plugin's [snapshot parameters definition](/References/Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema). Once `sync` or `resync` is selected, the property will be passed into [linked.pre_snapshot](/References/Plugin_Operations.md#staged-linked-source-pre-snapshot) and [linked.post_snapshot](/References/Plugin_Operations.md#staged-linked-source-post-snapshot) as a [snapshot parameter](/References/Glossary.md#snapshot-parameters).
 
 A regular `sync` is the default and is executed as part of policy driven syncs. A `resync` is only executed during initial ingestion or if the Delphix user manually starts one. The customer can manually trigger a `resync` via the UI by selecting the dSource, going to more options and selecting **Resynchronize dSource**. ![Screenshot](images/Resync.png)