Update with async management support

Author: kevin-bates

.travis.yml

@@ -1,9 +1,10 @@
 language: python
 
 python:
+    - 3.8
+    - 3.7
     - 3.6
     - 3.5
-    - 3.4
 
 install:
   - pip install --upgrade setuptools pip flake8 pytest-cov codecov pytest-console-scripts

README.md

@@ -13,7 +13,7 @@ The Kubernetes Kernel Provider package provides support necessary for launching
 pip install kubenetes_kernel_provider
 ```
 
-##Usage
+## Usage
 Because this version of Jupyter kernel management is still in its experimental stages, a [special branch of Notebook](https://github.com/takluyver/notebook/tree/jupyter-kernel-mgmt) is required, which includes the machinery to leverage the new framework.  An installable build of this branch is available as an asset on the [interim-dev release](https://github.com/gateway-experiments/remote_kernel_provider/releases/tag/v0.1-interim-dev) of the Remote Kernel Provider on which Kubernetes Kernel Provider depends.
 
 ### Kubernetes Kernel Specifications

kubernetes_kernel_provider/k8s.py

@@ -19,7 +19,11 @@
 enterprise_gateway_namespace = os.environ.get('EG_NAMESPACE', 'default')
 default_kernel_service_account_name = os.environ.get('EG_DEFAULT_KERNEL_SERVICE_ACCOUNT_NAME', 'default')
 kernel_cluster_role = os.environ.get('EG_KERNEL_CLUSTER_ROLE', 'cluster-admin')
-share_gateway_namespace = bool(os.environ.get('EG_SHARED_NAMESPACE', 'False').lower() == 'true')
+
+# TODO: The default for this value should probably flip for single-user/Notebook scenarios (True) vs.
+# multi-tenant/Gateway scenarios (False).  The app config will be available from `kernel_manager.app_config`
+# so we should be able to move this into constructor (or after) and infer from that information.
+shared_namespace = bool(os.environ.get('EG_SHARED_NAMESPACE', 'False').lower() == 'true')
 
 config.load_incluster_config()
 
@@ -33,7 +37,7 @@ def __init__(self, kernel_manager, lifecycle_config):
         self.kernel_namespace = None
         self.delete_kernel_namespace = False
 
-    def launch_process(self, kernel_cmd, **kwargs):
+    async def launch_process(self, kernel_cmd, **kwargs):
         """Launches the specified process within a Kubernetes environment."""
         # Set env before superclass call so we see these in the debug output
 
@@ -43,7 +47,7 @@ def launch_process(self, kernel_cmd, **kwargs):
         self.kernel_pod_name = self._determine_kernel_pod_name(**kwargs)
         self.kernel_namespace = self._determine_kernel_namespace(**kwargs)  # will create namespace if not provided
 
-        return super(KubernetesKernelLifecycleManager, self).launch_process(kernel_cmd, **kwargs)
+        return await super(KubernetesKernelLifecycleManager, self).launch_process(kernel_cmd, **kwargs)
 
     def get_initial_states(self):
         """Return list of states indicating container is starting (includes running)."""
@@ -146,7 +150,7 @@ def _determine_kernel_namespace(self, **kwargs):
         namespace = kwargs['env'].get('KERNEL_NAMESPACE')
         if namespace is None:
             # check if share gateway namespace is configured...
-            if share_gateway_namespace:  # if so, set to EG namespace
+            if shared_namespace:  # if so, set to EG namespace
                 namespace = enterprise_gateway_namespace
                 self.log.warning("Shared namespace has been configured.  All kernels will reside in EG namespace: {}".
                                  format(namespace))

kubernetes_kernel_provider/provider.py

@@ -1,4 +1,5 @@
 """Provides support for launching and managing kernels within a Kubernetes cluster."""
+import asyncio
 import os
 from datetime import datetime
 from kubernetes import config
@@ -17,6 +18,7 @@ class KubernetesKernelProvider(RemoteKernelProviderBase):
     kernel_file = 'k8skp_kernel.json'
     lifecycle_manager_classes = ['kubernetes_kernel_provider.k8s.KubernetesKernelLifecycleManager']
 
+    @asyncio.coroutine
     def find_kernels(self):
         """ Ensures the provider is running within a Kubernetes cluster.  If not, it will
             log a warning message and no kernelspecs will be returned.  Since find_kernels()

kubernetes_kernel_provider/tests/test_kernelspec_app.py

@@ -10,7 +10,7 @@
 from tempfile import mkdtemp
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture()
 def mock_kernels_dir():
     kernels_dir = mkdtemp(prefix="kernels_")
     orig_data_dir = os.environ.get("JUPYTER_DATA_DIR")
@@ -23,55 +23,48 @@ def mock_kernels_dir():
         os.environ.pop("JUPYTER_DATA_DIR")
 
 
-@pytest.mark.script_launch_mode('subprocess')
 def test_no_opts(script_runner):
     ret = script_runner.run('jupyter-k8s-kernelspec')
     assert ret.success is False
     assert ret.stdout.startswith("No subcommand specified.")
     assert ret.stderr == ''
 
 
-@pytest.mark.script_launch_mode('subprocess')
 def test_bad_subcommand(script_runner):
     ret = script_runner.run('jupyter', 'k8s-kernelspec', 'bogus-subcommand')
     assert ret.success is False
     assert ret.stdout.startswith("No subcommand specified.")
     assert ret.stderr == ''
 
 
-@pytest.mark.script_launch_mode('subprocess')
 def test_help_all(script_runner):
     ret = script_runner.run('jupyter-k8s-kernelspec', 'install', '--help-all')
     assert ret.success
     assert ret.stdout.startswith("A Jupyter kernel for use within a Kubernetes cluster")
     assert ret.stderr == ''
 
 
-@pytest.mark.script_launch_mode('subprocess')
 def test_bad_argument(script_runner):
     ret = script_runner.run('jupyter-k8s-kernelspec', 'install', '--bogus-argument')
     assert ret.success is False
     assert ret.stdout.startswith("A Jupyter kernel for use within a Kubernetes cluster")
     assert "[K8SKP_SpecInstaller] CRITICAL | Unrecognized flag: \'--bogus-argument\'" in ret.stderr
 
 
-@pytest.mark.script_launch_mode('subprocess')
 def test_mutually_exclusive(script_runner):
     ret = script_runner.run('jupyter-k8s-kernelspec', 'install', '--spark', '--tensorflow')
     assert ret.success is False
     assert ret.stdout == ''
     assert "[K8SKP_SpecInstaller] ERROR | Tensorflow support is mutually exclusive with Spark support." in ret.stderr
 
 
-@pytest.mark.script_launch_mode('subprocess')
 def test_bad_language(script_runner):
     ret = script_runner.run('jupyter-k8s-kernelspec', 'install', '--language=R', '--tensorflow')
     assert ret.success is False
     assert ret.stdout == ''
     assert "[K8SKP_SpecInstaller] ERROR | Tensorflow support is only available for use by Python kernels." in ret.stderr
 
 
-@pytest.mark.script_launch_mode('subprocess')
 def test_create_kernelspec(script_runner, mock_kernels_dir):
     my_env = os.environ.copy()
     my_env.update({"JUPYTER_DATA_DIR": mock_kernels_dir})
@@ -93,7 +86,6 @@ def test_create_kernelspec(script_runner, mock_kernels_dir):
             'elyra/kernel-spark-py:dev'
 
 
-@pytest.mark.script_launch_mode('subprocess')
 def test_create_python_kernelspec(script_runner, mock_kernels_dir):
     my_env = os.environ.copy()
     my_env.update({"JUPYTER_DATA_DIR": mock_kernels_dir})
@@ -120,7 +112,6 @@ def test_create_python_kernelspec(script_runner, mock_kernels_dir):
         assert kernel_json["metadata"]["lifecycle_manager"]["config"]["image_name"] == 'foo/bar:zed'
 
 
-@pytest.mark.script_launch_mode('subprocess')
 def test_create_r_kernelspec(script_runner, mock_kernels_dir):
     my_env = os.environ.copy()
     my_env.update({"JUPYTER_DATA_DIR": mock_kernels_dir})
@@ -142,7 +133,6 @@ def test_create_r_kernelspec(script_runner, mock_kernels_dir):
         assert argv[len(argv) - 1] == 'lazy'
 
 
-@pytest.mark.script_launch_mode('subprocess')
 def test_create_scala_kernelspec(script_runner, mock_kernels_dir):
     my_env = os.environ.copy()
     my_env.update({"JUPYTER_DATA_DIR": mock_kernels_dir})
@@ -163,7 +153,6 @@ def test_create_scala_kernelspec(script_runner, mock_kernels_dir):
         assert '--MyExtraSparkOpts' in kernel_json["env"]["__TOREE_SPARK_OPTS__"]
 
 
-@pytest.mark.script_launch_mode('subprocess')
 def test_create_tensorflow_kernelspec(script_runner, mock_kernels_dir):
     my_env = os.environ.copy()
     my_env.update({"JUPYTER_DATA_DIR": mock_kernels_dir})

pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+markers =
+    script_launch_mode
+
+script_launch_mode = subprocess

setup.py

@@ -63,11 +63,11 @@ def run(self):
     install_requires = [
         'entrypoints',
         'kubernetes>=4.0.0',
-        'jupyter_kernel_mgmt>=0.4.0',
-        'remote_kernel_provider>=0.2.0',
+        'jupyter_kernel_mgmt>=0.5.0',
+        'remote_kernel_provider>=0.3.0',
     ],
     extras_require   = {
-        'test': ['mock', 'pytest'],
+        'test': ['mock', 'pytest', 'pytest-console-scripts'],
     },
     entry_points={
         'console_scripts': [
@@ -77,7 +77,7 @@ def run(self):
             'k8skp = kubernetes_kernel_provider.provider:KubernetesKernelProvider',
         ]
     },
-    python_requires = ">=3.4",
+    python_requires = ">=3.5",
     cmdclass         = {
         'bdist_egg': bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled,
     },