Add launcher for k8s to this repo (from remote-kernel-provider)

Since the kubernetes launcher contains code specific for kubernetes only
and already has synergy with the lifecycle manager, it seems more
appropriate to have this "pod launcher" code in this repo.  Pure kernel
launchers (with listeners) will remain in base provider.

Author: kevin-bates

.gitignore

@@ -103,3 +103,6 @@ ENV/
 
 # mypy
 .mypy_cache/
+
+# mac - desktop services meta-files
+.DS_Store

MANIFEST.in

@@ -4,6 +4,7 @@ include LICENSE
 include README.rst
 
 recursive-include kubernetes_kernel_provider/kernelspecs *
+recursive-include kubernetes_kernel_provider/pod-launcher *
 recursive-include kubernetes_kernel_provider/tests *
 
 recursive-exclude * __pycache__

kubernetes_kernel_provider/kernelspecapp.py

@@ -66,8 +66,6 @@ class K8SKP_SpecInstaller(JupyterApp):
     def _kernel_spec_manager_default(self):
         return KernelSpecManager(kernel_file=KubernetesKernelProvider.kernel_file)
 
-    source_dir = Unicode()
-    staging_dir = Unicode()
     template_dir = Unicode()
 
     kernel_name = Unicode(DEFAULT_KERNEL_NAMES[DEFAULT_LANGUAGE], config=True,
@@ -152,16 +150,21 @@ def start(self):
         self._validate_parameters()
 
         # create staging dir
-        self.staging_dir = spec_utils.create_staging_directory()
+        staging_dir = spec_utils.create_staging_directory()
 
         # copy files from installed area to staging dir
-        self.source_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'kernelspecs', self.template_dir))
-        dir_util.copy_tree(src=self.source_dir, dst=self.staging_dir)
-        spec_utils.copy_kernelspec_files(self.staging_dir, launcher_type='kubernetes',
+        source_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'kernelspecs', self.template_dir))
+        dir_util.copy_tree(src=source_dir, dst=staging_dir)
+
+        source_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pod-launcher'))
+        dir_util.copy_tree(src=source_dir, dst=staging_dir)
+
+        # copy appropriate resource files
+        spec_utils.copy_kernelspec_files(staging_dir, launcher_type=None,
                                          resource_type=TENSORFLOW if self.tensorflow else self.language)
         # install to destination
         self.log.info("Installing Kubernetes Kernel Provider kernel specification for '{}'".format(self.display_name))
-        install_dir = self.kernel_spec_manager.install_kernel_spec(self.staging_dir,
+        install_dir = self.kernel_spec_manager.install_kernel_spec(staging_dir,
                                                                    kernel_name=self.kernel_name,
                                                                    user=self.user,
                                                                    prefix=self.prefix)

kubernetes_kernel_provider/pod-launcher/scripts/kernel-pod.yaml.j2

@@ -0,0 +1,71 @@
+# This file defines the Kubernetes objects necessary for Enterprise Gateway kernels to run witihin Kubernetes.
+# Substitution parameters are processed by the launch_kubernetes.py code located in the
+# same directory.  Some values are factory values, while others (typically prefixed with 'kernel_') can be
+# provided by the client.
+#
+# This file can be customized as needed.  No changes are required to launch_kubernetes.py provided kernel_
+# values are used - which be automatically set from corresponding KERNEL_ env values.  Updates will be required
+# to launch_kubernetes.py if new document sections (i.e., new k8s 'kind' objects) are introduced.
+#
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ kernel_pod_name }}"
+  namespace: "{{ kernel_namespace }}"
+  labels:
+    kernel_id: "{{ kernel_id }}"
+    app: enterprise-gateway
+    component: kernel
+spec:
+  restartPolicy: Never
+  serviceAccountName: "{{ kernel_service_account_name }}"
+# NOTE: that using runAsGroup requires that feature-gate RunAsGroup be enabled.
+# WARNING: Only using runAsUser w/o runAsGroup or NOT enabling the RunAsGroup feature-gate
+# will result in the new kernel pod's effective group of 0 (root)! although the user will
+# correspond to the runAsUser value.  As a result, BOTH should be uncommented AND the feature-gate
+# should be enabled to ensure expected behavior.  In addition, 'fsGroup: 100' is recommended so
+# that /home/jovyan can be written to via the 'users' group (gid: 100) irrespective of the
+# "kernel_uid" and "kernel_gid" values.
+  {% if kernel_uid is defined or kernel_gid is defined %}
+  securityContext:
+    {% if kernel_uid is defined %}
+    runAsUser: {{ kernel_uid | int }}
+    {% endif %}
+    {% if kernel_gid is defined %}
+    runAsGroup: {{ kernel_gid | int }}
+    {% endif %}
+    fsGroup: 100
+  {% endif %}
+  containers:
+  - env:
+    - name: EG_RESPONSE_ADDRESS
+      value: "{{ eg_response_address }}"
+    - name: KERNEL_LANGUAGE
+      value: "{{ kernel_language }}"
+    - name: KERNEL_SPARK_CONTEXT_INIT_MODE
+      value: "{{ kernel_spark_context_init_mode }}"
+    - name: KERNEL_NAME
+      value: "{{ kernel_name }}"
+    - name: KERNEL_USERNAME
+      value: "{{ kernel_username }}"
+    - name: KERNEL_ID
+      value: "{{ kernel_id }}"
+    - name: KERNEL_NAMESPACE
+      value: "{{ kernel_namespace }}"
+    image: "{{ kernel_image }}"
+    name: "{{ kernel_pod_name }}"
+    {% if kernel_working_dir is defined %}
+    workingDir: "{{ kernel_working_dir }}"
+    {% endif %}
+    {% if kernel_volume_mounts is defined %}
+    volumeMounts:
+      {% for volume_mount in kernel_volume_mounts %}
+    - {{ volume_mount }}
+      {% endfor %}
+    {% endif %}
+  {% if kernel_volumes is defined %}
+  volumes:
+    {% for volume in kernel_volumes %}
+  - {{ volume }}
+    {% endfor %}
+  {% endif %}

kubernetes_kernel_provider/pod-launcher/scripts/launch_kubernetes.py

@@ -0,0 +1,105 @@
+import os
+import sys
+import yaml
+import argparse
+from kubernetes import client, config
+import urllib3
+
+from jinja2 import FileSystemLoader, Environment
+
+urllib3.disable_warnings()
+
+KERNEL_POD_TEMPLATE_PATH = '/kernel-pod.yaml.j2'
+
+
+def generate_kernel_pod_yaml(keywords):
+    """Return the kubernetes pod spec as a yaml string.
+
+    - load jinja2 template from this file directory.
+    - substitute template variables with keywords items.
+    """
+    j_env = Environment(loader=FileSystemLoader(os.path.dirname(__file__)), trim_blocks=True, lstrip_blocks=True)
+    # jinja2 template substitutes template variables with None though keywords doesn't contain corresponding item.
+    # Therefore, no need to check if any are left unsubstituted; Kubernetes API server will validate the pod spec.
+    k8s_yaml = j_env.get_template(KERNEL_POD_TEMPLATE_PATH).render(**keywords)
+
+    return k8s_yaml
+
+
+def launch_kubernetes_kernel(kernel_id, response_addr, spark_context_init_mode):
+    # Launches a containerized kernel as a kubernetes pod.
+
+    config.load_incluster_config()
+
+    # Capture keywords and their values.
+    keywords = dict()
+
+    # Factory values...
+    # Since jupyter lower cases the kernel directory as the kernel-name, we need to capture its case-sensitive
+    # value since this is used to locate the kernel launch script within the image.
+    keywords['kernel_name'] = os.path.basename(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    keywords['kernel_id'] = kernel_id
+    keywords['eg_response_address'] = response_addr
+    keywords['kernel_spark_context_init_mode'] = spark_context_init_mode
+
+    # Walk env variables looking for names prefixed with KERNEL_.  When found, set corresponding keyword value
+    # with name in lower case.
+    for name, value in os.environ.items():
+        if name.startswith('KERNEL_'):
+            keywords[name.lower()] = yaml.safe_load(value)
+
+    # Substitute all template variable (wrapped with {{ }}) and generate `yaml` string.
+    k8s_yaml = generate_kernel_pod_yaml(keywords)
+
+    # For each k8s object (kind), call the appropriate API method.  Too bad there isn't a method
+    # that can take a set of objects.
+    #
+    # Creation for additional kinds of k8s objects can be added below.  Refer to
+    # https://github.com/kubernetes-client/python for API signatures.  Other examples can be found in
+    # https://github.com/jupyter-incubator/enterprise_gateway/blob/master/enterprise_gateway/services/processproxies/k8s.py
+    #
+    kernel_namespace = keywords['kernel_namespace']
+    k8s_objs = yaml.safe_load_all(k8s_yaml)
+    for k8s_obj in k8s_objs:
+        if k8s_obj.get('kind'):
+            if k8s_obj['kind'] == 'Pod':
+                # print("{}".format(k8s_obj))  # useful for debug
+                client.CoreV1Api(client.ApiClient()).create_namespaced_pod(body=k8s_obj, namespace=kernel_namespace)
+            elif k8s_obj['kind'] == 'Secret':
+                client.CoreV1Api(client.ApiClient()).create_namespaced_secret(body=k8s_obj, namespace=kernel_namespace)
+            elif k8s_obj['kind'] == 'PersistentVolumeClaim':
+                client.CoreV1Api(client.ApiClient()).create_namespaced_persistent_volume_claim(
+                    body=k8s_obj, namespace=kernel_namespace)
+            elif k8s_obj['kind'] == 'PersistentVolume':
+                client.CoreV1Api(client.ApiClient()).create_persistent_volume(body=k8s_obj)
+            else:
+                sys.exit("ERROR - Unhandled Kubernetes object kind '{}' found in yaml file - "
+                         "kernel launch terminating!".format(k8s_obj['kind']))
+        else:
+            sys.exit("ERROR - Unknown Kubernetes object '{}' found in yaml file - kernel launch terminating!".
+                     format(k8s_obj))
+
+
+if __name__ == '__main__':
+    """
+        Usage: launch_kubernetes_kernel
+                    [--RemoteProcessProxy.kernel-id <kernel_id>]
+                    [--RemoteProcessProxy.response-address <response_addr>]
+                    [--RemoteProcessProxy.spark-context-initialization-mode <mode>]
+    """
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--RemoteProcessProxy.kernel-id', dest='kernel_id', nargs='?',
+                        help='Indicates the id associated with the launched kernel.')
+    parser.add_argument('--RemoteProcessProxy.response-address', dest='response_address', nargs='?',
+                        metavar='<ip>:<port>', help='Connection address (<ip>:<port>) for returning connection file')
+    parser.add_argument('--RemoteProcessProxy.spark-context-initialization-mode', dest='spark_context_init_mode',
+                        nargs='?', help='Indicates whether or how a spark context should be created',
+                        default='none')
+
+    arguments = vars(parser.parse_args())
+    kernel_id = arguments['kernel_id']
+    response_addr = arguments['response_address']
+    spark_context_init_mode = arguments['spark_context_init_mode']
+
+    launch_kubernetes_kernel(kernel_id, response_addr, spark_context_init_mode)

kubernetes_kernel_provider/tests/test_kernelspec_app.py

@@ -107,6 +107,12 @@ def test_create_python_kernelspec(script_runner, mock_kernels_dir):
     assert os.path.isdir(os.path.join(mock_kernels_dir, 'kernels', 'my_python_kernel'))
     assert os.path.isfile(os.path.join(mock_kernels_dir, 'kernels', 'my_python_kernel', 'k8skp_kernel.json'))
 
+    assert os.path.isdir(os.path.join(mock_kernels_dir, 'kernels', 'my_python_kernel', 'scripts'))
+    assert os.path.isfile(os.path.join(mock_kernels_dir, 'kernels', 'my_python_kernel', 'scripts',
+                                       'launch_kubernetes.py'))
+    assert os.path.isfile(os.path.join(mock_kernels_dir, 'kernels', 'my_python_kernel', 'scripts',
+                                       'kernel-pod.yaml.j2'))
+
     with open(os.path.join(mock_kernels_dir, 'kernels', 'my_python_kernel', 'k8skp_kernel.json'), "r") as fd:
         kernel_json = json.load(fd)
         assert kernel_json["display_name"] == 'My Python Kernel'