From 151528d966955d52d78f0db9c720d672c3a7a053 Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Mon, 6 May 2024 22:02:15 +0200 Subject: [PATCH 01/31] Add checking custom kernel specs --- jupyter_client/manager.py | 27 +++++++++++++++++++ jupyter_client/multikernelmanager.py | 1 + .../provisioning/local_provisioner.py | 4 +++ 3 files changed, 32 insertions(+) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 088acd6c..baaf8166 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -300,6 +300,26 @@ def update_env(self, *, env: t.Dict[str, str]) -> None: ): self._launch_args["env"].update(env) # type: ignore [unreachable] + def replace_spec_parameter(self, variable, value, spec): + regexp = r"\{"+variable+"\}" + pattern = re.compile(regexp) + return pattern.sub(pattern, value, spec) + + def update_kernel_specs(self, custom_kernel_specs: dict[str, t.Any] = None)-> None: + assert self.kernel_spec is not None +#go through custom kernel specs + for custom_kernel_spec, custom_kernel_spec_value in custom_kernel_specs.items(): + #go through kernel specs + for kernel_spec, kernel_spec_value in self.kernel_spec: + #if kernel spec is array then go through kernel this array + if isinstance(kernel_spec_value, list): + for kernel_spec_item in kernel_spec: + kernel_spec_item = self.replace_spec_parameter(custom_kernel_spec, custom_kernel_spec_value, kernel_spec_item) + else: + kernel_spec_value = self.replace_spec_parameter(custom_kernel_spec, custom_kernel_spec_value, kernel_spec_value) + return self.kernel_spec + + def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> t.List[str]: """Replace templated args (e.g. {connection_file})""" extra_arguments = extra_arguments or [] @@ -333,6 +353,8 @@ def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> ns["resource_dir"] = self.kernel_spec.resource_dir assert isinstance(self._launch_args, dict) + print("----format_kernel_cmd---_launch_args---") + print(self._launch_args) ns.update(self._launch_args) pat = re.compile(r"\{([A-Za-z0-9_]+)\}") @@ -391,6 +413,10 @@ async def _async_pre_start_kernel( # save kwargs for use in restart # assigning Traitlets Dicts to Dict make mypy unhappy but is ok self._launch_args = kw.copy() # type:ignore [assignment] + print("------_async_pre_start_kernel---start---") + print(self._launch_args) + print("------_async_pre_start_kernel---end---") + if self.provisioner is None: # will not be None on restarts self.provisioner = KPF.instance(parent=self.parent).create_provisioner_instance( self.kernel_id, @@ -431,6 +457,7 @@ async def _async_start_kernel(self, **kw: t.Any) -> None: keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ + #here? self._attempted_start = True kernel_cmd, kw = await self._async_pre_start_kernel(**kw) diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index d14a3f84..a65b7aa1 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -259,6 +259,7 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: The kernel ID for the newly started kernel is returned. """ + #here km, kernel_name, kernel_id = self.pre_start_kernel(kernel_name, kwargs) if not isinstance(km, KernelManager): self.log.warning( # type:ignore[unreachable] diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index 42d8d32d..9fc04e76 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -155,6 +155,7 @@ async def cleanup(self, restart: bool = False) -> None: lpc.return_port(port) async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: + # """Perform any steps in preparation for kernel process launch. This includes applying additional substitutions to the kernel launch command and env. @@ -166,6 +167,8 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: # This should be considered temporary until a better division of labor can be defined. km = self.parent if km: + if "custom_kernel_specs" in kwargs: + km.update_kernel_specs(kwargs["custom_kernel_specs"]) if km.transport == "tcp" and not is_local_ip(km.ip): msg = ( "Can only launch a kernel on a local interface. " @@ -207,6 +210,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> KernelConnectionInfo: """Launch a kernel with a command.""" scrubbed_kwargs = LocalProvisioner._scrub_kwargs(kwargs) + # self.process = launch_kernel(cmd, **scrubbed_kwargs) pgid = None if hasattr(os, "getpgid"): From f5fb1783bd717b782ca2f3541a15756f7b49ec0e Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Wed, 8 May 2024 00:27:05 +0200 Subject: [PATCH 02/31] Add replacing custom kernel specs --- jupyter_client/manager.py | 64 +++++++++++++------ jupyter_client/multikernelmanager.py | 2 + .../provisioning/local_provisioner.py | 8 ++- 3 files changed, 50 insertions(+), 24 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index baaf8166..bc69db87 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -292,40 +292,59 @@ def update_env(self, *, env: t.Dict[str, str]) -> None: .. version-added: 8.5 """ + print("update_env ----- here- no multi") # Mypy think this is unreachable as it see _launch_args as Dict, not t.Dict if ( isinstance(self._launch_args, dict) and "env" in self._launch_args and isinstance(self._launch_args["env"], dict) # type: ignore [unreachable] ): + #check whether env have custom kernel spc variables + newEnv = {} + if self._launch_args["custom_kernel_specs"]: + for custom_kernel_spec, custom_kernel_spec_value in self._launch_args["custom_kernel_specs"].items(): + for env_key, env_item in env.items(): + kernel_spec_item = self.replace_spec_parameter(custom_kernel_spec, custom_kernel_spec_value, env_item) + newEnv[env_key]=kernel_spec_item + if len(newEnv) > 0: + env = newEnv self._launch_args["env"].update(env) # type: ignore [unreachable] - - def replace_spec_parameter(self, variable, value, spec): + + + def replace_spec_parameter(self, variable, value, spec)->str: regexp = r"\{"+variable+"\}" pattern = re.compile(regexp) - return pattern.sub(pattern, value, spec) - - def update_kernel_specs(self, custom_kernel_specs: dict[str, t.Any] = None)-> None: - assert self.kernel_spec is not None -#go through custom kernel specs - for custom_kernel_spec, custom_kernel_spec_value in custom_kernel_specs.items(): - #go through kernel specs - for kernel_spec, kernel_spec_value in self.kernel_spec: - #if kernel spec is array then go through kernel this array - if isinstance(kernel_spec_value, list): - for kernel_spec_item in kernel_spec: - kernel_spec_item = self.replace_spec_parameter(custom_kernel_spec, custom_kernel_spec_value, kernel_spec_item) - else: - kernel_spec_value = self.replace_spec_parameter(custom_kernel_spec, custom_kernel_spec_value, kernel_spec_value) - return self.kernel_spec - + return pattern.sub(value, spec) + + def clear_custom_kernel_variable(self, **kwargs: t.Any): + print("dfd") + new_argv = [] + new_env = {} + + if "argv" in kwargs: + for argv_item in enumerate(kwargs["argv"]): + pattern = re.compile(r"\{([A-Za-z0-9_]+)\}") + matches = pattern.findall(argv_item) + if len(matches) == 0: + new_argv.append(argv_item) + if "env" in kwargs: + for env_key, env_item in kwargs["env"].items(): + pattern = re.compile(r"\{([A-Za-z0-9_]+)\}") + matches = pattern.findall(argv_item) + if len(matches) == 0: + new_env[env_key] = env_item + return new_argv, new_env def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> t.List[str]: """Replace templated args (e.g. {connection_file})""" extra_arguments = extra_arguments or [] assert self.kernel_spec is not None cmd = self.kernel_spec.argv + extra_arguments + #if not self._launch_args["custom_kernel_specs"]: + #cmd = self.clear_custom_kernel_variable(cmd) + print("----extra_arguments----") + print(extra_arguments) if cmd and cmd[0] in { "python", "python%i" % sys.version_info[0], @@ -349,6 +368,10 @@ def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> "prefix": sys.prefix, } + #Updating ns if there is custom kernel specs variables + if self._launch_args["custom_kernel_specs"]: + for custom_kernel_spec_key, custom_kernel_spec_value in self._launch_args["custom_kernel_specs"].items(): + ns[custom_kernel_spec_key] = custom_kernel_spec_value if self.kernel_spec: # type:ignore[truthy-bool] ns["resource_dir"] = self.kernel_spec.resource_dir assert isinstance(self._launch_args, dict) @@ -372,6 +395,7 @@ async def _async_launch_kernel(self, kernel_cmd: t.List[str], **kw: t.Any) -> No Note that provisioners can now be used to customize kernel environments and """ + # assert self.provisioner is not None connection_info = await self.provisioner.launch_kernel(kernel_cmd, **kw) assert self.provisioner.has_process @@ -413,9 +437,7 @@ async def _async_pre_start_kernel( # save kwargs for use in restart # assigning Traitlets Dicts to Dict make mypy unhappy but is ok self._launch_args = kw.copy() # type:ignore [assignment] - print("------_async_pre_start_kernel---start---") - print(self._launch_args) - print("------_async_pre_start_kernel---end---") + # if self.provisioner is None: # will not be None on restarts self.provisioner = KPF.instance(parent=self.parent).create_provisioner_instance( diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index a65b7aa1..c37a4b23 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -222,12 +222,14 @@ def update_env(self, *, kernel_id: str, env: t.Dict[str, str]) -> None: .. version-added: 8.5 """ + print("update_env ----- here- multi") if kernel_id in self: self._kernels[kernel_id].update_env(env=env) async def _add_kernel_when_ready( self, kernel_id: str, km: KernelManager, kernel_awaitable: t.Awaitable ) -> None: + # try: await kernel_awaitable self._kernels[kernel_id] = km diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index 9fc04e76..7389cfe0 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -167,8 +167,8 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: # This should be considered temporary until a better division of labor can be defined. km = self.parent if km: - if "custom_kernel_specs" in kwargs: - km.update_kernel_specs(kwargs["custom_kernel_specs"]) + #if "custom_kernel_specs" in kwargs: + # self.kernel_spec = km.update_kernel_specs(kwargs["custom_kernel_specs"]) if km.transport == "tcp" and not is_local_ip(km.ip): msg = ( "Can only launch a kernel on a local interface. " @@ -192,6 +192,9 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: km.control_port = lpc.find_available_port(km.ip) self.ports_cached = True if "env" in kwargs: + #update env if there is custom kernel specs variables for env + km.update_env(env=kwargs["env"]) + jupyter_session = kwargs["env"].get("JPY_SESSION_NAME", "") km.write_connection_file(jupyter_session=jupyter_session) else: @@ -210,7 +213,6 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> KernelConnectionInfo: """Launch a kernel with a command.""" scrubbed_kwargs = LocalProvisioner._scrub_kwargs(kwargs) - # self.process = launch_kernel(cmd, **scrubbed_kwargs) pgid = None if hasattr(os, "getpgid"): From ce0864cf4c90d239ff6bb4d4777a2102143eea80 Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Fri, 10 May 2024 20:48:16 +0200 Subject: [PATCH 03/31] Clear kernel spec variables from kernel.json --- jupyter_client/launcher.py | 3 + jupyter_client/manager.py | 67 +++++++++++-------- .../provisioning/local_provisioner.py | 4 +- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/jupyter_client/launcher.py b/jupyter_client/launcher.py index f0d07ad1..77baea1a 100644 --- a/jupyter_client/launcher.py +++ b/jupyter_client/launcher.py @@ -59,6 +59,9 @@ def launch_kernel( # If this process has been backgrounded, our stdin is invalid. Since there # is no compelling reason for the kernel to inherit our stdin anyway, we'll # place this one safe and always redirect. + + if "custom_kernel_specs" in kw: + del kw["custom_kernel_specs"] redirect_in = True _stdin = PIPE if stdin is None else stdin diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index bc69db87..6b982d08 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -292,59 +292,72 @@ def update_env(self, *, env: t.Dict[str, str]) -> None: .. version-added: 8.5 """ - print("update_env ----- here- no multi") + # Mypy think this is unreachable as it see _launch_args as Dict, not t.Dict if ( isinstance(self._launch_args, dict) and "env" in self._launch_args and isinstance(self._launch_args["env"], dict) # type: ignore [unreachable] ): - #check whether env have custom kernel spc variables + # check whether env has custom kernel spec variables newEnv = {} if self._launch_args["custom_kernel_specs"]: for custom_kernel_spec, custom_kernel_spec_value in self._launch_args["custom_kernel_specs"].items(): for env_key, env_item in env.items(): kernel_spec_item = self.replace_spec_parameter(custom_kernel_spec, custom_kernel_spec_value, env_item) newEnv[env_key]=kernel_spec_item + else: + # check whether there are custom kernel spec variables into kernel.json, + # if yes but a user has not configured them, + # we should clean them + newEnv = self.clear_custom_kernel_parameters(env) + if len(newEnv) > 0: env = newEnv self._launch_args["env"].update(env) # type: ignore [unreachable] def replace_spec_parameter(self, variable, value, spec)->str: - regexp = r"\{"+variable+"\}" + regexp = r"\\{" + variable + "\\}" pattern = re.compile(regexp) return pattern.sub(value, spec) - def clear_custom_kernel_variable(self, **kwargs: t.Any): - print("dfd") - new_argv = [] - new_env = {} - - if "argv" in kwargs: - for argv_item in enumerate(kwargs["argv"]): - pattern = re.compile(r"\{([A-Za-z0-9_]+)\}") - matches = pattern.findall(argv_item) - if len(matches) == 0: - new_argv.append(argv_item) - if "env" in kwargs: - for env_key, env_item in kwargs["env"].items(): - pattern = re.compile(r"\{([A-Za-z0-9_]+)\}") - matches = pattern.findall(argv_item) - if len(matches) == 0: - new_env[env_key] = env_item - return new_argv, new_env + def check_existence_custom_kernel_spec(self, item:str): + pattern = re.compile(r"\{([A-Za-z0-9_]+)\}") + matches = pattern.findall(item) + isMatch = False + if len(matches) > 0: + isMatch = True + return isMatch + + # Clear kernel specs files if user has not configured them themselves + # we shoud return only that has not kernel custom variables + # if there are no metadata specification for custom kernel + + def clear_custom_kernel_parameters(self, kernel_parameters: t.Any)->t.Any: + clean_parameters = None + if isinstance(kernel_parameters, list): + clean_parameters = [] + for argv_item in kernel_parameters: + isMatch = self.check_existence_custom_kernel_spec(argv_item) + if not isMatch: + clean_parameters.append(argv_item) + + elif isinstance(kernel_parameters, dict): + clean_parameters = {} + for env_key, env_item in kernel_parameters.items(): + isMatch = self.check_existence_custom_kernel_spec(env_item) + if not isMatch: + clean_parameters[env_key] = env_item + if len(clean_parameters) == 0: + clean_parameters = kernel_parameters + return clean_parameters def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> t.List[str]: """Replace templated args (e.g. {connection_file})""" extra_arguments = extra_arguments or [] assert self.kernel_spec is not None cmd = self.kernel_spec.argv + extra_arguments - #if not self._launch_args["custom_kernel_specs"]: - #cmd = self.clear_custom_kernel_variable(cmd) - - print("----extra_arguments----") - print(extra_arguments) if cmd and cmd[0] in { "python", "python%i" % sys.version_info[0], @@ -376,8 +389,6 @@ def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> ns["resource_dir"] = self.kernel_spec.resource_dir assert isinstance(self._launch_args, dict) - print("----format_kernel_cmd---_launch_args---") - print(self._launch_args) ns.update(self._launch_args) pat = re.compile(r"\{([A-Za-z0-9_]+)\}") diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index 7389cfe0..376a2448 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -208,6 +208,8 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: extra_arguments = kwargs.pop("extra_arguments", []) kernel_cmd = self.kernel_spec.argv + extra_arguments + kernel_cmd = km.clear_custom_kernel_parameters(kernel_cmd) + print("cmd--------",kernel_cmd) return await super().pre_launch(cmd=kernel_cmd, **kwargs) async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> KernelConnectionInfo: @@ -228,7 +230,7 @@ async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> KernelConnection @staticmethod def _scrub_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: """Remove any keyword arguments that Popen does not tolerate.""" - keywords_to_scrub: List[str] = ["extra_arguments", "kernel_id"] + keywords_to_scrub: List[str] = ["extra_arguments", "kernel_id", "custom_kernel_specs"] scrubbed_kwargs = kwargs.copy() for kw in keywords_to_scrub: scrubbed_kwargs.pop(kw, None) From d7962942d93b6df0183e23d9484a928352d06742 Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Tue, 14 May 2024 09:46:12 +0200 Subject: [PATCH 04/31] Fix cleanup --- jupyter_client/manager.py | 13 +++++++++---- jupyter_client/multikernelmanager.py | 3 +++ jupyter_client/provisioning/provisioner_base.py | 1 + 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 6b982d08..5dac15b6 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -303,9 +303,10 @@ def update_env(self, *, env: t.Dict[str, str]) -> None: newEnv = {} if self._launch_args["custom_kernel_specs"]: for custom_kernel_spec, custom_kernel_spec_value in self._launch_args["custom_kernel_specs"].items(): - for env_key, env_item in env.items(): - kernel_spec_item = self.replace_spec_parameter(custom_kernel_spec, custom_kernel_spec_value, env_item) - newEnv[env_key]=kernel_spec_item + if custom_kernel_spec_value != "": + for env_key, env_item in env.items(): + kernel_spec_item = self.replace_spec_parameter(custom_kernel_spec, custom_kernel_spec_value, env_item) + newEnv[env_key]=kernel_spec_item else: # check whether there are custom kernel spec variables into kernel.json, # if yes but a user has not configured them, @@ -314,6 +315,8 @@ def update_env(self, *, env: t.Dict[str, str]) -> None: if len(newEnv) > 0: env = newEnv + else: + env = self.clear_custom_kernel_parameters(env) self._launch_args["env"].update(env) # type: ignore [unreachable] @@ -384,7 +387,8 @@ def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> #Updating ns if there is custom kernel specs variables if self._launch_args["custom_kernel_specs"]: for custom_kernel_spec_key, custom_kernel_spec_value in self._launch_args["custom_kernel_specs"].items(): - ns[custom_kernel_spec_key] = custom_kernel_spec_value + if custom_kernel_spec_value != "": + ns[custom_kernel_spec_key] = custom_kernel_spec_value if self.kernel_spec: # type:ignore[truthy-bool] ns["resource_dir"] = self.kernel_spec.resource_dir assert isinstance(self._launch_args, dict) @@ -645,6 +649,7 @@ async def _async_restart_kernel( msg = "Cannot restart the kernel. No previous call to 'start_kernel'." raise RuntimeError(msg) + print('££££? REstart ----') # Stop currently running kernel. await self._async_shutdown_kernel(now=now, restart=True) diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index c37a4b23..bc9a3f33 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -271,6 +271,9 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: ) kwargs["kernel_id"] = kernel_id # Make kernel_id available to manager and provisioner + + print("---_async_start_kernel--") + print("kwargs",kwargs) starter = ensure_async(km.start_kernel(**kwargs)) task = asyncio.create_task(self._add_kernel_when_ready(kernel_id, km, starter)) self._pending_kernels[kernel_id] = task diff --git a/jupyter_client/provisioning/provisioner_base.py b/jupyter_client/provisioning/provisioner_base.py index eff89432..d74766c6 100644 --- a/jupyter_client/provisioning/provisioner_base.py +++ b/jupyter_client/provisioning/provisioner_base.py @@ -155,6 +155,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: :meth:`launch_kernel()`. """ env = kwargs.pop("env", os.environ).copy() + # here!!! env.update(self.__apply_env_substitutions(env)) self._finalize_env(env) kwargs["env"] = env From 4aecf4201eca32267d6a2070bf037dc0e546b88a Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Thu, 16 May 2024 17:22:28 +0200 Subject: [PATCH 05/31] Add small fixes for validation and cleanup --- jupyter_client/launcher.py | 6 +++++ jupyter_client/manager.py | 14 +++++++++- jupyter_client/multikernelmanager.py | 27 +++++++++++++++++++ .../provisioning/local_provisioner.py | 6 +++++ .../provisioning/provisioner_base.py | 1 + 5 files changed, 53 insertions(+), 1 deletion(-) diff --git a/jupyter_client/launcher.py b/jupyter_client/launcher.py index 77baea1a..645773f9 100644 --- a/jupyter_client/launcher.py +++ b/jupyter_client/launcher.py @@ -62,6 +62,9 @@ def launch_kernel( if "custom_kernel_specs" in kw: del kw["custom_kernel_specs"] + + print("----kw----") + print(kw) redirect_in = True _stdin = PIPE if stdin is None else stdin @@ -77,6 +80,9 @@ def launch_kernel( env = env if (env is not None) else os.environ.copy() + + print("launcher---------------") + kwargs = kw.copy() main_args = { "stdin": _stdin, diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 5dac15b6..a6badbb5 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -447,6 +447,7 @@ async def _async_pre_start_kernel( keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ + print("---_async_pre_start_kernel---") self.shutting_down = False self.kernel_id = self.kernel_id or kw.pop("kernel_id", str(uuid.uuid4())) # save kwargs for use in restart @@ -460,8 +461,12 @@ async def _async_pre_start_kernel( self.kernel_spec, parent=self, ) + print("---self.provisioner---") + print(self.provisioner) kw = await self.provisioner.pre_launch(**kw) kernel_cmd = kw.pop("cmd") + if "custom_kernel_specs" in kw: + del kw["custom_kernel_specs"] return kernel_cmd, kw pre_start_kernel = run_sync(_async_pre_start_kernel) @@ -495,6 +500,7 @@ async def _async_start_kernel(self, **kw: t.Any) -> None: and launching the kernel (e.g. Popen kwargs). """ #here? + print('Maybe--------------------') self._attempted_start = True kernel_cmd, kw = await self._async_pre_start_kernel(**kw) @@ -603,17 +609,20 @@ async def _async_shutdown_kernel(self, now: bool = False, restart: bool = False) self.stop_restarter() if self.has_kernel: + print("---self.has_kernel--") await self._async_interrupt_kernel() if now: + print("---_async_kill_kernel---") await self._async_kill_kernel() else: + print("---_async_request_shutdown---") await self._async_request_shutdown(restart=restart) # Don't send any additional kernel kill messages immediately, to give # the kernel a chance to properly execute shutdown actions. Wait for at # most 1s, checking every 0.1s. await self._async_finish_shutdown(restart=restart) - + print("---_async_cleanup_resources---") await self._async_cleanup_resources(restart=restart) shutdown_kernel = run_sync(_async_shutdown_kernel) @@ -650,10 +659,13 @@ async def _async_restart_kernel( raise RuntimeError(msg) print('££££? REstart ----') + print("----self._launch_args----") + print(self._launch_args) # Stop currently running kernel. await self._async_shutdown_kernel(now=now, restart=True) if newports: + print("---cleanup_random_ports---") self.cleanup_random_ports() # Start new kernel. diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index bc9a3f33..ada260f8 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -5,6 +5,7 @@ import asyncio import json +import re import os import socket import typing as t @@ -252,6 +253,26 @@ def _using_pending_kernels(self) -> bool: this multikernelmanager is using pending kernels or not """ return getattr(self, "use_pending_kernels", False) + + def validate(self, string)->str: + sanitazed_string = re.sub(r'[;&|]', '', string) + print('----sanitazed_string-------') + print(sanitazed_string) + match = re.match(r"'", sanitazed_string) + if match: + sanitazed_string = "'" + re.sub(r"'", "'\''", sanitazed_string) +"'" + else: + print('----does not match-------') + return sanitazed_string + + def validate_kernel_parameters(self, kwargs: t.Any)-> None: + if "custom_kernel_specs" in kwargs: + for custom_kernel_spec, custom_kernel_spec_value in kwargs["custom_kernel_specs"].items(): + sanitazed_string = self.validate(custom_kernel_spec_value) + if (sanitazed_string !=''): + print('----sanitazed_string is not null------') + kwargs["custom_kernel_specs"][custom_kernel_spec] = sanitazed_string + return kwargs async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: t.Any) -> str: """Start a new kernel. @@ -262,6 +283,12 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: The kernel ID for the newly started kernel is returned. """ #here + print("---kwargs--before") + print(kwargs) + print('----validate----') + kwargs = self.validate_kernel_parameters(kwargs) + print(kwargs) + print("---kwargs--after") km, kernel_name, kernel_id = self.pre_start_kernel(kernel_name, kwargs) if not isinstance(km, KernelManager): self.log.warning( # type:ignore[unreachable] diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index 376a2448..e4637e8a 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -209,7 +209,13 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: kernel_cmd = self.kernel_spec.argv + extra_arguments kernel_cmd = km.clear_custom_kernel_parameters(kernel_cmd) + print("cmd--------",kernel_cmd) + + if "custom_kernel_specs" in kwargs: + del kwargs["custom_kernel_specs"] + print("--After deleting--kwargs----") + print(kwargs) return await super().pre_launch(cmd=kernel_cmd, **kwargs) async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> KernelConnectionInfo: diff --git a/jupyter_client/provisioning/provisioner_base.py b/jupyter_client/provisioning/provisioner_base.py index d74766c6..d1e60909 100644 --- a/jupyter_client/provisioning/provisioner_base.py +++ b/jupyter_client/provisioning/provisioner_base.py @@ -154,6 +154,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: Returns the (potentially updated) keyword arguments that are passed to :meth:`launch_kernel()`. """ + print("---provisiioning base---") env = kwargs.pop("env", os.environ).copy() # here!!! env.update(self.__apply_env_substitutions(env)) From 49e3b9a20aaa181d5dbf54067d222c02f4c7271f Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Tue, 21 May 2024 23:14:04 +0200 Subject: [PATCH 06/31] Getting default values and fix updating env --- jupyter_client/manager.py | 92 ++++++++++++++++--- jupyter_client/multikernelmanager.py | 28 +++--- .../provisioning/local_provisioner.py | 10 +- .../provisioning/provisioner_base.py | 5 + 4 files changed, 101 insertions(+), 34 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index a6badbb5..fdc56d4a 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -228,6 +228,8 @@ def ipykernel(self) -> bool: shutting_down: bool = False + custom_kernel_default_value: dict = {} + def __del__(self) -> None: self._close_control_socket() self.cleanup_connection_file() @@ -301,27 +303,42 @@ def update_env(self, *, env: t.Dict[str, str]) -> None: ): # check whether env has custom kernel spec variables newEnv = {} + custom_kernel_dict = {} + if self._launch_args["custom_kernel_specs"]: - for custom_kernel_spec, custom_kernel_spec_value in self._launch_args["custom_kernel_specs"].items(): - if custom_kernel_spec_value != "": - for env_key, env_item in env.items(): - kernel_spec_item = self.replace_spec_parameter(custom_kernel_spec, custom_kernel_spec_value, env_item) - newEnv[env_key]=kernel_spec_item + custom_kernel_dict = self._launch_args["custom_kernel_specs"] + # check is custom kernel variables are full if not then we should take default ones + if self.custom_kernel_default_value: + for key, value in self.custom_kernel_default_value.items(): + if key not in custom_kernel_dict: + custom_kernel_dict[key] = value + elif self.custom_kernel_default_value: + # if not but default values are present into a kernel.json file then we have to take them + custom_kernel_dict = self.custom_kernel_default_value + print('custom_kernel_dict') + print(custom_kernel_dict) + + if len(custom_kernel_dict) > 0: + for custom_kernel_spec, custom_kernel_spec_value in custom_kernel_dict.items(): + for env_key, env_item in env.items(): + kernel_spec_item = self.replace_spec_parameter(custom_kernel_spec, custom_kernel_spec_value, env_item) + newEnv[env_key]=kernel_spec_item else: # check whether there are custom kernel spec variables into kernel.json, - # if yes but a user has not configured them, + # if yes but a user has not configured them and default ones are not present , # we should clean them newEnv = self.clear_custom_kernel_parameters(env) - + if len(newEnv) > 0: - env = newEnv + env = self.clear_custom_kernel_parameters(newEnv) else: env = self.clear_custom_kernel_parameters(env) + print(env) self._launch_args["env"].update(env) # type: ignore [unreachable] def replace_spec_parameter(self, variable, value, spec)->str: - regexp = r"\\{" + variable + "\\}" + regexp = r"\{" + variable + "\\}" pattern = re.compile(regexp) return pattern.sub(value, spec) @@ -340,21 +357,43 @@ def check_existence_custom_kernel_spec(self, item:str): def clear_custom_kernel_parameters(self, kernel_parameters: t.Any)->t.Any: clean_parameters = None if isinstance(kernel_parameters, list): + clean_parameters = [] for argv_item in kernel_parameters: isMatch = self.check_existence_custom_kernel_spec(argv_item) if not isMatch: clean_parameters.append(argv_item) - elif isinstance(kernel_parameters, dict): + clean_parameters = {} for env_key, env_item in kernel_parameters.items(): + isMatch = self.check_existence_custom_kernel_spec(env_item) if not isMatch: clean_parameters[env_key] = env_item if len(clean_parameters) == 0: clean_parameters = kernel_parameters return clean_parameters + + def get_default_custom_kernel_specs_value(self): + assert self.kernel_spec is not None + custom_kernel_default_value = {} + if ( + self.kernel_spec.metadata + and isinstance(self.kernel_spec.metadata, dict) + and "parameters" in self.kernel_spec.metadata + and isinstance(self.kernel_spec.metadata["parameters"], dict) + and "properties" in self.kernel_spec.metadata["parameters"] + and isinstance(self.kernel_spec.metadata["parameters"]["properties"], dict) + ): + propetries = self.kernel_spec.metadata["parameters"]["properties"].items() + for property_key, property_value in propetries: + if "default" in property_value: + custom_kernel_default_value[property_key] = property_value["default"] + print("custom_kernel_default_value") + print(custom_kernel_default_value) + self.custom_kernel_default_value = custom_kernel_default_value + def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> t.List[str]: """Replace templated args (e.g. {connection_file})""" @@ -384,11 +423,22 @@ def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> "prefix": sys.prefix, } - #Updating ns if there is custom kernel specs variables + # Updating ns if there are custom kernel specs variables + custom_kernel_dict = {} if self._launch_args["custom_kernel_specs"]: - for custom_kernel_spec_key, custom_kernel_spec_value in self._launch_args["custom_kernel_specs"].items(): - if custom_kernel_spec_value != "": - ns[custom_kernel_spec_key] = custom_kernel_spec_value + custom_kernel_dict = self._launch_args["custom_kernel_specs"] + if self.custom_kernel_default_value: + for key, value in self.custom_kernel_default_value.items(): + if key not in custom_kernel_dict: + custom_kernel_dict[key] = value + elif self.custom_kernel_default_value: + # if not but default values are present into a kernel.json file then we have to take them + custom_kernel_dict = self.custom_kernel_default_value + + if len(custom_kernel_dict) > 0: + for custom_kernel_spec_key, custom_kernel_spec_value in custom_kernel_dict.items(): + ns[custom_kernel_spec_key] = custom_kernel_spec_value + if self.kernel_spec: # type:ignore[truthy-bool] ns["resource_dir"] = self.kernel_spec.resource_dir assert isinstance(self._launch_args, dict) @@ -464,9 +514,19 @@ async def _async_pre_start_kernel( print("---self.provisioner---") print(self.provisioner) kw = await self.provisioner.pre_launch(**kw) + # update env + if "env" in kw: + print('update!!!') + self.update_env(env=kw["env"]) + kw["env"] = self._launch_args["env"] kernel_cmd = kw.pop("cmd") if "custom_kernel_specs" in kw: del kw["custom_kernel_specs"] + # if "custom_kernel_specs" in self._launch_args: + # del self._launch_args['custom_kernel_specs'] + + #print("_async_pre_start_kernel- km---after deletinh") + # print(kw) return kernel_cmd, kw pre_start_kernel = run_sync(_async_pre_start_kernel) @@ -659,7 +719,7 @@ async def _async_restart_kernel( raise RuntimeError(msg) print('££££? REstart ----') - print("----self._launch_args----") + print("----self._launch_args---- before") print(self._launch_args) # Stop currently running kernel. await self._async_shutdown_kernel(now=now, restart=True) @@ -670,6 +730,8 @@ async def _async_restart_kernel( # Start new kernel. self._launch_args.update(kw) + print("----self._launch_args---- after") + print(self._launch_args) await self._async_start_kernel(**self._launch_args) restart_kernel = run_sync(_async_restart_kernel) diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index ada260f8..341e380f 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -213,6 +213,9 @@ def pre_start_kernel( kernel_name=kernel_name, **constructor_kwargs, ) + print('pre_start_kernel') + print('---km---') + print(km) return km, kernel_name, kernel_id def update_env(self, *, kernel_id: str, env: t.Dict[str, str]) -> None: @@ -255,9 +258,9 @@ def _using_pending_kernels(self) -> bool: return getattr(self, "use_pending_kernels", False) def validate(self, string)->str: - sanitazed_string = re.sub(r'[;&|]', '', string) - print('----sanitazed_string-------') - print(sanitazed_string) + sanitazed_string = re.sub(r'[;&|$#]', '', string) + #print('----sanitazed_string-------') + #print(sanitazed_string) match = re.match(r"'", sanitazed_string) if match: sanitazed_string = "'" + re.sub(r"'", "'\''", sanitazed_string) +"'" @@ -267,11 +270,13 @@ def validate(self, string)->str: def validate_kernel_parameters(self, kwargs: t.Any)-> None: if "custom_kernel_specs" in kwargs: - for custom_kernel_spec, custom_kernel_spec_value in kwargs["custom_kernel_specs"].items(): - sanitazed_string = self.validate(custom_kernel_spec_value) - if (sanitazed_string !=''): - print('----sanitazed_string is not null------') - kwargs["custom_kernel_specs"][custom_kernel_spec] = sanitazed_string + custom_kernel_specs = kwargs.get("custom_kernel_specs") + if custom_kernel_specs is not None: + for custom_kernel_spec, custom_kernel_spec_value in kwargs["custom_kernel_specs"].items(): + sanitazed_string = self.validate(custom_kernel_spec_value) + if (sanitazed_string !=''): + print('----sanitazed_string is not null------') + kwargs["custom_kernel_specs"][custom_kernel_spec] = sanitazed_string return kwargs async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: t.Any) -> str: @@ -283,12 +288,7 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: The kernel ID for the newly started kernel is returned. """ #here - print("---kwargs--before") - print(kwargs) - print('----validate----') kwargs = self.validate_kernel_parameters(kwargs) - print(kwargs) - print("---kwargs--after") km, kernel_name, kernel_id = self.pre_start_kernel(kernel_name, kwargs) if not isinstance(km, KernelManager): self.log.warning( # type:ignore[unreachable] @@ -300,7 +300,7 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: print("---_async_start_kernel--") - print("kwargs",kwargs) + starter = ensure_async(km.start_kernel(**kwargs)) task = asyncio.create_task(self._add_kernel_when_ready(kernel_id, km, starter)) self._pending_kernels[kernel_id] = task diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index e4637e8a..cf720dfc 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -167,8 +167,9 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: # This should be considered temporary until a better division of labor can be defined. km = self.parent if km: - #if "custom_kernel_specs" in kwargs: - # self.kernel_spec = km.update_kernel_specs(kwargs["custom_kernel_specs"]) + # Get default values from kernel.json file if there is a custom kernel + km.get_default_custom_kernel_specs_value() + if km.transport == "tcp" and not is_local_ip(km.ip): msg = ( "Can only launch a kernel on a local interface. " @@ -209,13 +210,12 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: kernel_cmd = self.kernel_spec.argv + extra_arguments kernel_cmd = km.clear_custom_kernel_parameters(kernel_cmd) - print("cmd--------",kernel_cmd) if "custom_kernel_specs" in kwargs: del kwargs["custom_kernel_specs"] - print("--After deleting--kwargs----") - print(kwargs) + #print("--After deleting--kwargs----") + #print(kwargs) return await super().pre_launch(cmd=kernel_cmd, **kwargs) async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> KernelConnectionInfo: diff --git a/jupyter_client/provisioning/provisioner_base.py b/jupyter_client/provisioning/provisioner_base.py index d1e60909..c717d3c4 100644 --- a/jupyter_client/provisioning/provisioner_base.py +++ b/jupyter_client/provisioning/provisioner_base.py @@ -158,7 +158,11 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: env = kwargs.pop("env", os.environ).copy() # here!!! env.update(self.__apply_env_substitutions(env)) + self._finalize_env(env) + + print('new env') + print(env) kwargs["env"] = env return kwargs @@ -236,6 +240,7 @@ def _finalize_env(self, env: Dict[str, str]) -> None: env.pop("PYTHONEXECUTABLE", None) def __apply_env_substitutions(self, substitution_values: Dict[str, str]) -> Dict[str, str]: + print('????') """ Walks entries in the kernelspec's env stanza and applies substitutions from current env. From 6518491b74c7078b7553e620ec75ecb1000fe110 Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Fri, 31 May 2024 17:27:06 +0200 Subject: [PATCH 07/31] Fix defining custom env parameters --- jupyter_client/manager.py | 67 +++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index fdc56d4a..939dcdd0 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -302,40 +302,46 @@ def update_env(self, *, env: t.Dict[str, str]) -> None: and isinstance(self._launch_args["env"], dict) # type: ignore [unreachable] ): # check whether env has custom kernel spec variables - newEnv = {} - custom_kernel_dict = {} - - if self._launch_args["custom_kernel_specs"]: - custom_kernel_dict = self._launch_args["custom_kernel_specs"] - # check is custom kernel variables are full if not then we should take default ones - if self.custom_kernel_default_value: - for key, value in self.custom_kernel_default_value.items(): - if key not in custom_kernel_dict: - custom_kernel_dict[key] = value - elif self.custom_kernel_default_value: - # if not but default values are present into a kernel.json file then we have to take them - custom_kernel_dict = self.custom_kernel_default_value - print('custom_kernel_dict') - print(custom_kernel_dict) + env = self.update_custom_env_parameters(env=env) + print('yes') + print(env) + + self._launch_args["env"].update(env) # type: ignore [unreachable] + + def update_custom_env_parameters(self, env: t.Dict[str, str]) -> t.Dict[str, str]: + newEnv = {} + custom_kernel_dict = {} + if self._launch_args["custom_kernel_specs"]: + custom_kernel_dict = self._launch_args["custom_kernel_specs"] + # check is custom kernel variables are full if not then we should take default ones + if self.custom_kernel_default_value: + for key, value in self.custom_kernel_default_value.items(): + if key not in custom_kernel_dict: + custom_kernel_dict[key] = value + elif self.custom_kernel_default_value: + # if not but default values are present into a kernel.json file then we have to take them + custom_kernel_dict = self.custom_kernel_default_value + print('custom_kernel_dict') + print(custom_kernel_dict) - if len(custom_kernel_dict) > 0: - for custom_kernel_spec, custom_kernel_spec_value in custom_kernel_dict.items(): - for env_key, env_item in env.items(): - kernel_spec_item = self.replace_spec_parameter(custom_kernel_spec, custom_kernel_spec_value, env_item) - newEnv[env_key]=kernel_spec_item - else: + if len(custom_kernel_dict) > 0: + for custom_kernel_spec, custom_kernel_spec_value in custom_kernel_dict.items(): + for env_key, env_item in env.items(): + kernel_spec_item = self.replace_spec_parameter(custom_kernel_spec, custom_kernel_spec_value, env_item) + newEnv[env_key]=kernel_spec_item + else: # check whether there are custom kernel spec variables into kernel.json, # if yes but a user has not configured them and default ones are not present , # we should clean them - newEnv = self.clear_custom_kernel_parameters(env) + newEnv = self.clear_custom_kernel_parameters(env) - if len(newEnv) > 0: - env = self.clear_custom_kernel_parameters(newEnv) - else: - env = self.clear_custom_kernel_parameters(env) - print(env) - self._launch_args["env"].update(env) # type: ignore [unreachable] - + if len(newEnv) > 0: + env = self.clear_custom_kernel_parameters(newEnv) + else: + env = self.clear_custom_kernel_parameters(env) + print('update_custom_env_parameters env') + print(env) + return env def replace_spec_parameter(self, variable, value, spec)->str: regexp = r"\{" + variable + "\\}" @@ -517,8 +523,7 @@ async def _async_pre_start_kernel( # update env if "env" in kw: print('update!!!') - self.update_env(env=kw["env"]) - kw["env"] = self._launch_args["env"] + kw["env"] = self.update_custom_env_parameters(env=kw["env"]) kernel_cmd = kw.pop("cmd") if "custom_kernel_specs" in kw: del kw["custom_kernel_specs"] From 7c21d476a1709e4c9d21186c7d5c2c03c7ec798d Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Mon, 10 Jun 2024 10:57:13 +0200 Subject: [PATCH 08/31] add logging --- jupyter_client/manager.py | 11 ++++++++++- jupyter_client/multikernelmanager.py | 2 ++ jupyter_client/provisioning/local_provisioner.py | 4 +++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 939dcdd0..9ef287bd 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -301,6 +301,8 @@ def update_env(self, *, env: t.Dict[str, str]) -> None: and "env" in self._launch_args and isinstance(self._launch_args["env"], dict) # type: ignore [unreachable] ): + print('update env -----------------------------------j') + print(self._launch_args["env"]) # check whether env has custom kernel spec variables env = self.update_custom_env_parameters(env=env) print('yes') @@ -323,6 +325,9 @@ def update_custom_env_parameters(self, env: t.Dict[str, str]) -> t.Dict[str, str custom_kernel_dict = self.custom_kernel_default_value print('custom_kernel_dict') print(custom_kernel_dict) + + print('update_custom_env_parameters env before-----') + print(env) if len(custom_kernel_dict) > 0: for custom_kernel_spec, custom_kernel_spec_value in custom_kernel_dict.items(): @@ -339,7 +344,7 @@ def update_custom_env_parameters(self, env: t.Dict[str, str]) -> t.Dict[str, str env = self.clear_custom_kernel_parameters(newEnv) else: env = self.clear_custom_kernel_parameters(env) - print('update_custom_env_parameters env') + print('update_custom_env_parameters env after ---') print(env) return env @@ -469,6 +474,8 @@ async def _async_launch_kernel(self, kernel_cmd: t.List[str], **kw: t.Any) -> No # assert self.provisioner is not None connection_info = await self.provisioner.launch_kernel(kernel_cmd, **kw) + print('connection_info') + print(connection_info) assert self.provisioner.has_process # Provisioner provides the connection information. Load into kernel manager # and write the connection file, if not already done. @@ -524,6 +531,8 @@ async def _async_pre_start_kernel( if "env" in kw: print('update!!!') kw["env"] = self.update_custom_env_parameters(env=kw["env"]) + print('skw["env"]') + print(kw["env"]) kernel_cmd = kw.pop("cmd") if "custom_kernel_specs" in kw: del kw["custom_kernel_specs"] diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index 341e380f..4d7269d8 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -288,6 +288,8 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: The kernel ID for the newly started kernel is returned. """ #here + print('----validate_kernel_parameters----') + print(kwargs) kwargs = self.validate_kernel_parameters(kwargs) km, kernel_name, kernel_id = self.pre_start_kernel(kernel_name, kwargs) if not isinstance(km, KernelManager): diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index cf720dfc..1d9a66f3 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -193,8 +193,9 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: km.control_port = lpc.find_available_port(km.ip) self.ports_cached = True if "env" in kwargs: + print('pre_launch update env ----') #update env if there is custom kernel specs variables for env - km.update_env(env=kwargs["env"]) + # km.update_env(env=kwargs["env"]) jupyter_session = kwargs["env"].get("JPY_SESSION_NAME", "") km.write_connection_file(jupyter_session=jupyter_session) @@ -221,6 +222,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> KernelConnectionInfo: """Launch a kernel with a command.""" scrubbed_kwargs = LocalProvisioner._scrub_kwargs(kwargs) + print('local provisioner launch_kernel') self.process = launch_kernel(cmd, **scrubbed_kwargs) pgid = None if hasattr(os, "getpgid"): From 2e62e53489b05c198810de457d2ab7f14390b6a1 Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Wed, 12 Jun 2024 14:25:38 +0200 Subject: [PATCH 09/31] Fix custom parameters of a kernel --- jupyter_client/manager.py | 17 +++++++---------- jupyter_client/multikernelmanager.py | 6 ++++-- jupyter_client/provisioning/provisioner_base.py | 4 +--- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 9ef287bd..4b1c673a 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -313,7 +313,7 @@ def update_env(self, *, env: t.Dict[str, str]) -> None: def update_custom_env_parameters(self, env: t.Dict[str, str]) -> t.Dict[str, str]: newEnv = {} custom_kernel_dict = {} - if self._launch_args["custom_kernel_specs"]: + if "custom_kernel_specs" in self._launch_args: custom_kernel_dict = self._launch_args["custom_kernel_specs"] # check is custom kernel variables are full if not then we should take default ones if self.custom_kernel_default_value: @@ -325,9 +325,6 @@ def update_custom_env_parameters(self, env: t.Dict[str, str]) -> t.Dict[str, str custom_kernel_dict = self.custom_kernel_default_value print('custom_kernel_dict') print(custom_kernel_dict) - - print('update_custom_env_parameters env before-----') - print(env) if len(custom_kernel_dict) > 0: for custom_kernel_spec, custom_kernel_spec_value in custom_kernel_dict.items(): @@ -344,8 +341,7 @@ def update_custom_env_parameters(self, env: t.Dict[str, str]) -> t.Dict[str, str env = self.clear_custom_kernel_parameters(newEnv) else: env = self.clear_custom_kernel_parameters(env) - print('update_custom_env_parameters env after ---') - print(env) + return env def replace_spec_parameter(self, variable, value, spec)->str: @@ -436,7 +432,7 @@ def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> # Updating ns if there are custom kernel specs variables custom_kernel_dict = {} - if self._launch_args["custom_kernel_specs"]: + if "custom_kernel_specs" in self._launch_args: custom_kernel_dict = self._launch_args["custom_kernel_specs"] if self.custom_kernel_default_value: for key, value in self.custom_kernel_default_value.items(): @@ -453,7 +449,8 @@ def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> if self.kernel_spec: # type:ignore[truthy-bool] ns["resource_dir"] = self.kernel_spec.resource_dir assert isinstance(self._launch_args, dict) - + print('----ns[custom_kernel_spec_key]----') + print(ns) ns.update(self._launch_args) pat = re.compile(r"\{([A-Za-z0-9_]+)\}") @@ -531,8 +528,8 @@ async def _async_pre_start_kernel( if "env" in kw: print('update!!!') kw["env"] = self.update_custom_env_parameters(env=kw["env"]) - print('skw["env"]') - print(kw["env"]) + #print('skw["env"]') + #print(kw["env"]) kernel_cmd = kw.pop("cmd") if "custom_kernel_specs" in kw: del kw["custom_kernel_specs"] diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index 4d7269d8..da77b3fb 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -194,6 +194,8 @@ def pre_start_kernel( self, kernel_name: str | None, kwargs: t.Any ) -> tuple[KernelManager, str, str]: # kwargs should be mutable, passing it as a dict argument. + print('----kwargs---') + print(kwargs) kernel_id = kwargs.pop("kernel_id", self.new_kernel_id(**kwargs)) if kernel_id in self: raise DuplicateKernelError("Kernel already exists: %s" % kernel_id) @@ -288,8 +290,8 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: The kernel ID for the newly started kernel is returned. """ #here - print('----validate_kernel_parameters----') - print(kwargs) + #print('----validate_kernel_parameters----') + #print(kwargs) kwargs = self.validate_kernel_parameters(kwargs) km, kernel_name, kernel_id = self.pre_start_kernel(kernel_name, kwargs) if not isinstance(km, KernelManager): diff --git a/jupyter_client/provisioning/provisioner_base.py b/jupyter_client/provisioning/provisioner_base.py index c717d3c4..ab3fff57 100644 --- a/jupyter_client/provisioning/provisioner_base.py +++ b/jupyter_client/provisioning/provisioner_base.py @@ -160,9 +160,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: env.update(self.__apply_env_substitutions(env)) self._finalize_env(env) - - print('new env') - print(env) + kwargs["env"] = env return kwargs From f6440a903ce21b0f6eeb1a379c6ed57ee99a4a5a Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Thu, 13 Jun 2024 16:45:53 +0200 Subject: [PATCH 10/31] Fix restarting a kernel with custom env variable and clean printing variables --- jupyter_client/manager.py | 36 ++++++++----------- jupyter_client/multikernelmanager.py | 14 +------- .../provisioning/local_provisioner.py | 6 ++-- .../provisioning/provisioner_base.py | 1 - 4 files changed, 17 insertions(+), 40 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 4b1c673a..07446b23 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -301,12 +301,17 @@ def update_env(self, *, env: t.Dict[str, str]) -> None: and "env" in self._launch_args and isinstance(self._launch_args["env"], dict) # type: ignore [unreachable] ): - print('update env -----------------------------------j') - print(self._launch_args["env"]) + # if self._launch_args["env"] has custom kernel variable for env but env does not have then we have to fill env with it + if "custom_kernel_specs" in self._launch_args: + saved_env = self._launch_args.get("env", {}) + custom_kernel_dict = self._launch_args["custom_kernel_specs"] + if isinstance(custom_kernel_dict, dict): + for key, value in custom_kernel_dict.items(): + if key in saved_env and key not in env: + env[key] = saved_env[key] + # check whether env has custom kernel spec variables env = self.update_custom_env_parameters(env=env) - print('yes') - print(env) self._launch_args["env"].update(env) # type: ignore [unreachable] @@ -318,15 +323,13 @@ def update_custom_env_parameters(self, env: t.Dict[str, str]) -> t.Dict[str, str # check is custom kernel variables are full if not then we should take default ones if self.custom_kernel_default_value: for key, value in self.custom_kernel_default_value.items(): - if key not in custom_kernel_dict: + if isinstance(custom_kernel_dict, dict) and key not in custom_kernel_dict: custom_kernel_dict[key] = value elif self.custom_kernel_default_value: # if not but default values are present into a kernel.json file then we have to take them custom_kernel_dict = self.custom_kernel_default_value - print('custom_kernel_dict') - print(custom_kernel_dict) - if len(custom_kernel_dict) > 0: + if isinstance(custom_kernel_dict, dict) and len(custom_kernel_dict) > 0: for custom_kernel_spec, custom_kernel_spec_value in custom_kernel_dict.items(): for env_key, env_item in env.items(): kernel_spec_item = self.replace_spec_parameter(custom_kernel_spec, custom_kernel_spec_value, env_item) @@ -436,21 +439,19 @@ def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> custom_kernel_dict = self._launch_args["custom_kernel_specs"] if self.custom_kernel_default_value: for key, value in self.custom_kernel_default_value.items(): - if key not in custom_kernel_dict: + if isinstance(custom_kernel_dict, dict) and key not in custom_kernel_dict: custom_kernel_dict[key] = value elif self.custom_kernel_default_value: # if not but default values are present into a kernel.json file then we have to take them custom_kernel_dict = self.custom_kernel_default_value - if len(custom_kernel_dict) > 0: + if isinstance(custom_kernel_dict, dict) and len(custom_kernel_dict) > 0: for custom_kernel_spec_key, custom_kernel_spec_value in custom_kernel_dict.items(): ns[custom_kernel_spec_key] = custom_kernel_spec_value if self.kernel_spec: # type:ignore[truthy-bool] ns["resource_dir"] = self.kernel_spec.resource_dir assert isinstance(self._launch_args, dict) - print('----ns[custom_kernel_spec_key]----') - print(ns) ns.update(self._launch_args) pat = re.compile(r"\{([A-Za-z0-9_]+)\}") @@ -471,8 +472,6 @@ async def _async_launch_kernel(self, kernel_cmd: t.List[str], **kw: t.Any) -> No # assert self.provisioner is not None connection_info = await self.provisioner.launch_kernel(kernel_cmd, **kw) - print('connection_info') - print(connection_info) assert self.provisioner.has_process # Provisioner provides the connection information. Load into kernel manager # and write the connection file, if not already done. @@ -526,10 +525,8 @@ async def _async_pre_start_kernel( kw = await self.provisioner.pre_launch(**kw) # update env if "env" in kw: - print('update!!!') kw["env"] = self.update_custom_env_parameters(env=kw["env"]) - #print('skw["env"]') - #print(kw["env"]) + self._launch_args["env"].update(kw["env"]) kernel_cmd = kw.pop("cmd") if "custom_kernel_specs" in kw: del kw["custom_kernel_specs"] @@ -570,8 +567,6 @@ async def _async_start_kernel(self, **kw: t.Any) -> None: keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ - #here? - print('Maybe--------------------') self._attempted_start = True kernel_cmd, kw = await self._async_pre_start_kernel(**kw) @@ -729,9 +724,6 @@ async def _async_restart_kernel( msg = "Cannot restart the kernel. No previous call to 'start_kernel'." raise RuntimeError(msg) - print('££££? REstart ----') - print("----self._launch_args---- before") - print(self._launch_args) # Stop currently running kernel. await self._async_shutdown_kernel(now=now, restart=True) diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index da77b3fb..9f456f8d 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -194,8 +194,6 @@ def pre_start_kernel( self, kernel_name: str | None, kwargs: t.Any ) -> tuple[KernelManager, str, str]: # kwargs should be mutable, passing it as a dict argument. - print('----kwargs---') - print(kwargs) kernel_id = kwargs.pop("kernel_id", self.new_kernel_id(**kwargs)) if kernel_id in self: raise DuplicateKernelError("Kernel already exists: %s" % kernel_id) @@ -215,9 +213,6 @@ def pre_start_kernel( kernel_name=kernel_name, **constructor_kwargs, ) - print('pre_start_kernel') - print('---km---') - print(km) return km, kernel_name, kernel_id def update_env(self, *, kernel_id: str, env: t.Dict[str, str]) -> None: @@ -261,13 +256,9 @@ def _using_pending_kernels(self) -> bool: def validate(self, string)->str: sanitazed_string = re.sub(r'[;&|$#]', '', string) - #print('----sanitazed_string-------') - #print(sanitazed_string) match = re.match(r"'", sanitazed_string) if match: sanitazed_string = "'" + re.sub(r"'", "'\''", sanitazed_string) +"'" - else: - print('----does not match-------') return sanitazed_string def validate_kernel_parameters(self, kwargs: t.Any)-> None: @@ -277,7 +268,6 @@ def validate_kernel_parameters(self, kwargs: t.Any)-> None: for custom_kernel_spec, custom_kernel_spec_value in kwargs["custom_kernel_specs"].items(): sanitazed_string = self.validate(custom_kernel_spec_value) if (sanitazed_string !=''): - print('----sanitazed_string is not null------') kwargs["custom_kernel_specs"][custom_kernel_spec] = sanitazed_string return kwargs @@ -289,9 +279,7 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: The kernel ID for the newly started kernel is returned. """ - #here - #print('----validate_kernel_parameters----') - #print(kwargs) + kwargs = self.validate_kernel_parameters(kwargs) km, kernel_name, kernel_id = self.pre_start_kernel(kernel_name, kwargs) if not isinstance(km, KernelManager): diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index 1d9a66f3..fc325dfc 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -193,9 +193,8 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: km.control_port = lpc.find_available_port(km.ip) self.ports_cached = True if "env" in kwargs: - print('pre_launch update env ----') - #update env if there is custom kernel specs variables for env - # km.update_env(env=kwargs["env"]) + # update env if there is custom kernel specs variables for env + km.update_env(env=kwargs["env"]) jupyter_session = kwargs["env"].get("JPY_SESSION_NAME", "") km.write_connection_file(jupyter_session=jupyter_session) @@ -222,7 +221,6 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> KernelConnectionInfo: """Launch a kernel with a command.""" scrubbed_kwargs = LocalProvisioner._scrub_kwargs(kwargs) - print('local provisioner launch_kernel') self.process = launch_kernel(cmd, **scrubbed_kwargs) pgid = None if hasattr(os, "getpgid"): diff --git a/jupyter_client/provisioning/provisioner_base.py b/jupyter_client/provisioning/provisioner_base.py index ab3fff57..a3804b8e 100644 --- a/jupyter_client/provisioning/provisioner_base.py +++ b/jupyter_client/provisioning/provisioner_base.py @@ -238,7 +238,6 @@ def _finalize_env(self, env: Dict[str, str]) -> None: env.pop("PYTHONEXECUTABLE", None) def __apply_env_substitutions(self, substitution_values: Dict[str, str]) -> Dict[str, str]: - print('????') """ Walks entries in the kernelspec's env stanza and applies substitutions from current env. From 8cdac5caff7a92e2b3fb9485ddafab0c6ceac6a7 Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Thu, 13 Jun 2024 17:26:48 +0200 Subject: [PATCH 11/31] Clean printing --- jupyter_client/launcher.py | 4 ---- jupyter_client/manager.py | 14 -------------- jupyter_client/multikernelmanager.py | 4 +--- jupyter_client/provisioning/local_provisioner.py | 2 -- jupyter_client/provisioning/provisioner_base.py | 1 - 5 files changed, 1 insertion(+), 24 deletions(-) diff --git a/jupyter_client/launcher.py b/jupyter_client/launcher.py index 645773f9..d4b56d0d 100644 --- a/jupyter_client/launcher.py +++ b/jupyter_client/launcher.py @@ -63,8 +63,6 @@ def launch_kernel( if "custom_kernel_specs" in kw: del kw["custom_kernel_specs"] - print("----kw----") - print(kw) redirect_in = True _stdin = PIPE if stdin is None else stdin @@ -81,8 +79,6 @@ def launch_kernel( env = env if (env is not None) else os.environ.copy() - print("launcher---------------") - kwargs = kw.copy() main_args = { "stdin": _stdin, diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 07446b23..fd8e74c8 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -400,8 +400,6 @@ def get_default_custom_kernel_specs_value(self): for property_key, property_value in propetries: if "default" in property_value: custom_kernel_default_value[property_key] = property_value["default"] - print("custom_kernel_default_value") - print(custom_kernel_default_value) self.custom_kernel_default_value = custom_kernel_default_value @@ -506,7 +504,6 @@ async def _async_pre_start_kernel( keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ - print("---_async_pre_start_kernel---") self.shutting_down = False self.kernel_id = self.kernel_id or kw.pop("kernel_id", str(uuid.uuid4())) # save kwargs for use in restart @@ -520,7 +517,6 @@ async def _async_pre_start_kernel( self.kernel_spec, parent=self, ) - print("---self.provisioner---") print(self.provisioner) kw = await self.provisioner.pre_launch(**kw) # update env @@ -530,11 +526,7 @@ async def _async_pre_start_kernel( kernel_cmd = kw.pop("cmd") if "custom_kernel_specs" in kw: del kw["custom_kernel_specs"] - # if "custom_kernel_specs" in self._launch_args: - # del self._launch_args['custom_kernel_specs'] - #print("_async_pre_start_kernel- km---after deletinh") - # print(kw) return kernel_cmd, kw pre_start_kernel = run_sync(_async_pre_start_kernel) @@ -675,20 +667,16 @@ async def _async_shutdown_kernel(self, now: bool = False, restart: bool = False) self.stop_restarter() if self.has_kernel: - print("---self.has_kernel--") await self._async_interrupt_kernel() if now: - print("---_async_kill_kernel---") await self._async_kill_kernel() else: - print("---_async_request_shutdown---") await self._async_request_shutdown(restart=restart) # Don't send any additional kernel kill messages immediately, to give # the kernel a chance to properly execute shutdown actions. Wait for at # most 1s, checking every 0.1s. await self._async_finish_shutdown(restart=restart) - print("---_async_cleanup_resources---") await self._async_cleanup_resources(restart=restart) shutdown_kernel = run_sync(_async_shutdown_kernel) @@ -728,12 +716,10 @@ async def _async_restart_kernel( await self._async_shutdown_kernel(now=now, restart=True) if newports: - print("---cleanup_random_ports---") self.cleanup_random_ports() # Start new kernel. self._launch_args.update(kw) - print("----self._launch_args---- after") print(self._launch_args) await self._async_start_kernel(**self._launch_args) diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index 9f456f8d..f24a3307 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -223,7 +223,7 @@ def update_env(self, *, kernel_id: str, env: t.Dict[str, str]) -> None: .. version-added: 8.5 """ - print("update_env ----- here- multi") + if kernel_id in self: self._kernels[kernel_id].update_env(env=env) @@ -290,8 +290,6 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: ) kwargs["kernel_id"] = kernel_id # Make kernel_id available to manager and provisioner - - print("---_async_start_kernel--") starter = ensure_async(km.start_kernel(**kwargs)) task = asyncio.create_task(self._add_kernel_when_ready(kernel_id, km, starter)) diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index fc325dfc..a6274c77 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -214,8 +214,6 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: if "custom_kernel_specs" in kwargs: del kwargs["custom_kernel_specs"] - #print("--After deleting--kwargs----") - #print(kwargs) return await super().pre_launch(cmd=kernel_cmd, **kwargs) async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> KernelConnectionInfo: diff --git a/jupyter_client/provisioning/provisioner_base.py b/jupyter_client/provisioning/provisioner_base.py index a3804b8e..9291d9d7 100644 --- a/jupyter_client/provisioning/provisioner_base.py +++ b/jupyter_client/provisioning/provisioner_base.py @@ -154,7 +154,6 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: Returns the (potentially updated) keyword arguments that are passed to :meth:`launch_kernel()`. """ - print("---provisiioning base---") env = kwargs.pop("env", os.environ).copy() # here!!! env.update(self.__apply_env_substitutions(env)) From 1b0c92e6d5f62ec455011960a0ebb4515394fa80 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:43:41 +0000 Subject: [PATCH 12/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- jupyter_client/launcher.py | 1 - jupyter_client/manager.py | 58 +++++++++---------- jupyter_client/multikernelmanager.py | 35 +++++------ .../provisioning/local_provisioner.py | 6 +- .../provisioning/provisioner_base.py | 4 +- 5 files changed, 51 insertions(+), 53 deletions(-) diff --git a/jupyter_client/launcher.py b/jupyter_client/launcher.py index d4b56d0d..f361641e 100644 --- a/jupyter_client/launcher.py +++ b/jupyter_client/launcher.py @@ -78,7 +78,6 @@ def launch_kernel( env = env if (env is not None) else os.environ.copy() - kwargs = kw.copy() main_args = { "stdin": _stdin, diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index fd8e74c8..885795cd 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -303,9 +303,9 @@ def update_env(self, *, env: t.Dict[str, str]) -> None: ): # if self._launch_args["env"] has custom kernel variable for env but env does not have then we have to fill env with it if "custom_kernel_specs" in self._launch_args: - saved_env = self._launch_args.get("env", {}) - custom_kernel_dict = self._launch_args["custom_kernel_specs"] - if isinstance(custom_kernel_dict, dict): + saved_env = self._launch_args.get("env", {}) + custom_kernel_dict = self._launch_args["custom_kernel_specs"] + if isinstance(custom_kernel_dict, dict): for key, value in custom_kernel_dict.items(): if key in saved_env and key not in env: env[key] = saved_env[key] @@ -328,16 +328,18 @@ def update_custom_env_parameters(self, env: t.Dict[str, str]) -> t.Dict[str, str elif self.custom_kernel_default_value: # if not but default values are present into a kernel.json file then we have to take them custom_kernel_dict = self.custom_kernel_default_value - + if isinstance(custom_kernel_dict, dict) and len(custom_kernel_dict) > 0: for custom_kernel_spec, custom_kernel_spec_value in custom_kernel_dict.items(): for env_key, env_item in env.items(): - kernel_spec_item = self.replace_spec_parameter(custom_kernel_spec, custom_kernel_spec_value, env_item) - newEnv[env_key]=kernel_spec_item + kernel_spec_item = self.replace_spec_parameter( + custom_kernel_spec, custom_kernel_spec_value, env_item + ) + newEnv[env_key] = kernel_spec_item else: - # check whether there are custom kernel spec variables into kernel.json, - # if yes but a user has not configured them and default ones are not present , - # we should clean them + # check whether there are custom kernel spec variables into kernel.json, + # if yes but a user has not configured them and default ones are not present , + # we should clean them newEnv = self.clear_custom_kernel_parameters(env) if len(newEnv) > 0: @@ -346,45 +348,42 @@ def update_custom_env_parameters(self, env: t.Dict[str, str]) -> t.Dict[str, str env = self.clear_custom_kernel_parameters(env) return env - - def replace_spec_parameter(self, variable, value, spec)->str: + + def replace_spec_parameter(self, variable, value, spec) -> str: regexp = r"\{" + variable + "\\}" pattern = re.compile(regexp) return pattern.sub(value, spec) - - def check_existence_custom_kernel_spec(self, item:str): + + def check_existence_custom_kernel_spec(self, item: str): pattern = re.compile(r"\{([A-Za-z0-9_]+)\}") matches = pattern.findall(item) isMatch = False if len(matches) > 0: isMatch = True return isMatch - + # Clear kernel specs files if user has not configured them themselves # we shoud return only that has not kernel custom variables # if there are no metadata specification for custom kernel - def clear_custom_kernel_parameters(self, kernel_parameters: t.Any)->t.Any: + def clear_custom_kernel_parameters(self, kernel_parameters: t.Any) -> t.Any: clean_parameters = None if isinstance(kernel_parameters, list): - clean_parameters = [] for argv_item in kernel_parameters: isMatch = self.check_existence_custom_kernel_spec(argv_item) if not isMatch: clean_parameters.append(argv_item) elif isinstance(kernel_parameters, dict): - clean_parameters = {} for env_key, env_item in kernel_parameters.items(): - isMatch = self.check_existence_custom_kernel_spec(env_item) if not isMatch: clean_parameters[env_key] = env_item if len(clean_parameters) == 0: clean_parameters = kernel_parameters return clean_parameters - + def get_default_custom_kernel_specs_value(self): assert self.kernel_spec is not None custom_kernel_default_value = {} @@ -395,14 +394,13 @@ def get_default_custom_kernel_specs_value(self): and isinstance(self.kernel_spec.metadata["parameters"], dict) and "properties" in self.kernel_spec.metadata["parameters"] and isinstance(self.kernel_spec.metadata["parameters"]["properties"], dict) - ): - propetries = self.kernel_spec.metadata["parameters"]["properties"].items() - for property_key, property_value in propetries: - if "default" in property_value: - custom_kernel_default_value[property_key] = property_value["default"] + ): + propetries = self.kernel_spec.metadata["parameters"]["properties"].items() + for property_key, property_value in propetries: + if "default" in property_value: + custom_kernel_default_value[property_key] = property_value["default"] self.custom_kernel_default_value = custom_kernel_default_value - def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> t.List[str]: """Replace templated args (e.g. {connection_file})""" extra_arguments = extra_arguments or [] @@ -436,9 +434,9 @@ def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> if "custom_kernel_specs" in self._launch_args: custom_kernel_dict = self._launch_args["custom_kernel_specs"] if self.custom_kernel_default_value: - for key, value in self.custom_kernel_default_value.items(): - if isinstance(custom_kernel_dict, dict) and key not in custom_kernel_dict: - custom_kernel_dict[key] = value + for key, value in self.custom_kernel_default_value.items(): + if isinstance(custom_kernel_dict, dict) and key not in custom_kernel_dict: + custom_kernel_dict[key] = value elif self.custom_kernel_default_value: # if not but default values are present into a kernel.json file then we have to take them custom_kernel_dict = self.custom_kernel_default_value @@ -521,8 +519,8 @@ async def _async_pre_start_kernel( kw = await self.provisioner.pre_launch(**kw) # update env if "env" in kw: - kw["env"] = self.update_custom_env_parameters(env=kw["env"]) - self._launch_args["env"].update(kw["env"]) + kw["env"] = self.update_custom_env_parameters(env=kw["env"]) + self._launch_args["env"].update(kw["env"]) kernel_cmd = kw.pop("cmd") if "custom_kernel_specs" in kw: del kw["custom_kernel_specs"] diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index f24a3307..c5bb6e3c 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -5,8 +5,8 @@ import asyncio import json -import re import os +import re import socket import typing as t import uuid @@ -223,7 +223,7 @@ def update_env(self, *, kernel_id: str, env: t.Dict[str, str]) -> None: .. version-added: 8.5 """ - + if kernel_id in self: self._kernels[kernel_id].update_env(env=env) @@ -253,23 +253,25 @@ def _using_pending_kernels(self) -> bool: this multikernelmanager is using pending kernels or not """ return getattr(self, "use_pending_kernels", False) - - def validate(self, string)->str: - sanitazed_string = re.sub(r'[;&|$#]', '', string) - match = re.match(r"'", sanitazed_string) - if match: - sanitazed_string = "'" + re.sub(r"'", "'\''", sanitazed_string) +"'" - return sanitazed_string - - def validate_kernel_parameters(self, kwargs: t.Any)-> None: + + def validate(self, string) -> str: + sanitazed_string = re.sub(r"[;&|$#]", "", string) + match = re.match(r"'", sanitazed_string) + if match: + sanitazed_string = "'" + re.sub(r"'", "'''", sanitazed_string) + "'" + return sanitazed_string + + def validate_kernel_parameters(self, kwargs: t.Any) -> None: if "custom_kernel_specs" in kwargs: - custom_kernel_specs = kwargs.get("custom_kernel_specs") - if custom_kernel_specs is not None: - for custom_kernel_spec, custom_kernel_spec_value in kwargs["custom_kernel_specs"].items(): + custom_kernel_specs = kwargs.get("custom_kernel_specs") + if custom_kernel_specs is not None: + for custom_kernel_spec, custom_kernel_spec_value in kwargs[ + "custom_kernel_specs" + ].items(): sanitazed_string = self.validate(custom_kernel_spec_value) - if (sanitazed_string !=''): + if sanitazed_string != "": kwargs["custom_kernel_specs"][custom_kernel_spec] = sanitazed_string - return kwargs + return kwargs async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: t.Any) -> str: """Start a new kernel. @@ -290,7 +292,6 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: ) kwargs["kernel_id"] = kernel_id # Make kernel_id available to manager and provisioner - starter = ensure_async(km.start_kernel(**kwargs)) task = asyncio.create_task(self._add_kernel_when_ready(kernel_id, km, starter)) self._pending_kernels[kernel_id] = task diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index a6274c77..a5587795 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -195,7 +195,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: if "env" in kwargs: # update env if there is custom kernel specs variables for env km.update_env(env=kwargs["env"]) - + jupyter_session = kwargs["env"].get("JPY_SESSION_NAME", "") km.write_connection_file(jupyter_session=jupyter_session) else: @@ -210,8 +210,8 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: kernel_cmd = self.kernel_spec.argv + extra_arguments kernel_cmd = km.clear_custom_kernel_parameters(kernel_cmd) - print("cmd--------",kernel_cmd) - + print("cmd--------", kernel_cmd) + if "custom_kernel_specs" in kwargs: del kwargs["custom_kernel_specs"] return await super().pre_launch(cmd=kernel_cmd, **kwargs) diff --git a/jupyter_client/provisioning/provisioner_base.py b/jupyter_client/provisioning/provisioner_base.py index 9291d9d7..51864896 100644 --- a/jupyter_client/provisioning/provisioner_base.py +++ b/jupyter_client/provisioning/provisioner_base.py @@ -157,9 +157,9 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: env = kwargs.pop("env", os.environ).copy() # here!!! env.update(self.__apply_env_substitutions(env)) - + self._finalize_env(env) - + kwargs["env"] = env return kwargs From 610f018b267c24781cbdad098a1640f567ca9b45 Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Fri, 21 Jun 2024 14:37:13 +0200 Subject: [PATCH 13/31] Fix updating launch parameters when a new kernel is selected --- jupyter_client/manager.py | 11 +++++++++-- jupyter_client/multikernelmanager.py | 12 +++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 885795cd..708d0fb4 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -363,7 +363,7 @@ def check_existence_custom_kernel_spec(self, item: str): return isMatch # Clear kernel specs files if user has not configured them themselves - # we shoud return only that has not kernel custom variables + # we should return only that has not kernel custom variables # if there are no metadata specification for custom kernel def clear_custom_kernel_parameters(self, kernel_parameters: t.Any) -> t.Any: @@ -515,7 +515,6 @@ async def _async_pre_start_kernel( self.kernel_spec, parent=self, ) - print(self.provisioner) kw = await self.provisioner.pre_launch(**kw) # update env if "env" in kw: @@ -558,7 +557,14 @@ async def _async_start_kernel(self, **kw: t.Any) -> None: and launching the kernel (e.g. Popen kwargs). """ self._attempted_start = True + print('kw') + print(kw) kernel_cmd, kw = await self._async_pre_start_kernel(**kw) + print('kernel_cmd') + print(kernel_cmd) + + print('self._launch_args') + print(self._launch_args) # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) @@ -718,6 +724,7 @@ async def _async_restart_kernel( # Start new kernel. self._launch_args.update(kw) + print('restart') print(self._launch_args) await self._async_start_kernel(**self._launch_args) diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index c5bb6e3c..66b12bbc 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -281,6 +281,15 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: The kernel ID for the newly started kernel is returned. """ + + if "custom_kernel_specs" in kwargs: + custom_kernel_specs = kwargs.get("custom_kernel_specs") + if custom_kernel_specs is None or (isinstance(custom_kernel_specs, dict) and len(custom_kernel_specs) == 0): + del kwargs["custom_kernel_specs"] + if hasattr(self, '_launch_args') and self._launch_args: + if "custom_kernel_specs" in self._launch_args: + if "custom_kernel_specs" not in kwargs: + del self._launch_args["custom_kernel_specs"] kwargs = self.validate_kernel_parameters(kwargs) km, kernel_name, kernel_id = self.pre_start_kernel(kernel_name, kwargs) @@ -291,7 +300,8 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: ) ) kwargs["kernel_id"] = kernel_id # Make kernel_id available to manager and provisioner - + print('kwargs') + print(kwargs) starter = ensure_async(km.start_kernel(**kwargs)) task = asyncio.create_task(self._add_kernel_when_ready(kernel_id, km, starter)) self._pending_kernels[kernel_id] = task From db2b88eb8fb3da8eb0dccb6c12a254b4fed26eff Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 21 Jun 2024 12:38:05 +0000 Subject: [PATCH 14/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- jupyter_client/manager.py | 8 ++++---- jupyter_client/multikernelmanager.py | 10 ++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 708d0fb4..92c282b0 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -557,13 +557,13 @@ async def _async_start_kernel(self, **kw: t.Any) -> None: and launching the kernel (e.g. Popen kwargs). """ self._attempted_start = True - print('kw') + print("kw") print(kw) kernel_cmd, kw = await self._async_pre_start_kernel(**kw) - print('kernel_cmd') + print("kernel_cmd") print(kernel_cmd) - print('self._launch_args') + print("self._launch_args") print(self._launch_args) # launch the kernel subprocess @@ -724,7 +724,7 @@ async def _async_restart_kernel( # Start new kernel. self._launch_args.update(kw) - print('restart') + print("restart") print(self._launch_args) await self._async_start_kernel(**self._launch_args) diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index 66b12bbc..8d90497b 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -281,12 +281,14 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: The kernel ID for the newly started kernel is returned. """ - + if "custom_kernel_specs" in kwargs: custom_kernel_specs = kwargs.get("custom_kernel_specs") - if custom_kernel_specs is None or (isinstance(custom_kernel_specs, dict) and len(custom_kernel_specs) == 0): + if custom_kernel_specs is None or ( + isinstance(custom_kernel_specs, dict) and len(custom_kernel_specs) == 0 + ): del kwargs["custom_kernel_specs"] - if hasattr(self, '_launch_args') and self._launch_args: + if hasattr(self, "_launch_args") and self._launch_args: if "custom_kernel_specs" in self._launch_args: if "custom_kernel_specs" not in kwargs: del self._launch_args["custom_kernel_specs"] @@ -300,7 +302,7 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: ) ) kwargs["kernel_id"] = kernel_id # Make kernel_id available to manager and provisioner - print('kwargs') + print("kwargs") print(kwargs) starter = ensure_async(km.start_kernel(**kwargs)) task = asyncio.create_task(self._add_kernel_when_ready(kernel_id, km, starter)) From 2d3c9237953bd59f02d9060079d6cdaa19cb1a81 Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Fri, 21 Jun 2024 17:26:11 +0200 Subject: [PATCH 15/31] Fix updating launch parameters when a new kernel is selected --- jupyter_client/manager.py | 8 ++++---- jupyter_client/multikernelmanager.py | 10 ++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 708d0fb4..92c282b0 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -557,13 +557,13 @@ async def _async_start_kernel(self, **kw: t.Any) -> None: and launching the kernel (e.g. Popen kwargs). """ self._attempted_start = True - print('kw') + print("kw") print(kw) kernel_cmd, kw = await self._async_pre_start_kernel(**kw) - print('kernel_cmd') + print("kernel_cmd") print(kernel_cmd) - print('self._launch_args') + print("self._launch_args") print(self._launch_args) # launch the kernel subprocess @@ -724,7 +724,7 @@ async def _async_restart_kernel( # Start new kernel. self._launch_args.update(kw) - print('restart') + print("restart") print(self._launch_args) await self._async_start_kernel(**self._launch_args) diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index 66b12bbc..8d90497b 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -281,12 +281,14 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: The kernel ID for the newly started kernel is returned. """ - + if "custom_kernel_specs" in kwargs: custom_kernel_specs = kwargs.get("custom_kernel_specs") - if custom_kernel_specs is None or (isinstance(custom_kernel_specs, dict) and len(custom_kernel_specs) == 0): + if custom_kernel_specs is None or ( + isinstance(custom_kernel_specs, dict) and len(custom_kernel_specs) == 0 + ): del kwargs["custom_kernel_specs"] - if hasattr(self, '_launch_args') and self._launch_args: + if hasattr(self, "_launch_args") and self._launch_args: if "custom_kernel_specs" in self._launch_args: if "custom_kernel_specs" not in kwargs: del self._launch_args["custom_kernel_specs"] @@ -300,7 +302,7 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: ) ) kwargs["kernel_id"] = kernel_id # Make kernel_id available to manager and provisioner - print('kwargs') + print("kwargs") print(kwargs) starter = ensure_async(km.start_kernel(**kwargs)) task = asyncio.create_task(self._add_kernel_when_ready(kernel_id, km, starter)) From 4e56a30abb5a48941995cfee8f275cb2e8ec228c Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Mon, 24 Jun 2024 18:08:14 +0200 Subject: [PATCH 16/31] Add switching parameter --- jupyter_client/kernelspec.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 41ed2bad..1acb9043 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -228,6 +228,15 @@ def find_kernel_specs(self) -> dict[str, str]: return d # TODO: Caching? + def allow_parameterized_kernels(self, isParameterizedKernel): + self.isParameterizedKernel = isParameterizedKernel + + def _checkParameterizedKernel(self, kspec:KernelSpec)->KernelSpec: + print('kspec') + print(kspec) + if not self.isParameterizedKernel: + return kspec + def _get_kernel_spec_by_name(self, kernel_name: str, resource_dir: str) -> KernelSpec: """Returns a :class:`KernelSpec` instance for a given kernel_name and resource_dir. @@ -248,6 +257,8 @@ def _get_kernel_spec_by_name(self, kernel_name: str, resource_dir: str) -> Kerne if not KPF.instance(parent=self.parent).is_provisioner_available(kspec): raise NoSuchKernel(kernel_name) + + kspec = self._checkParameterizedKernel(kspec) return kspec From 6c6d9af5c766b670be99d4817ed2520e1f628c3e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 16:09:29 +0000 Subject: [PATCH 17/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- jupyter_client/kernelspec.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 1acb9043..044d8aff 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -231,8 +231,8 @@ def find_kernel_specs(self) -> dict[str, str]: def allow_parameterized_kernels(self, isParameterizedKernel): self.isParameterizedKernel = isParameterizedKernel - def _checkParameterizedKernel(self, kspec:KernelSpec)->KernelSpec: - print('kspec') + def _checkParameterizedKernel(self, kspec: KernelSpec) -> KernelSpec: + print("kspec") print(kspec) if not self.isParameterizedKernel: return kspec @@ -257,7 +257,7 @@ def _get_kernel_spec_by_name(self, kernel_name: str, resource_dir: str) -> Kerne if not KPF.instance(parent=self.parent).is_provisioner_available(kspec): raise NoSuchKernel(kernel_name) - + kspec = self._checkParameterizedKernel(kspec) return kspec From e6f68e2b47178b3a7e5604257a0795ca06be2ed2 Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Tue, 25 Jun 2024 10:56:54 +0200 Subject: [PATCH 18/31] Support replacing env and argv by default parameters --- jupyter_client/kernelspec.py | 53 ++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 1acb9043..9355a68c 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -231,11 +231,58 @@ def find_kernel_specs(self) -> dict[str, str]: def allow_parameterized_kernels(self, isParameterizedKernel): self.isParameterizedKernel = isParameterizedKernel - def _checkParameterizedKernel(self, kspec:KernelSpec)->KernelSpec: + def _check_parameterized_kernel(self, kspec:KernelSpec)->KernelSpec: print('kspec') print(kspec) - if not self.isParameterizedKernel: + if self.isParameterizedKernel: return kspec + else: + if ( + kspec.metadata + and isinstance(kspec.metadata, dict) + and "parameters" in kspec.metadata + and isinstance(kspec.metadata["parameters"], dict) + and "properties" in kspec.metadata["parameters"] + and isinstance(kspec.metadata["parameters"]["properties"], dict) + ): + propetries = kspec.metadata["parameters"]["properties"].items() + for property_key, property_value in propetries: + if "default" in property_value: + kspec = self.replaceByDefault(kspec, property_key, property_value['default']) + else: + return kspec + + def replace_spec_parameter(self, variable, value, spec) -> str: + regexp = r"\{" + variable + "\\}" + pattern = re.compile(regexp) + return pattern.sub(value, spec) + + def replaceByDefault(self, kspec, kernel_variable, default_value): + new_env = {} + new_argv = [] + + env = kspec['env'] + argv = kspec['argv'] + + # check and replace env variables + for env_key, env_item in env.items(): + new_env_item = self.replace_spec_parameter( + kernel_variable, default_value, env_item + ) + new_env[env_key] = new_env_item + if len(new_env) > 0 : + kspec['env'] = new_env + + # check and replace argv parameters + for argv_item in argv: + new_argv_item = self.replace_spec_parameter( + kernel_variable, default_value, argv_item + ) + new_argv.append(new_argv_item) + + if len(new_argv)>0: + kspec['argv'] = new_argv + return kspec def _get_kernel_spec_by_name(self, kernel_name: str, resource_dir: str) -> KernelSpec: """Returns a :class:`KernelSpec` instance for a given kernel_name @@ -258,7 +305,7 @@ def _get_kernel_spec_by_name(self, kernel_name: str, resource_dir: str) -> Kerne if not KPF.instance(parent=self.parent).is_provisioner_available(kspec): raise NoSuchKernel(kernel_name) - kspec = self._checkParameterizedKernel(kspec) + #kspec = self._check_parameterized_kernel(kspec) return kspec From b7fac4a3940fc3ebdc72765bc2db97cf36eac20f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 16:08:42 +0000 Subject: [PATCH 19/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- jupyter_client/kernelspec.py | 40 +++++++++++++++++------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 44dfedb1..6ba634ba 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -248,41 +248,39 @@ def _checkParameterizedKernel(self, kspec: KernelSpec) -> KernelSpec: propetries = kspec.metadata["parameters"]["properties"].items() for property_key, property_value in propetries: if "default" in property_value: - kspec = self.replaceByDefault(kspec, property_key, property_value['default']) + kspec = self.replaceByDefault( + kspec, property_key, property_value["default"] + ) else: return kspec - + def replace_spec_parameter(self, variable, value, spec) -> str: regexp = r"\{" + variable + "\\}" pattern = re.compile(regexp) return pattern.sub(value, spec) - + def replaceByDefault(self, kspec, kernel_variable, default_value): - new_env = {} - new_argv = [] + new_env = {} + new_argv = [] - env = kspec['env'] - argv = kspec['argv'] + env = kspec["env"] + argv = kspec["argv"] - # check and replace env variables - for env_key, env_item in env.items(): - new_env_item = self.replace_spec_parameter( - kernel_variable, default_value, env_item - ) + # check and replace env variables + for env_key, env_item in env.items(): + new_env_item = self.replace_spec_parameter(kernel_variable, default_value, env_item) new_env[env_key] = new_env_item - if len(new_env) > 0 : - kspec['env'] = new_env + if len(new_env) > 0: + kspec["env"] = new_env # check and replace argv parameters - for argv_item in argv: - new_argv_item = self.replace_spec_parameter( - kernel_variable, default_value, argv_item - ) + for argv_item in argv: + new_argv_item = self.replace_spec_parameter(kernel_variable, default_value, argv_item) new_argv.append(new_argv_item) - if len(new_argv)>0: - kspec['argv'] = new_argv - return kspec + if len(new_argv) > 0: + kspec["argv"] = new_argv + return kspec def _get_kernel_spec_by_name(self, kernel_name: str, resource_dir: str) -> KernelSpec: """Returns a :class:`KernelSpec` instance for a given kernel_name From 85d08299460b74815fe0e20f019f22702cadcefa Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Wed, 26 Jun 2024 16:31:03 +0200 Subject: [PATCH 20/31] Update a parameter --- jupyter_client/kernelspec.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 44dfedb1..b50f72b2 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -168,6 +168,8 @@ def _user_kernel_dir_default(self) -> str: "whitelist": ("allowed_kernelspecs", "7.0"), } + _is_allowed_insecure_kernel_specs = False + # Method copied from # https://github.com/jupyterhub/jupyterhub/blob/d1a85e53dccfc7b1dd81b0c1985d158cc6b61820/jupyterhub/auth.py#L143-L161 @observe(*list(_deprecated_aliases)) @@ -228,15 +230,18 @@ def find_kernel_specs(self) -> dict[str, str]: return d # TODO: Caching? - def allow_parameterized_kernels(self, isParameterizedKernel): - self.isParameterizedKernel = isParameterizedKernel + def allow_insecure_kernelspec_params(self, is_allowed_insecure_kernel_specs): + print('is_allowed_insecure_kernel_specs') + print(is_allowed_insecure_kernel_specs) + self._is_allowed_insecure_kernel_specs = is_allowed_insecure_kernel_specs - def _checkParameterizedKernel(self, kspec: KernelSpec) -> KernelSpec: + def _check_parameterized_kernel(self, kspec: KernelSpec) -> KernelSpec: print("kspec") - print(kspec) - if self.isParameterizedKernel: + print(kspec.argv) + if self._is_allowed_insecure_kernel_specs == True: return kspec else: + print('yess') if ( kspec.metadata and isinstance(kspec.metadata, dict) @@ -245,7 +250,10 @@ def _checkParameterizedKernel(self, kspec: KernelSpec) -> KernelSpec: and "properties" in kspec.metadata["parameters"] and isinstance(kspec.metadata["parameters"]["properties"], dict) ): + print('yesstart') propetries = kspec.metadata["parameters"]["properties"].items() + print('propetries') + print(propetries) for property_key, property_value in propetries: if "default" in property_value: kspec = self.replaceByDefault(kspec, property_key, property_value['default']) @@ -264,6 +272,12 @@ def replaceByDefault(self, kspec, kernel_variable, default_value): env = kspec['env'] argv = kspec['argv'] + print('replaceByDefault env') + print(env) + + print('replaceByDefault argv') + print(argv) + # check and replace env variables for env_key, env_item in env.items(): new_env_item = self.replace_spec_parameter( @@ -305,7 +319,7 @@ def _get_kernel_spec_by_name(self, kernel_name: str, resource_dir: str) -> Kerne if not KPF.instance(parent=self.parent).is_provisioner_available(kspec): raise NoSuchKernel(kernel_name) - kspec = self._checkParameterizedKernel(kspec) + kspec = self._check_parameterized_kernel(kspec) return kspec From b0442078c7a698e86e1f02e77be49175335f93da Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:33:56 +0000 Subject: [PATCH 21/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- jupyter_client/kernelspec.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 3fc8be9d..df0b7171 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -231,7 +231,7 @@ def find_kernel_specs(self) -> dict[str, str]: # TODO: Caching? def allow_insecure_kernelspec_params(self, is_allowed_insecure_kernel_specs): - print('is_allowed_insecure_kernel_specs') + print("is_allowed_insecure_kernel_specs") print(is_allowed_insecure_kernel_specs) self._is_allowed_insecure_kernel_specs = is_allowed_insecure_kernel_specs @@ -241,7 +241,7 @@ def _check_parameterized_kernel(self, kspec: KernelSpec) -> KernelSpec: if self._is_allowed_insecure_kernel_specs == True: return kspec else: - print('yess') + print("yess") if ( kspec.metadata and isinstance(kspec.metadata, dict) @@ -250,9 +250,9 @@ def _check_parameterized_kernel(self, kspec: KernelSpec) -> KernelSpec: and "properties" in kspec.metadata["parameters"] and isinstance(kspec.metadata["parameters"]["properties"], dict) ): - print('yesstart') + print("yesstart") propetries = kspec.metadata["parameters"]["properties"].items() - print('propetries') + print("propetries") print(propetries) for property_key, property_value in propetries: if "default" in property_value: @@ -276,11 +276,9 @@ def replaceByDefault(self, kspec, kernel_variable, default_value): # check and replace env variables for env_key, env_item in env.items(): - new_env_item = self.replace_spec_parameter( - kernel_variable, default_value, env_item - ) + new_env_item = self.replace_spec_parameter(kernel_variable, default_value, env_item) new_env[env_key] = new_env_item - + if len(new_env) > 0: kspec["env"] = new_env From 072da9aff352d64a7556cb841f716b56953db762 Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Mon, 1 Jul 2024 18:22:32 +0200 Subject: [PATCH 22/31] Count kernel variables --- jupyter_client/kernelspec.py | 158 ++++++++++++++++++++++++++++------- 1 file changed, 129 insertions(+), 29 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 3fc8be9d..7f21453d 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -168,7 +168,7 @@ def _user_kernel_dir_default(self) -> str: "whitelist": ("allowed_kernelspecs", "7.0"), } - _is_allowed_insecure_kernel_specs = False + _allow_insecure_kernelspec_params = False # Method copied from # https://github.com/jupyterhub/jupyterhub/blob/d1a85e53dccfc7b1dd81b0c1985d158cc6b61820/jupyterhub/auth.py#L143-L161 @@ -230,37 +230,133 @@ def find_kernel_specs(self) -> dict[str, str]: return d # TODO: Caching? - def allow_insecure_kernelspec_params(self, is_allowed_insecure_kernel_specs): - print('is_allowed_insecure_kernel_specs') - print(is_allowed_insecure_kernel_specs) - self._is_allowed_insecure_kernel_specs = is_allowed_insecure_kernel_specs + def allow_insecure_kernelspec_params(self, allow_insecure_kernelspec_params): + print('allow_insecure_kernelspec_params') + print(allow_insecure_kernelspec_params) + self._allow_insecure_kernelspec_params = allow_insecure_kernelspec_params def _check_parameterized_kernel(self, kspec: KernelSpec) -> KernelSpec: print("kspec") print(kspec.argv) - if self._is_allowed_insecure_kernel_specs == True: - return kspec + is_secure = self.check_kernel_is_secure(kspec=kspec) + if is_secure == True: + return kspec # a kernel spec is allowed else: - print('yess') - if ( - kspec.metadata - and isinstance(kspec.metadata, dict) - and "parameters" in kspec.metadata - and isinstance(kspec.metadata["parameters"], dict) - and "properties" in kspec.metadata["parameters"] - and isinstance(kspec.metadata["parameters"]["properties"], dict) - ): - print('yesstart') - propetries = kspec.metadata["parameters"]["properties"].items() - print('propetries') - print(propetries) - for property_key, property_value in propetries: - if "default" in property_value: - kspec = self.replaceByDefault( - kspec, property_key, property_value["default"] - ) + if self._allow_insecure_kernelspec_params == True: + return kspec # a kernel spec is allowed + else: + kspec_data = self.check_kernel_custom_all_default_values(kspec=kspec) + if kspec_data.is_default == True: + return kspec_data.kspec # a kernel spec is modyfied and is allowed + + def check_kernel_is_secure(self, kspec): + is_secure = False + if ( + kspec.metadata + and isinstance(kspec.metadata, dict) + and "parameters" in kspec.metadata + and isinstance(kspec.metadata["parameters"], dict) + and "properties" in kspec.metadata["parameters"] + and isinstance(kspec.metadata["parameters"]["properties"], dict) + ): + counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=kspec.metadata["parameters"], counter_secure_kernel_variables=0) + print('counter_secure_kernel_variables') + + env = None + argv = None + sum_argv_kernel_variables = 0 + sum_env_kernel_variables = 0 + if "env" in kspec: + env = kspec.env + sum_env_kernel_variables = self.get_count_all_kernel_variables(self, env) + print("sum_env_kernel_variables") + print(sum_env_kernel_variables) + if "argv" in kspec: + argv = kspec.argv + sum_argv_kernel_variables = self.get_count_all_kernel_variables(self, argv) + print("sum_argv_kernel_variables") + print(sum_argv_kernel_variables) + total_sum_kernel_variables = sum_env_kernel_variables + sum_argv_kernel_variables + if counter_secure_kernel_variables == total_sum_kernel_variables: + is_secure = True else: - return kspec + is_secure = False + else: + is_secure = True + return is_secure + + + def get_count_secure_kernel_variables(self, obj, counter_secure_kernel_variables): + print('get_count_secure_kernel_variables') + print('---obj---') + print(obj) + print('counter_secure_kernel_variables') + print(counter_secure_kernel_variables) + if "properties" in obj: + propetries = obj["properties"].items() + print('propetries') + print(propetries) + if len(propetries) > 0: + for property_key, property_value in propetries: + if property_value["enum"]: + counter_secure_kernel_variables = counter_secure_kernel_variables + 1 + print('if enum counter_secure_kernel_variables') + print(propetries) + elif property_value['type'] == 'object': + counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=obj, counter_secure_kernel_variables=counter_secure_kernel_variables) + print('if object counter_secure_kernel_variables') + print(propetries) + return counter_secure_kernel_variables + + def get_count_all_kernel_variables(self, parameters): + sum = 0 + if isinstance(parameters, list): + for argv_item in parameters: + is_variable = self.has_variable(argv_item) + if is_variable: + sum = sum + 1 + elif isinstance(parameters, dict): + for env_key, env_item in parameters.items(): + is_variable = self.has_variable(env_item) + if is_variable: + sum = sum + 1 + return sum + + + def has_variable(self, string): + pattern = re.compile(r"\{connection_file\}") + match = pattern.match(string) + if match is None: + pattern = re.compile(r"\{([A-Za-z0-9_]+)\}") + match = pattern.match(string) + if match.group(1): + return True + else: + return False + else: + return False + + def check_kernel_custom_all_default_values(self, kspec): + print('yess') + if ( + kspec.metadata + and isinstance(kspec.metadata, dict) + and "parameters" in kspec.metadata + and isinstance(kspec.metadata["parameters"], dict) + and "properties" in kspec.metadata["parameters"] + and isinstance(kspec.metadata["parameters"]["properties"], dict) + ): + print('yesstart') + propetries = kspec.metadata["parameters"]["properties"].items() + print('propetries') + print(propetries) + for property_key, property_value in propetries: + if "default" in property_value: + kspec = self.replaceByDefault( + kspec, property_key, property_value["default"] + ) + else: + return ""# def replace_spec_parameter(self, variable, value, spec) -> str: regexp = r"\{" + variable + "\\}" @@ -271,16 +367,20 @@ def replaceByDefault(self, kspec, kernel_variable, default_value): new_env = {} new_argv = [] - env = kspec["env"] - argv = kspec["argv"] + env = kspec.env + argv = kspec.argv + print('replaceByDefault env') + print(env) + print('replaceByDefault argv') + print() # check and replace env variables for env_key, env_item in env.items(): new_env_item = self.replace_spec_parameter( kernel_variable, default_value, env_item ) new_env[env_key] = new_env_item - + if len(new_env) > 0: kspec["env"] = new_env From b646dc1967bbcfdf35445047214e5b3e5178b3a0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 16:25:43 +0000 Subject: [PATCH 23/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- jupyter_client/kernelspec.py | 86 ++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 136f34be..193e12c0 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -231,7 +231,7 @@ def find_kernel_specs(self) -> dict[str, str]: # TODO: Caching? def allow_insecure_kernelspec_params(self, allow_insecure_kernelspec_params): - print('allow_insecure_kernelspec_params') + print("allow_insecure_kernelspec_params") print(allow_insecure_kernelspec_params) self._allow_insecure_kernelspec_params = allow_insecure_kernelspec_params @@ -240,15 +240,15 @@ def _check_parameterized_kernel(self, kspec: KernelSpec) -> KernelSpec: print(kspec.argv) is_secure = self.check_kernel_is_secure(kspec=kspec) if is_secure == True: - return kspec # a kernel spec is allowed + return kspec # a kernel spec is allowed else: if self._allow_insecure_kernelspec_params == True: - return kspec # a kernel spec is allowed + return kspec # a kernel spec is allowed else: kspec_data = self.check_kernel_custom_all_default_values(kspec=kspec) if kspec_data.is_default == True: - return kspec_data.kspec # a kernel spec is modyfied and is allowed - + return kspec_data.kspec # a kernel spec is modyfied and is allowed + def check_kernel_is_secure(self, kspec): is_secure = False if ( @@ -259,9 +259,11 @@ def check_kernel_is_secure(self, kspec): and "properties" in kspec.metadata["parameters"] and isinstance(kspec.metadata["parameters"]["properties"], dict) ): - counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=kspec.metadata["parameters"], counter_secure_kernel_variables=0) - print('counter_secure_kernel_variables') - + counter_secure_kernel_variables = self.get_count_secure_kernel_variables( + obj=kspec.metadata["parameters"], counter_secure_kernel_variables=0 + ) + print("counter_secure_kernel_variables") + env = None argv = None sum_argv_kernel_variables = 0 @@ -284,60 +286,60 @@ def check_kernel_is_secure(self, kspec): else: is_secure = True return is_secure - def get_count_secure_kernel_variables(self, obj, counter_secure_kernel_variables): - print('get_count_secure_kernel_variables') - print('---obj---') - print(obj) - print('counter_secure_kernel_variables') - print(counter_secure_kernel_variables) - if "properties" in obj: + print("get_count_secure_kernel_variables") + print("---obj---") + print(obj) + print("counter_secure_kernel_variables") + print(counter_secure_kernel_variables) + if "properties" in obj: propetries = obj["properties"].items() - print('propetries') + print("propetries") print(propetries) if len(propetries) > 0: for property_key, property_value in propetries: if property_value["enum"]: counter_secure_kernel_variables = counter_secure_kernel_variables + 1 - print('if enum counter_secure_kernel_variables') + print("if enum counter_secure_kernel_variables") print(propetries) - elif property_value['type'] == 'object': - counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=obj, counter_secure_kernel_variables=counter_secure_kernel_variables) - print('if object counter_secure_kernel_variables') + elif property_value["type"] == "object": + counter_secure_kernel_variables = self.get_count_secure_kernel_variables( + obj=obj, counter_secure_kernel_variables=counter_secure_kernel_variables + ) + print("if object counter_secure_kernel_variables") print(propetries) - return counter_secure_kernel_variables - + return counter_secure_kernel_variables + def get_count_all_kernel_variables(self, parameters): - sum = 0 - if isinstance(parameters, list): + sum = 0 + if isinstance(parameters, list): for argv_item in parameters: is_variable = self.has_variable(argv_item) if is_variable: sum = sum + 1 - elif isinstance(parameters, dict): + elif isinstance(parameters, dict): for env_key, env_item in parameters.items(): is_variable = self.has_variable(env_item) if is_variable: sum = sum + 1 - return sum + return sum - def has_variable(self, string): - pattern = re.compile(r"\{connection_file\}") - match = pattern.match(string) - if match is None: + pattern = re.compile(r"\{connection_file\}") + match = pattern.match(string) + if match is None: pattern = re.compile(r"\{([A-Za-z0-9_]+)\}") match = pattern.match(string) if match.group(1): return True - else: + else: return False - else: - return False - + else: + return False + def check_kernel_custom_all_default_values(self, kspec): - print('yess') + print("yess") if ( kspec.metadata and isinstance(kspec.metadata, dict) @@ -346,17 +348,15 @@ def check_kernel_custom_all_default_values(self, kspec): and "properties" in kspec.metadata["parameters"] and isinstance(kspec.metadata["parameters"]["properties"], dict) ): - print('yesstart') + print("yesstart") propetries = kspec.metadata["parameters"]["properties"].items() - print('propetries') + print("propetries") print(propetries) for property_key, property_value in propetries: if "default" in property_value: - kspec = self.replaceByDefault( - kspec, property_key, property_value["default"] - ) + kspec = self.replaceByDefault(kspec, property_key, property_value["default"]) else: - return ""# + return "" # def replace_spec_parameter(self, variable, value, spec) -> str: regexp = r"\{" + variable + "\\}" @@ -370,9 +370,9 @@ def replaceByDefault(self, kspec, kernel_variable, default_value): env = kspec.env argv = kspec.argv - print('replaceByDefault env') + print("replaceByDefault env") print(env) - print('replaceByDefault argv') + print("replaceByDefault argv") print() # check and replace env variables for env_key, env_item in env.items(): From 7620d386a2974b81e888469456ee5bb3249d502d Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Tue, 2 Jul 2024 19:36:48 +0200 Subject: [PATCH 24/31] Update filtering kernels spec --- jupyter_client/kernelspec.py | 125 +++++++++++++++++++++++------------ 1 file changed, 82 insertions(+), 43 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 136f34be..b2adfaaf 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -240,13 +240,16 @@ def _check_parameterized_kernel(self, kspec: KernelSpec) -> KernelSpec: print(kspec.argv) is_secure = self.check_kernel_is_secure(kspec=kspec) if is_secure == True: + print('---is_secure---') return kspec # a kernel spec is allowed else: if self._allow_insecure_kernelspec_params == True: + print('---_allow_insecure_kernelspec_params---') return kspec # a kernel spec is allowed else: + print('---should default---') kspec_data = self.check_kernel_custom_all_default_values(kspec=kspec) - if kspec_data.is_default == True: + if kspec_data.all_has_default == True: return kspec_data.kspec # a kernel spec is modyfied and is allowed def check_kernel_is_secure(self, kspec): @@ -262,21 +265,8 @@ def check_kernel_is_secure(self, kspec): counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=kspec.metadata["parameters"], counter_secure_kernel_variables=0) print('counter_secure_kernel_variables') - env = None - argv = None - sum_argv_kernel_variables = 0 - sum_env_kernel_variables = 0 - if "env" in kspec: - env = kspec.env - sum_env_kernel_variables = self.get_count_all_kernel_variables(self, env) - print("sum_env_kernel_variables") - print(sum_env_kernel_variables) - if "argv" in kspec: - argv = kspec.argv - sum_argv_kernel_variables = self.get_count_all_kernel_variables(self, argv) - print("sum_argv_kernel_variables") - print(sum_argv_kernel_variables) - total_sum_kernel_variables = sum_env_kernel_variables + sum_argv_kernel_variables + total_sum_kernel_variables = self.get_argv_env_kernel_variables(kspec=kspec) + if counter_secure_kernel_variables == total_sum_kernel_variables: is_secure = True else: @@ -285,6 +275,27 @@ def check_kernel_is_secure(self, kspec): is_secure = True return is_secure + def get_argv_env_kernel_variables(self, kspec): + total_sum_kernel_variables = 0 + env = None + argv = None + sum_argv_kernel_variables = 0 + sum_env_kernel_variables = 0 + if hasattr(kspec, 'env'): + env = kspec.env + sum_env_kernel_variables = self.get_count_all_kernel_variables(parameters=env) + print("sum_env_kernel_variables") + print(sum_env_kernel_variables) + if hasattr(kspec, 'argv'): + argv = kspec.argv + sum_argv_kernel_variables = self.get_count_all_kernel_variables(parameters=argv) + print("sum_argv_kernel_variables") + print(sum_argv_kernel_variables) + total_sum_kernel_variables = sum_env_kernel_variables + sum_argv_kernel_variables + print('total_sum_kernel_variables') + print(total_sum_kernel_variables) + return total_sum_kernel_variables + def get_count_secure_kernel_variables(self, obj, counter_secure_kernel_variables): print('get_count_secure_kernel_variables') @@ -298,14 +309,14 @@ def get_count_secure_kernel_variables(self, obj, counter_secure_kernel_variables print(propetries) if len(propetries) > 0: for property_key, property_value in propetries: - if property_value["enum"]: + if property_value.get("enum"): counter_secure_kernel_variables = counter_secure_kernel_variables + 1 print('if enum counter_secure_kernel_variables') - print(propetries) - elif property_value['type'] == 'object': + print(counter_secure_kernel_variables) + elif property_value.get("type") == 'object': counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=obj, counter_secure_kernel_variables=counter_secure_kernel_variables) print('if object counter_secure_kernel_variables') - print(propetries) + print(counter_secure_kernel_variables) return counter_secure_kernel_variables def get_count_all_kernel_variables(self, parameters): @@ -329,9 +340,12 @@ def has_variable(self, string): if match is None: pattern = re.compile(r"\{([A-Za-z0-9_]+)\}") match = pattern.match(string) - if match.group(1): - return True - else: + if match: + if match.group(1): + return True + else: + return False + else: return False else: return False @@ -348,15 +362,33 @@ def check_kernel_custom_all_default_values(self, kspec): ): print('yesstart') propetries = kspec.metadata["parameters"]["properties"].items() + + new_kspec = {} print('propetries') print(propetries) for property_key, property_value in propetries: if "default" in property_value: - kspec = self.replaceByDefault( + new_kspec = self.replaceByDefault( kspec, property_key, property_value["default"] ) + total_sum_kernel_variables = self.get_argv_env_kernel_variables(kspec=new_kspec) + if total_sum_kernel_variables > 0: + result = { + "kspec": kspec, + "all_have_default" : False + } + + return result + else: + result = { + "kspec": new_kspec, + "all_have_default" : True + } else: - return ""# + result = { + "kspec": kspec, + "all_have_default" : False + } def replace_spec_parameter(self, variable, value, spec) -> str: regexp = r"\{" + variable + "\\}" @@ -366,29 +398,36 @@ def replace_spec_parameter(self, variable, value, spec) -> str: def replaceByDefault(self, kspec, kernel_variable, default_value): new_env = {} new_argv = [] + if hasattr(kspec, 'env'): + tmp_env = kspec.env.copy() + print('----tmp_env---') + print(tmp_env) + env = tmp_env.env + + - env = kspec.env - argv = kspec.argv + print('replaceByDefault env') + print(env) + # check and replace env variables - print('replaceByDefault env') - print(env) - print('replaceByDefault argv') - print() - # check and replace env variables - for env_key, env_item in env.items(): - new_env_item = self.replace_spec_parameter(kernel_variable, default_value, env_item) - new_env[env_key] = new_env_item + for env_key, env_item in env.items(): + new_env_item = self.replace_spec_parameter(kernel_variable, default_value, env_item) + new_env[env_key] = new_env_item - if len(new_env) > 0: - kspec["env"] = new_env + if len(new_env) > 0: + tmp_env.update(new_env) + kspec.env = tmp_env # check and replace argv parameters - for argv_item in argv: - new_argv_item = self.replace_spec_parameter(kernel_variable, default_value, argv_item) - new_argv.append(new_argv_item) - - if len(new_argv) > 0: - kspec["argv"] = new_argv + if hasattr(kspec, 'argv'): + argv = kspec.argv.copy() + for argv_item in argv: + new_argv_item = self.replace_spec_parameter(kernel_variable, default_value, argv_item) + new_argv.append(new_argv_item) + + if len(new_argv) > 0: + argv = new_argv + kspec.argv = new_argv return kspec def _get_kernel_spec_by_name(self, kernel_name: str, resource_dir: str) -> KernelSpec: From b6f2406db7c810e519e1314ff9c732aca800f3e6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 17:50:34 +0000 Subject: [PATCH 25/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- jupyter_client/kernelspec.py | 82 +++++++++++++++++------------------- 1 file changed, 38 insertions(+), 44 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 796156ab..c9cf5750 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -240,18 +240,18 @@ def _check_parameterized_kernel(self, kspec: KernelSpec) -> KernelSpec: print(kspec.argv) is_secure = self.check_kernel_is_secure(kspec=kspec) if is_secure == True: - print('---is_secure---') - return kspec # a kernel spec is allowed + print("---is_secure---") + return kspec # a kernel spec is allowed else: if self._allow_insecure_kernelspec_params == True: - print('---_allow_insecure_kernelspec_params---') - return kspec # a kernel spec is allowed + print("---_allow_insecure_kernelspec_params---") + return kspec # a kernel spec is allowed else: - print('---should default---') + print("---should default---") kspec_data = self.check_kernel_custom_all_default_values(kspec=kspec) if kspec_data.all_has_default == True: - return kspec_data.kspec # a kernel spec is modyfied and is allowed - + return kspec_data.kspec # a kernel spec is modyfied and is allowed + def check_kernel_is_secure(self, kspec): is_secure = False if ( @@ -262,11 +262,13 @@ def check_kernel_is_secure(self, kspec): and "properties" in kspec.metadata["parameters"] and isinstance(kspec.metadata["parameters"]["properties"], dict) ): - counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=kspec.metadata["parameters"], counter_secure_kernel_variables=0) - print('counter_secure_kernel_variables') - + counter_secure_kernel_variables = self.get_count_secure_kernel_variables( + obj=kspec.metadata["parameters"], counter_secure_kernel_variables=0 + ) + print("counter_secure_kernel_variables") + total_sum_kernel_variables = self.get_argv_env_kernel_variables(kspec=kspec) - + if counter_secure_kernel_variables == total_sum_kernel_variables: is_secure = True else: @@ -274,29 +276,28 @@ def check_kernel_is_secure(self, kspec): else: is_secure = True return is_secure - + def get_argv_env_kernel_variables(self, kspec): total_sum_kernel_variables = 0 env = None argv = None sum_argv_kernel_variables = 0 sum_env_kernel_variables = 0 - if hasattr(kspec, 'env'): + if hasattr(kspec, "env"): env = kspec.env sum_env_kernel_variables = self.get_count_all_kernel_variables(parameters=env) print("sum_env_kernel_variables") print(sum_env_kernel_variables) - if hasattr(kspec, 'argv'): + if hasattr(kspec, "argv"): argv = kspec.argv sum_argv_kernel_variables = self.get_count_all_kernel_variables(parameters=argv) print("sum_argv_kernel_variables") print(sum_argv_kernel_variables) total_sum_kernel_variables = sum_env_kernel_variables + sum_argv_kernel_variables - print('total_sum_kernel_variables') + print("total_sum_kernel_variables") print(total_sum_kernel_variables) return total_sum_kernel_variables - def get_count_secure_kernel_variables(self, obj, counter_secure_kernel_variables): print("get_count_secure_kernel_variables") print("---obj---") @@ -311,14 +312,16 @@ def get_count_secure_kernel_variables(self, obj, counter_secure_kernel_variables for property_key, property_value in propetries: if property_value.get("enum"): counter_secure_kernel_variables = counter_secure_kernel_variables + 1 - print('if enum counter_secure_kernel_variables') + print("if enum counter_secure_kernel_variables") print(counter_secure_kernel_variables) - elif property_value.get("type") == 'object': - counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=obj, counter_secure_kernel_variables=counter_secure_kernel_variables) - print('if object counter_secure_kernel_variables') + elif property_value.get("type") == "object": + counter_secure_kernel_variables = self.get_count_secure_kernel_variables( + obj=obj, counter_secure_kernel_variables=counter_secure_kernel_variables + ) + print("if object counter_secure_kernel_variables") print(counter_secure_kernel_variables) return counter_secure_kernel_variables - + def get_count_all_kernel_variables(self, parameters): sum = 0 if isinstance(parameters, list): @@ -342,7 +345,7 @@ def has_variable(self, string): if match: if match.group(1): return True - else: + else: return False else: return False @@ -361,9 +364,9 @@ def check_kernel_custom_all_default_values(self, kspec): ): print("yesstart") propetries = kspec.metadata["parameters"]["properties"].items() - + new_kspec = {} - print('propetries') + print("propetries") print(propetries) for property_key, property_value in propetries: if "default" in property_value: @@ -372,22 +375,13 @@ def check_kernel_custom_all_default_values(self, kspec): ) total_sum_kernel_variables = self.get_argv_env_kernel_variables(kspec=new_kspec) if total_sum_kernel_variables > 0: - result = { - "kspec": kspec, - "all_have_default" : False - } + result = {"kspec": kspec, "all_have_default": False} return result - else: - result = { - "kspec": new_kspec, - "all_have_default" : True - } + else: + result = {"kspec": new_kspec, "all_have_default": True} else: - result = { - "kspec": kspec, - "all_have_default" : False - } + result = {"kspec": kspec, "all_have_default": False} def replace_spec_parameter(self, variable, value, spec) -> str: regexp = r"\{" + variable + "\\}" @@ -397,15 +391,13 @@ def replace_spec_parameter(self, variable, value, spec) -> str: def replaceByDefault(self, kspec, kernel_variable, default_value): new_env = {} new_argv = [] - if hasattr(kspec, 'env'): + if hasattr(kspec, "env"): tmp_env = kspec.env.copy() - print('----tmp_env---') + print("----tmp_env---") print(tmp_env) env = tmp_env.env - - - print('replaceByDefault env') + print("replaceByDefault env") print(env) # check and replace env variables @@ -418,10 +410,12 @@ def replaceByDefault(self, kspec, kernel_variable, default_value): kspec.env = tmp_env # check and replace argv parameters - if hasattr(kspec, 'argv'): + if hasattr(kspec, "argv"): argv = kspec.argv.copy() for argv_item in argv: - new_argv_item = self.replace_spec_parameter(kernel_variable, default_value, argv_item) + new_argv_item = self.replace_spec_parameter( + kernel_variable, default_value, argv_item + ) new_argv.append(new_argv_item) if len(new_argv) > 0: From 3444a5ec5ea0853222e4d8bd61027583a704645b Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Thu, 4 Jul 2024 12:00:24 +0200 Subject: [PATCH 26/31] Fix checking default values,setup is_secure flag for a kernel spec file --- jupyter_client/kernelspec.py | 146 +++++++++++++++++++++++------------ 1 file changed, 97 insertions(+), 49 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 796156ab..44446cff 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -240,19 +240,38 @@ def _check_parameterized_kernel(self, kspec: KernelSpec) -> KernelSpec: print(kspec.argv) is_secure = self.check_kernel_is_secure(kspec=kspec) if is_secure == True: + if kspec.metadata and isinstance(kspec.metadata, dict): + kspec.metadata.update({'is_secure':True}) + else: + kspec.metadata = {} + kspec.metadata.update({'is_secure':True}) print('---is_secure---') return kspec # a kernel spec is allowed else: + if kspec.metadata and isinstance(kspec.metadata, dict): + kspec.metadata.update({'is_secure':False}) + else: + kspec.metadata = {} + kspec.metadata.update({'is_secure':False}) if self._allow_insecure_kernelspec_params == True: print('---_allow_insecure_kernelspec_params---') return kspec # a kernel spec is allowed else: print('---should default---') kspec_data = self.check_kernel_custom_all_default_values(kspec=kspec) - if kspec_data.all_has_default == True: - return kspec_data.kspec # a kernel spec is modyfied and is allowed + print("kspec_data") + print(kspec_data) + print('??????') + print(kspec_data["all_have_default"]) + + if kspec_data['all_have_default'] == True: + print('default_true') + return kspec_data["kspec"] # a kernel spec is modyfied and is allowed + else: + return None def check_kernel_is_secure(self, kspec): + print('check_kernel_is_secure') is_secure = False if ( kspec.metadata @@ -263,7 +282,6 @@ def check_kernel_is_secure(self, kspec): and isinstance(kspec.metadata["parameters"]["properties"], dict) ): counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=kspec.metadata["parameters"], counter_secure_kernel_variables=0) - print('counter_secure_kernel_variables') total_sum_kernel_variables = self.get_argv_env_kernel_variables(kspec=kspec) @@ -301,28 +319,40 @@ def get_count_secure_kernel_variables(self, obj, counter_secure_kernel_variables print("get_count_secure_kernel_variables") print("---obj---") print(obj) - print("counter_secure_kernel_variables") + print(counter_secure_kernel_variables) + is_secure = True if "properties" in obj: propetries = obj["properties"].items() - print("propetries") - print(propetries) if len(propetries) > 0: for property_key, property_value in propetries: - if property_value.get("enum"): + if property_value.get("type") == 'string' or property_value.get("type") == 'null': + if property_value.get("enum"): + counter_secure_kernel_variables = counter_secure_kernel_variables + 1 + else: + print('string, null') + is_secure = False + elif property_value.get("enum"): counter_secure_kernel_variables = counter_secure_kernel_variables + 1 print('if enum counter_secure_kernel_variables') - print(counter_secure_kernel_variables) + elif property_value.get("type") == 'object': counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=obj, counter_secure_kernel_variables=counter_secure_kernel_variables) print('if object counter_secure_kernel_variables') - print(counter_secure_kernel_variables) + + if is_secure == False: + counter_secure_kernel_variables = 0 + print("counter_secure_kernel_variables") + print(counter_secure_kernel_variables) + return counter_secure_kernel_variables def get_count_all_kernel_variables(self, parameters): sum = 0 if isinstance(parameters, list): for argv_item in parameters: + print("argv_item") + print(argv_item) is_variable = self.has_variable(argv_item) if is_variable: sum = sum + 1 @@ -333,24 +363,24 @@ def get_count_all_kernel_variables(self, parameters): sum = sum + 1 return sum - def has_variable(self, string): + def has_variable(self, string: str): pattern = re.compile(r"\{connection_file\}") match = pattern.match(string) if match is None: pattern = re.compile(r"\{([A-Za-z0-9_]+)\}") - match = pattern.match(string) - if match: - if match.group(1): - return True - else: - return False - else: + matches = pattern.findall(string) + print('matches') + print(matches) + if len(matches) > 0: + print('-match yes') + return True + else: return False else: return False def check_kernel_custom_all_default_values(self, kspec): - print("yess") + print("check_kernel_custom_all_default_values") if ( kspec.metadata and isinstance(kspec.metadata, dict) @@ -359,35 +389,47 @@ def check_kernel_custom_all_default_values(self, kspec): and "properties" in kspec.metadata["parameters"] and isinstance(kspec.metadata["parameters"]["properties"], dict) ): - print("yesstart") + has_default = True propetries = kspec.metadata["parameters"]["properties"].items() new_kspec = {} - print('propetries') - print(propetries) for property_key, property_value in propetries: if "default" in property_value: new_kspec = self.replaceByDefault( kspec, property_key, property_value["default"] ) - total_sum_kernel_variables = self.get_argv_env_kernel_variables(kspec=new_kspec) - if total_sum_kernel_variables > 0: - result = { - "kspec": kspec, - "all_have_default" : False - } + else: + has_default = False - return result - else: + if has_default == False: result = { - "kspec": new_kspec, - "all_have_default" : True + "kspec": kspec, + "all_have_default" : False } + else: + #check if there is anything after replacing + total_sum_kernel_variables = self.get_argv_env_kernel_variables(kspec=new_kspec) + print('---default total_sum_kernel_variables---') + print(total_sum_kernel_variables) + + if total_sum_kernel_variables > 0: + result = { + "kspec": kspec, + "all_have_default" : False + } + else: + result = { + "kspec": new_kspec, + "all_have_default" : True + } else: result = { "kspec": kspec, "all_have_default" : False } + print("result") + print(result["all_have_default"]) + return result def replace_spec_parameter(self, variable, value, spec) -> str: regexp = r"\{" + variable + "\\}" @@ -399,26 +441,26 @@ def replaceByDefault(self, kspec, kernel_variable, default_value): new_argv = [] if hasattr(kspec, 'env'): tmp_env = kspec.env.copy() - print('----tmp_env---') - print(tmp_env) - env = tmp_env.env + if 'env' in tmp_env: + print('----tmp_env---') + print(tmp_env) + env = tmp_env.env + - - - print('replaceByDefault env') - print(env) - # check and replace env variables + print('replaceByDefault env') + print(env) + # check and replace env variables - for env_key, env_item in env.items(): - new_env_item = self.replace_spec_parameter(kernel_variable, default_value, env_item) - new_env[env_key] = new_env_item + for env_key, env_item in env.items(): + new_env_item = self.replace_spec_parameter(kernel_variable, default_value, env_item) + new_env[env_key] = new_env_item - if len(new_env) > 0: - tmp_env.update(new_env) - kspec.env = tmp_env + if len(new_env) > 0: + tmp_env.update(new_env) + kspec.env = tmp_env # check and replace argv parameters - if hasattr(kspec, 'argv'): + if hasattr(kspec, 'argv') and kspec.argv is not None: argv = kspec.argv.copy() for argv_item in argv: new_argv_item = self.replace_spec_parameter(kernel_variable, default_value, argv_item) @@ -451,8 +493,14 @@ def _get_kernel_spec_by_name(self, kernel_name: str, resource_dir: str) -> Kerne raise NoSuchKernel(kernel_name) kspec = self._check_parameterized_kernel(kspec) + print('new kspec') + print(kspec) + if kspec is not None: + return kspec + else: + return None - return kspec + def _find_spec_directory(self, kernel_name: str) -> str | None: """Find the resource directory of a named kernel spec""" @@ -513,8 +561,8 @@ def get_all_specs(self) -> dict[str, t.Any]: # which may have overridden find_kernel_specs # and get_kernel_spec, but not the newer get_all_specs spec = self.get_kernel_spec(kname) - - res[kname] = {"resource_dir": resource_dir, "spec": spec.to_dict()} + if spec != None: + res[kname] = {"resource_dir": resource_dir, "spec": spec.to_dict()} except NoSuchKernel: pass # The appropriate warning has already been logged except Exception: From 3e29e21573de26d23cff20078b1e49b191db607e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 4 Jul 2024 10:02:46 +0000 Subject: [PATCH 27/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- jupyter_client/kernelspec.py | 106 +++++++++++++++++------------------ 1 file changed, 50 insertions(+), 56 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 2bdb8d36..0cfbaa9b 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -241,18 +241,18 @@ def _check_parameterized_kernel(self, kspec: KernelSpec) -> KernelSpec: is_secure = self.check_kernel_is_secure(kspec=kspec) if is_secure == True: if kspec.metadata and isinstance(kspec.metadata, dict): - kspec.metadata.update({'is_secure':True}) + kspec.metadata.update({"is_secure": True}) else: kspec.metadata = {} - kspec.metadata.update({'is_secure':True}) - print('---is_secure---') - return kspec # a kernel spec is allowed + kspec.metadata.update({"is_secure": True}) + print("---is_secure---") + return kspec # a kernel spec is allowed else: if kspec.metadata and isinstance(kspec.metadata, dict): - kspec.metadata.update({'is_secure':False}) + kspec.metadata.update({"is_secure": False}) else: kspec.metadata = {} - kspec.metadata.update({'is_secure':False}) + kspec.metadata.update({"is_secure": False}) if self._allow_insecure_kernelspec_params == True: print("---_allow_insecure_kernelspec_params---") return kspec # a kernel spec is allowed @@ -261,17 +261,17 @@ def _check_parameterized_kernel(self, kspec: KernelSpec) -> KernelSpec: kspec_data = self.check_kernel_custom_all_default_values(kspec=kspec) print("kspec_data") print(kspec_data) - print('??????') + print("??????") print(kspec_data["all_have_default"]) - if kspec_data['all_have_default'] == True: - print('default_true') - return kspec_data["kspec"] # a kernel spec is modyfied and is allowed + if kspec_data["all_have_default"] == True: + print("default_true") + return kspec_data["kspec"] # a kernel spec is modyfied and is allowed else: return None - + def check_kernel_is_secure(self, kspec): - print('check_kernel_is_secure') + print("check_kernel_is_secure") is_secure = False if ( kspec.metadata @@ -281,8 +281,10 @@ def check_kernel_is_secure(self, kspec): and "properties" in kspec.metadata["parameters"] and isinstance(kspec.metadata["parameters"]["properties"], dict) ): - counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=kspec.metadata["parameters"], counter_secure_kernel_variables=0) - + counter_secure_kernel_variables = self.get_count_secure_kernel_variables( + obj=kspec.metadata["parameters"], counter_secure_kernel_variables=0 + ) + total_sum_kernel_variables = self.get_argv_env_kernel_variables(kspec=kspec) if counter_secure_kernel_variables == total_sum_kernel_variables: @@ -318,27 +320,32 @@ def get_count_secure_kernel_variables(self, obj, counter_secure_kernel_variables print("get_count_secure_kernel_variables") print("---obj---") print(obj) - + print(counter_secure_kernel_variables) is_secure = True if "properties" in obj: propetries = obj["properties"].items() if len(propetries) > 0: for property_key, property_value in propetries: - if property_value.get("type") == 'string' or property_value.get("type") == 'null': + if ( + property_value.get("type") == "string" + or property_value.get("type") == "null" + ): if property_value.get("enum"): counter_secure_kernel_variables = counter_secure_kernel_variables + 1 else: - print('string, null') + print("string, null") is_secure = False elif property_value.get("enum"): counter_secure_kernel_variables = counter_secure_kernel_variables + 1 - print('if enum counter_secure_kernel_variables') - - elif property_value.get("type") == 'object': - counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=obj, counter_secure_kernel_variables=counter_secure_kernel_variables) - print('if object counter_secure_kernel_variables') - + print("if enum counter_secure_kernel_variables") + + elif property_value.get("type") == "object": + counter_secure_kernel_variables = self.get_count_secure_kernel_variables( + obj=obj, counter_secure_kernel_variables=counter_secure_kernel_variables + ) + print("if object counter_secure_kernel_variables") + if is_secure == False: counter_secure_kernel_variables = 0 print("counter_secure_kernel_variables") @@ -368,12 +375,12 @@ def has_variable(self, string: str): if match is None: pattern = re.compile(r"\{([A-Za-z0-9_]+)\}") matches = pattern.findall(string) - print('matches') + print("matches") print(matches) if len(matches) > 0: - print('-match yes') + print("-match yes") return True - else: + else: return False else: return False @@ -401,31 +408,19 @@ def check_kernel_custom_all_default_values(self, kspec): has_default = False if has_default == False: - result = { - "kspec": kspec, - "all_have_default" : False - } + result = {"kspec": kspec, "all_have_default": False} else: - #check if there is anything after replacing + # check if there is anything after replacing total_sum_kernel_variables = self.get_argv_env_kernel_variables(kspec=new_kspec) - print('---default total_sum_kernel_variables---') + print("---default total_sum_kernel_variables---") print(total_sum_kernel_variables) if total_sum_kernel_variables > 0: - result = { - "kspec": kspec, - "all_have_default" : False - } - else: - result = { - "kspec": new_kspec, - "all_have_default" : True - } + result = {"kspec": kspec, "all_have_default": False} + else: + result = {"kspec": new_kspec, "all_have_default": True} else: - result = { - "kspec": kspec, - "all_have_default" : False - } + result = {"kspec": kspec, "all_have_default": False} print("result") print(result["all_have_default"]) return result @@ -440,18 +435,19 @@ def replaceByDefault(self, kspec, kernel_variable, default_value): new_argv = [] if hasattr(kspec, "env"): tmp_env = kspec.env.copy() - if 'env' in tmp_env: - print('----tmp_env---') + if "env" in tmp_env: + print("----tmp_env---") print(tmp_env) env = tmp_env.env - - - print('replaceByDefault env') + + print("replaceByDefault env") print(env) # check and replace env variables for env_key, env_item in env.items(): - new_env_item = self.replace_spec_parameter(kernel_variable, default_value, env_item) + new_env_item = self.replace_spec_parameter( + kernel_variable, default_value, env_item + ) new_env[env_key] = new_env_item if len(new_env) > 0: @@ -459,7 +455,7 @@ def replaceByDefault(self, kspec, kernel_variable, default_value): kspec.env = tmp_env # check and replace argv parameters - if hasattr(kspec, 'argv') and kspec.argv is not None: + if hasattr(kspec, "argv") and kspec.argv is not None: argv = kspec.argv.copy() for argv_item in argv: new_argv_item = self.replace_spec_parameter( @@ -494,15 +490,13 @@ def _get_kernel_spec_by_name(self, kernel_name: str, resource_dir: str) -> Kerne raise NoSuchKernel(kernel_name) kspec = self._check_parameterized_kernel(kspec) - print('new kspec') + print("new kspec") print(kspec) if kspec is not None: - return kspec + return kspec else: return None - - def _find_spec_directory(self, kernel_name: str) -> str | None: """Find the resource directory of a named kernel spec""" for kernel_dir in [kd for kd in self.kernel_dirs if os.path.isdir(kd)]: From a55983bab76d7a407506a7d5da746e6710572897 Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Thu, 4 Jul 2024 17:59:25 +0200 Subject: [PATCH 28/31] Fix defining when a kernel is insecure --- jupyter_client/kernelspec.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 2bdb8d36..ad0cba29 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -273,6 +273,7 @@ def _check_parameterized_kernel(self, kspec: KernelSpec) -> KernelSpec: def check_kernel_is_secure(self, kspec): print('check_kernel_is_secure') is_secure = False + total_sum_kernel_variables = self.get_argv_env_kernel_variables(kspec=kspec) if ( kspec.metadata and isinstance(kspec.metadata, dict) @@ -283,14 +284,16 @@ def check_kernel_is_secure(self, kspec): ): counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=kspec.metadata["parameters"], counter_secure_kernel_variables=0) - total_sum_kernel_variables = self.get_argv_env_kernel_variables(kspec=kspec) - if counter_secure_kernel_variables == total_sum_kernel_variables: is_secure = True else: is_secure = False else: - is_secure = True + # check if there are kernel variables even metadata.parameters are empty + if total_sum_kernel_variables > 0: + is_secure = False + else: + is_secure = True return is_secure def get_argv_env_kernel_variables(self, kspec): From 4e93a128dbd3b7ca62677c67fd018e4477c828f4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 4 Jul 2024 16:01:27 +0000 Subject: [PATCH 29/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- jupyter_client/kernelspec.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 8197c37e..4ff857f6 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -282,14 +282,16 @@ def check_kernel_is_secure(self, kspec): and "properties" in kspec.metadata["parameters"] and isinstance(kspec.metadata["parameters"]["properties"], dict) ): - counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=kspec.metadata["parameters"], counter_secure_kernel_variables=0) - + counter_secure_kernel_variables = self.get_count_secure_kernel_variables( + obj=kspec.metadata["parameters"], counter_secure_kernel_variables=0 + ) + if counter_secure_kernel_variables == total_sum_kernel_variables: is_secure = True else: is_secure = False else: - # check if there are kernel variables even metadata.parameters are empty + # check if there are kernel variables even metadata.parameters are empty if total_sum_kernel_variables > 0: is_secure = False else: From ec2f5e7bcdd450d2884c813aa4920b401783e522 Mon Sep 17 00:00:00 2001 From: Anastasiia Sliusar Date: Fri, 5 Jul 2024 16:36:00 +0200 Subject: [PATCH 30/31] Fix filtering kernel spec files --- jupyter_client/kernelspec.py | 59 +++++----------------------- jupyter_client/manager.py | 11 +----- jupyter_client/multikernelmanager.py | 3 +- 3 files changed, 12 insertions(+), 61 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 8197c37e..6fb0fa3f 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -231,13 +231,9 @@ def find_kernel_specs(self) -> dict[str, str]: # TODO: Caching? def allow_insecure_kernelspec_params(self, allow_insecure_kernelspec_params): - print("allow_insecure_kernelspec_params") - print(allow_insecure_kernelspec_params) self._allow_insecure_kernelspec_params = allow_insecure_kernelspec_params def _check_parameterized_kernel(self, kspec: KernelSpec) -> KernelSpec: - print("kspec") - print(kspec.argv) is_secure = self.check_kernel_is_secure(kspec=kspec) if is_secure == True: if kspec.metadata and isinstance(kspec.metadata, dict): @@ -245,7 +241,6 @@ def _check_parameterized_kernel(self, kspec: KernelSpec) -> KernelSpec: else: kspec.metadata = {} kspec.metadata.update({"is_secure": True}) - print("---is_secure---") return kspec # a kernel spec is allowed else: if kspec.metadata and isinstance(kspec.metadata, dict): @@ -254,24 +249,16 @@ def _check_parameterized_kernel(self, kspec: KernelSpec) -> KernelSpec: kspec.metadata = {} kspec.metadata.update({"is_secure": False}) if self._allow_insecure_kernelspec_params == True: - print("---_allow_insecure_kernelspec_params---") return kspec # a kernel spec is allowed else: - print("---should default---") kspec_data = self.check_kernel_custom_all_default_values(kspec=kspec) - print("kspec_data") - print(kspec_data) - print("??????") - print(kspec_data["all_have_default"]) if kspec_data["all_have_default"] == True: - print("default_true") return kspec_data["kspec"] # a kernel spec is modyfied and is allowed else: return None def check_kernel_is_secure(self, kspec): - print("check_kernel_is_secure") is_secure = False total_sum_kernel_variables = self.get_argv_env_kernel_variables(kspec=kspec) if ( @@ -283,9 +270,11 @@ def check_kernel_is_secure(self, kspec): and isinstance(kspec.metadata["parameters"]["properties"], dict) ): counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=kspec.metadata["parameters"], counter_secure_kernel_variables=0) - - if counter_secure_kernel_variables == total_sum_kernel_variables: - is_secure = True + if total_sum_kernel_variables>0: + if counter_secure_kernel_variables == total_sum_kernel_variables: + is_secure = True + else: + is_secure = False else: is_secure = False else: @@ -305,24 +294,13 @@ def get_argv_env_kernel_variables(self, kspec): if hasattr(kspec, "env"): env = kspec.env sum_env_kernel_variables = self.get_count_all_kernel_variables(parameters=env) - print("sum_env_kernel_variables") - print(sum_env_kernel_variables) if hasattr(kspec, "argv"): argv = kspec.argv sum_argv_kernel_variables = self.get_count_all_kernel_variables(parameters=argv) - print("sum_argv_kernel_variables") - print(sum_argv_kernel_variables) total_sum_kernel_variables = sum_env_kernel_variables + sum_argv_kernel_variables - print("total_sum_kernel_variables") - print(total_sum_kernel_variables) return total_sum_kernel_variables def get_count_secure_kernel_variables(self, obj, counter_secure_kernel_variables): - print("get_count_secure_kernel_variables") - print("---obj---") - print(obj) - - print(counter_secure_kernel_variables) is_secure = True if "properties" in obj: propetries = obj["properties"].items() @@ -335,22 +313,19 @@ def get_count_secure_kernel_variables(self, obj, counter_secure_kernel_variables if property_value.get("enum"): counter_secure_kernel_variables = counter_secure_kernel_variables + 1 else: - print("string, null") is_secure = False + elif property_value.get("type") == "array": + print('Type of JSON Schema data is array and it is not supported now') + is_secure = False elif property_value.get("enum"): counter_secure_kernel_variables = counter_secure_kernel_variables + 1 - print("if enum counter_secure_kernel_variables") - elif property_value.get("type") == "object": counter_secure_kernel_variables = self.get_count_secure_kernel_variables( obj=obj, counter_secure_kernel_variables=counter_secure_kernel_variables ) - print("if object counter_secure_kernel_variables") if is_secure == False: counter_secure_kernel_variables = 0 - print("counter_secure_kernel_variables") - print(counter_secure_kernel_variables) return counter_secure_kernel_variables @@ -358,8 +333,6 @@ def get_count_all_kernel_variables(self, parameters): sum = 0 if isinstance(parameters, list): for argv_item in parameters: - print("argv_item") - print(argv_item) is_variable = self.has_variable(argv_item) if is_variable: sum = sum + 1 @@ -376,10 +349,7 @@ def has_variable(self, string: str): if match is None: pattern = re.compile(r"\{([A-Za-z0-9_]+)\}") matches = pattern.findall(string) - print("matches") - print(matches) if len(matches) > 0: - print("-match yes") return True else: return False @@ -387,7 +357,6 @@ def has_variable(self, string: str): return False def check_kernel_custom_all_default_values(self, kspec): - print("check_kernel_custom_all_default_values") if ( kspec.metadata and isinstance(kspec.metadata, dict) @@ -413,8 +382,6 @@ def check_kernel_custom_all_default_values(self, kspec): else: # check if there is anything after replacing total_sum_kernel_variables = self.get_argv_env_kernel_variables(kspec=new_kspec) - print("---default total_sum_kernel_variables---") - print(total_sum_kernel_variables) if total_sum_kernel_variables > 0: result = {"kspec": kspec, "all_have_default": False} @@ -422,8 +389,6 @@ def check_kernel_custom_all_default_values(self, kspec): result = {"kspec": new_kspec, "all_have_default": True} else: result = {"kspec": kspec, "all_have_default": False} - print("result") - print(result["all_have_default"]) return result def replace_spec_parameter(self, variable, value, spec) -> str: @@ -437,12 +402,7 @@ def replaceByDefault(self, kspec, kernel_variable, default_value): if hasattr(kspec, "env"): tmp_env = kspec.env.copy() if "env" in tmp_env: - print("----tmp_env---") - print(tmp_env) env = tmp_env.env - - print("replaceByDefault env") - print(env) # check and replace env variables for env_key, env_item in env.items(): @@ -491,8 +451,7 @@ def _get_kernel_spec_by_name(self, kernel_name: str, resource_dir: str) -> Kerne raise NoSuchKernel(kernel_name) kspec = self._check_parameterized_kernel(kspec) - print("new kspec") - print(kspec) + if kspec is not None: return kspec else: diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 92c282b0..b9beac83 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -557,14 +557,8 @@ async def _async_start_kernel(self, **kw: t.Any) -> None: and launching the kernel (e.g. Popen kwargs). """ self._attempted_start = True - print("kw") - print(kw) - kernel_cmd, kw = await self._async_pre_start_kernel(**kw) - print("kernel_cmd") - print(kernel_cmd) - print("self._launch_args") - print(self._launch_args) + kernel_cmd, kw = await self._async_pre_start_kernel(**kw) # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) @@ -724,8 +718,7 @@ async def _async_restart_kernel( # Start new kernel. self._launch_args.update(kw) - print("restart") - print(self._launch_args) + await self._async_start_kernel(**self._launch_args) restart_kernel = run_sync(_async_restart_kernel) diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index 8d90497b..268edc23 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -302,8 +302,7 @@ async def _async_start_kernel(self, *, kernel_name: str | None = None, **kwargs: ) ) kwargs["kernel_id"] = kernel_id # Make kernel_id available to manager and provisioner - print("kwargs") - print(kwargs) + starter = ensure_async(km.start_kernel(**kwargs)) task = asyncio.create_task(self._add_kernel_when_ready(kernel_id, km, starter)) self._pending_kernels[kernel_id] = task From 5e8bc79ec7b2c385b1f0b1a167b516cc90be9586 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 5 Jul 2024 14:37:40 +0000 Subject: [PATCH 31/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- jupyter_client/kernelspec.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 7471f718..86324223 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -269,8 +269,10 @@ def check_kernel_is_secure(self, kspec): and "properties" in kspec.metadata["parameters"] and isinstance(kspec.metadata["parameters"]["properties"], dict) ): - counter_secure_kernel_variables = self.get_count_secure_kernel_variables(obj=kspec.metadata["parameters"], counter_secure_kernel_variables=0) - if total_sum_kernel_variables>0: + counter_secure_kernel_variables = self.get_count_secure_kernel_variables( + obj=kspec.metadata["parameters"], counter_secure_kernel_variables=0 + ) + if total_sum_kernel_variables > 0: if counter_secure_kernel_variables == total_sum_kernel_variables: is_secure = True else: @@ -315,7 +317,7 @@ def get_count_secure_kernel_variables(self, obj, counter_secure_kernel_variables else: is_secure = False elif property_value.get("type") == "array": - print('Type of JSON Schema data is array and it is not supported now') + print("Type of JSON Schema data is array and it is not supported now") is_secure = False elif property_value.get("enum"): counter_secure_kernel_variables = counter_secure_kernel_variables + 1