diff --git a/fedcloudclient/cli.py b/fedcloudclient/cli.py index 2eb18e0..144a9fc 100644 --- a/fedcloudclient/cli.py +++ b/fedcloudclient/cli.py @@ -8,6 +8,7 @@ from fedcloudclient.conf import config from fedcloudclient.endpoint import endpoint from fedcloudclient.openstack import openstack, openstack_int +from fedcloudclient.jupyterhub import jupyterhub from fedcloudclient.secret import secret from fedcloudclient.select import select from fedcloudclient.sites import site @@ -29,6 +30,7 @@ def cli(): cli.add_command(openstack) cli.add_command(openstack_int) cli.add_command(config) +cli.add_command(jupyterhub) if __name__ == "__main__": cli() diff --git a/fedcloudclient/jupyterhub.py b/fedcloudclient/jupyterhub.py new file mode 100644 index 0000000..8e16b48 --- /dev/null +++ b/fedcloudclient/jupyterhub.py @@ -0,0 +1,485 @@ +""" +Implementation of "fedcloud jupyterhub" for communicating with JupyterHub instances +in OpenStack CLI like fashion +""" + +import json +from functools import wraps + +import click + +from fedcloudclient.jupyterhub_lib import ( + get_servers, + get_user, + start_server, + stop_server, + add_token, + list_tokens, + get_token, + delete_token, + exec_command, + upload_file, + add_path, + get_path, + delete_path, + add_shared_access, + remove_shared_access, + list_shared_access, +) + + +# Main function calling lib functions +def jupyterhub_full(callback_func, **kwargs): + """ + Calls provided callback func + """ + response_output = callback_func(**kwargs) + + if response_output is not None: + if "output" in kwargs and kwargs["output"] == "text": + print(response_output, end="") + else: + print(json.dumps(response_output, indent=4)) + + +# Decorator for required Jupyterhub hub param +def common_hub_params(func): + """ + Common Hub params func wrapper + """ + + @click.option( + "--hub-api-endpoint", + "-e", + required=True, + help="JupyterHub API endpoint", + ) + @click.option( + "--token", + "-t", + required=True, + help="JupyterHub API token", + ) + @click.option( + "--user", + help="ID of Jupyterhub user, if not specified token's owner ID is used", + ) + @wraps(func) + def wrapper(*args, **kwargs): + func(*args, **kwargs) + + return wrapper + + +# Decorator for server name param +def server_name(func): + """ + Server name param func wrapper + """ + + @click.option( + "--server", + required=True, + help='Name of the Jupyter server. For a nameless server use: --server ""', + ) + @wraps(func) + def wrapper(*args, **kwargs): + func(*args, **kwargs) + + return wrapper + + +# Decorator for include_stopped_servers param +def include_stopped_servers(func): + """ + Include stopped server param func wrapper + """ + + @click.option( + "--include-stopped-servers", + is_flag=True, + flag_value=True, + default=False, + help="Include stopped servers", + ) + @wraps(func) + def wrapper(*args, **kwargs): + func(*args, **kwargs) + + return wrapper + + +# Decorator for token_id param +def api_token_id(func): + """ + Api token param func wrapper + """ + + @click.option( + "--api-token-id", + required=True, + help="API token ID", + ) + @wraps(func) + def wrapper(*args, **kwargs): + func(*args, **kwargs) + + return wrapper + + +@click.group() +def jupyterhub(): + """ + Communicate with Jupyterhub + """ + + +@jupyterhub.group() +def user(): + """ + JupyterHub user subcommand + """ + + +@user.command("show") +@include_stopped_servers +@common_hub_params +def show_user(**kwargs): + """ + Get user details + """ + jupyterhub_full(get_user, **kwargs) + + +@jupyterhub.group() +def server(): + """ + JupyterHub server subcommand + """ + + +@server.command("list") +@include_stopped_servers +@common_hub_params +def list_servers(**kwargs): + """ + JupyterHub server subcommand + """ + jupyterhub_full(get_servers, **kwargs) + + +@server.command("start") +@click.option( + "--options", + help=""" + User options JSON to be passed to the server, e.g. different image, + spawner profile, etc., must valid json!!""", +) +@common_hub_params +@server_name +def start_user_server(**kwargs): + """ + Start named server + """ + jupyterhub_full(start_server, **kwargs) + + +@server.command("stop") +@common_hub_params +@server_name +def stop_user_server(**kwargs): + """ + Stop named server + """ + jupyterhub_full(stop_server, **kwargs) + + +@jupyterhub.group() +def token(): + """ + JupyterHub token subcommand + """ + + +@token.command("add") +@click.option( + "--expiration", + type=int, + help="API token duration before it expires in seconds, 0 or ommitting means no expiration", +) +@click.option( + "--note", + help="API token description note for a new token", +) +@click.option( + "--role", + "-r", + multiple=True, + help=""" + Scopes from a role for new token, can be specified multiple times, e.g. -r user -r admin + Cannot be specified together with --scope options""", +) +@click.option( + "--scope", + "-s", + multiple=True, + help=""" + Scope for new token, can be specified multiple times, e.g. -s access:servers -s inherit + Cannot be specified together with --role options""", +) +@common_hub_params +def generate_api_token(**kwargs): + """ + Generate API token + """ + jupyterhub_full(add_token, **kwargs) + + +@token.command("list") +@common_hub_params +def list_api_tokens(**kwargs): + """ + Lists existing API token + """ + jupyterhub_full(list_tokens, **kwargs) + + +@token.command("show") +@common_hub_params +@api_token_id +def show_api_token(**kwargs): + """ + Shows existing API token + """ + jupyterhub_full(get_token, **kwargs) + + +@token.command("rm") +@common_hub_params +@api_token_id +def delete_api_token(**kwargs): + """ + Deletes existing API token + """ + jupyterhub_full(delete_token, **kwargs) + + +@jupyterhub.group() +def sharing(): + """ + JupyterHub user share-access subcommand + """ + + +@sharing.command("add") +@click.option( + "--scope", + "-s", + multiple=True, + help=""" + Scope for granted access, can be specified multiple times, e.g. -s access:servers -s inherit + If no scopes are specified, access:servers!server=:username/:servername is going to be used. + """, +) +@click.option( + "--grant-to-user", + help="""ID of user to grant a shared access to the specified server, only one at the time, + cannot be specified when --grant-to-group is used""", +) +@click.option( + "--grant-to-group", + help="""ID of group to grant a shared access to the specified server, only one at the time, + cannot be specified when --grant-to-user is used""", +) +@server_name +@common_hub_params +def add_sharing(**kwargs): + """ + Add server share + """ + jupyterhub_full(add_shared_access, **kwargs) + + +@sharing.command("rm") +@click.option( + "--all", + is_flag=True, + flag_value=True, + default=False, + help="If specified, all shared access of all invitied users will be removed, regardless of specified scopes", +) +@click.option( + "--remove-from-user", + help="ID of user to remove a shared access from to the specified server, only one at the time", +) +@click.option( + "--remove-from-group", + help="ID of group to grant a shared access from to the specified server, only one at the time", +) +@click.option( + "--scope", + "-s", + multiple=True, + help=""" + Scopes, can be specified multiple times, e.g. -s access:servers -s inherit + If no scopes are specified, all scopes will be removed""", +) +@server_name +@common_hub_params +def remove_sharing(**kwargs): + """ + Remove server share + """ + jupyterhub_full(remove_shared_access, **kwargs) + + +@sharing.command("list") +@server_name +@common_hub_params +def list_sharing(**kwargs): + """ + List server shares + """ + jupyterhub_full(list_shared_access, **kwargs) + + +@jupyterhub.group() +def path(): + """ + Jupyterhub path subcommand + """ + + +@path.command("show") +@click.option( + "--path", + "-p", + required=True, + type=click.Path(readable=False), + help="Path to the file or directory to show", +) +@click.option( + "--show-content", + is_flag=True, + flag_value=True, + default=False, + help="If specified, path contents are displayed", +) +@server_name +@common_hub_params +def path_show(**kwargs): + """ + Lists file/directory contents on specified path + """ + jupyterhub_full(get_path, **kwargs) + + +@path.command("add") +@click.option( + "--destination", + "-d", + required=True, + type=click.Path( + readable=False, + ), + help="Path where new file/directory will be created on the running server", +) +@click.option( + "--name", + "-n", + type=click.Path( + readable=False, + ), + help="Name of the new file/directory to create, if not specified, default is used", +) +@click.option( + "--copy-from", + type=click.Path( + readable=False, + ), + help="Path on the server to copy content to the path on the running server", +) +@click.option( + "--type", + required=True, + type=click.Choice(["file", "directory"]), + help="Type of the item to create", +) +@server_name +@common_hub_params +def path_add(**kwargs): + """ + Add file or directory at the specified path on running server + """ + jupyterhub_full(add_path, **kwargs) + + +@path.command("rm") +@click.option( + "--path", + "-p", + required=True, + type=click.Path( + readable=False, + ), + help="Path to file to delete", +) +@server_name +@common_hub_params +def path_remove(**kwargs): + """ + Remove file or directory at the specified path on running server, + directory must be empty + """ + jupyterhub_full(delete_path, **kwargs) + + +@jupyterhub.group() +def file(): + """ + Jupyterhub file subcommand + """ + + +@file.command("add") +@click.option( + "--file", + "-f", + required=True, + type=click.Path(exists=True, dir_okay=False), + help="Path to the file to be uploaded on the running server", +) +@click.option( + "--destination", + "-d", + required=True, + type=click.Path(readable=False), + help="""Path to the file to be uploaded on the running server. + Important!, the root dir is the root dir used by Jupyter server process!!, + If the destination is file, it will be overwritten""", +) +@server_name +@common_hub_params +def file_add(**kwargs): + """ + Uploads and/or overwrites file at the specified path on running server + """ + jupyterhub_full(upload_file, **kwargs) + + +@jupyterhub.command("exec") +@click.option( + "--output", + "-o", + default="text", + type=click.Choice(["json", "text"]), + help="Type out the output to provide", +) +@click.argument("command", nargs=-1) +@server_name +@common_hub_params +def execute(**kwargs): + """ + JupyterHub exec subcommand + """ + jupyterhub_full(exec_command, **kwargs) diff --git a/fedcloudclient/jupyterhub_lib.py b/fedcloudclient/jupyterhub_lib.py new file mode 100644 index 0000000..6d4786a --- /dev/null +++ b/fedcloudclient/jupyterhub_lib.py @@ -0,0 +1,523 @@ +""" +JupyterHub adapter library to make requests +""" + +import os +import json +import base64 +import urllib + +import requests + + +def _make_request(**kwargs): + """ + Sends requests to Hub API endpoint + """ + if not kwargs["hub_api_endpoint"].endswith("/"): + kwargs["hub_api_endpoint"] = kwargs["hub_api_endpoint"] + "/" + + headers = {} + data = None + params = None + + if "data" in kwargs: + headers.update({"Content-Type": "application/json; charset=utf-8"}) + data = json.dumps(kwargs.pop("data")) + if "params" in kwargs: + params = kwargs.pop("params") + headers.update({"Authorization": f"Bearer {kwargs['token']}"}) + + response = getattr(requests, kwargs["method"])( + urllib.parse.urljoin( + kwargs["hub_api_endpoint"], kwargs["api_request_endpoint"] + ), + headers=headers, + params=params, + data=data, + ) + if not response.ok: + try: + decoded_response = response.json() + print( + f"Error - Hub responded with: {response.status_code} - {decoded_response['message']}" + ) + except (requests.JSONDecodeError, KeyError): + print( + f"Error - Hub responded with: {response.status_code} - {response.reason}" + ) + return None + return response + + +def _decode_response(response): + """ + Decodes JSON response from Hub API + """ + try: + return response.json() + except requests.JSONDecodeError: + print("Failed to decode Hub response") + return None + + +def _get_user_id(**kwargs): + """ + Gets user ID + """ + user_output = get_user(**kwargs) + + if user_output is not None: + try: + return user_output["name"] + except KeyError: + print("Failed to acquire user ID from Hub response") + return None + + +def _is_server_running(server_name, user_output): + """ + Verifies if server is running + """ + try: + if not user_output["servers"][server_name]["ready"]: + print(f'Error - "{server_name}" server is not running') + return False + except KeyError: + print("Error - Failed to decode Hub response") + return False + return True + + +def _get_full_user_output(**kwargs): + """ + Return full user output + """ + # Stopped servers are required to get all of them + kwargs.update({"include_stopped_servers": True}) + return get_user(**kwargs) + + +def _get_server_url(hub_api_endpoint, server_name, user_output): + """ + Returns server url + """ + try: + if server_name not in user_output["servers"]: + print("Error - Specified server does not exist") + return None + if "url" not in user_output["servers"][server_name]: + print(f'Error - "{server_name}" server does not have specified it\'s url') + return None + except KeyError: + print("Error - Failed to decode Hub response") + return None + + server_url_path = user_output["servers"][server_name]["url"] + + if not server_url_path.endswith("/"): + server_url_path = server_url_path + "/" + # Required for urllib joining to work correctly + if server_url_path.startswith("/"): + server_url_path = server_url_path[1:] + + hub_address = urllib.parse.urlsplit(hub_api_endpoint) + hub_address = urllib.parse.urlunsplit((hub_address[0], hub_address[1], "", "", "")) + + server_url = urllib.parse.urljoin(hub_address, server_url_path) + + return server_url + + +def get_user(**kwargs): + """ + Gets specified user output + """ + api_request_endpoint = "" + + # If specific user ID is provided, the module will request + # details for that user ID instead of token owner + if kwargs.get("user", None) is not None: + api_request_endpoint = f"users/{kwargs['user']}" + else: + api_request_endpoint = "user" + kwargs.update({"method": "get", "api_request_endpoint": api_request_endpoint}) + + kwargs.update( + { + "params": { + "include_stopped_servers": kwargs.get("include_stopped_servers", False) + } + } + ) + response = _make_request(**kwargs) + + if response is not None: + return _decode_response(response) + return None + + +def _server_start_stop(**kwargs): + """ + Starts or stops single user server + """ + user_id = _get_user_id(**kwargs) + + if "options" in kwargs and kwargs["options"] is not None: + try: + user_options = json.loads(kwargs["options"]) + kwargs.update({"data": user_options}) + except json.JSONDecodeError: + print("Error - passed user options json is invalid.") + return None + + if user_id is not None: + kwargs.update( + {"api_request_endpoint": f"users/{user_id}/servers/{kwargs['server']}"} + ) + _make_request(**kwargs) + return None + + +def get_servers(**kwargs): + """ + Lists named servers + """ + user_output = get_user(**kwargs) + if user_output is not None: + return user_output["servers"] + return None + + +def start_server(**kwargs): + """ + Starts named server + """ + kwargs.update({"method": "post"}) + _server_start_stop(**kwargs) + + +def stop_server(**kwargs): + """ + Stops names server + """ + kwargs.update({"method": "delete"}) + _server_start_stop(**kwargs) + + +def _shares_request(**kwargs): + """ + Sends requests to shares API endpoint + """ + if kwargs["method"] in ["post", "patch"]: + data = {} + data_items = [ + ("grant_to_group", "group"), + ("grant_to_user", "user"), + ("remove_from_user", "user"), + ("remove_from_group", "group"), + ("scope", "scopes"), + ] + for kwargs_key, api_item_key in data_items: + if kwargs.get(kwargs_key, None) is not None: + data.update({api_item_key: kwargs[kwargs_key]}) + if "user" not in data and "group" not in data: + print( + "Error - Either one user or group can be specified, not at the same time" + ) + return None + if "user" in data and "group" in data: + print("Error - User or group cannot be specified at the same time") + return None + + kwargs.update({"data": data}) + + user_id = ( + kwargs["user"] + if kwargs.get("user", None) is not None + else _get_user_id(**kwargs) + ) + + if user_id is not None: + api_request_endpoint = f"shares/{user_id}/{kwargs['server']}" + kwargs.update({"api_request_endpoint": api_request_endpoint}) + + response = _make_request(**kwargs) + # Response output is returned apart from token deletion + if response is not None and kwargs["method"] != "delete": + return _decode_response(response) + return None + + +def add_shared_access(**kwargs): + """ + Adds shared access to the user server + """ + kwargs.update({"method": "post"}) + return _shares_request(**kwargs) + + +def remove_shared_access(**kwargs): + """ + Removes shared access to the user server + """ + if kwargs.get("all", False): + kwargs.update({"method": "delete"}) + else: + kwargs.update({"method": "patch"}) + return _shares_request(**kwargs) + + +def list_shared_access(**kwargs): + """ + Lists shared access of the user server + """ + kwargs.update({"method": "get"}) + return _shares_request(**kwargs) + + +def _token_request(**kwargs): + """ + Sends requests to token API endpoint + """ + # This is for token generation + if kwargs["method"] == "post": + data = {} + data_items = [ + ("expiration", "expires_in"), + ("note", "note"), + ("role", "roles"), + ("scope", "scopes"), + ] + for kwargs_key, api_item_key in data_items: + kwargs_item = kwargs.get(kwargs_key, None) + + if kwargs_item is not None and kwargs_item != (): + data.update({api_item_key: kwargs[kwargs_key]}) + if "roles" in data and "scopes" in data: + print("Error - roles and scopes cannot be specified both at the same time") + return None + kwargs.update({"data": data}) + user_id = ( + kwargs["user"] + if kwargs.get("user", None) is not None + else _get_user_id(**kwargs) + ) + + if user_id is not None: + if "api_token_id" in kwargs: + api_request_endpoint = f"users/{user_id}/tokens/{kwargs['api_token_id']}" + else: + api_request_endpoint = f"users/{user_id}/tokens" + + kwargs.update({"api_request_endpoint": api_request_endpoint}) + response = _make_request(**kwargs) + + # Response output is returned apart from token deletion + if response is not None and kwargs["method"] != "delete": + return _decode_response(response) + return None + + +def get_token(**kwargs): + """ + Gets token details by token ID + """ + kwargs.update({"method": "get"}) + return _token_request(**kwargs) + + +def delete_token(**kwargs): + """ + Delete specified API token + """ + kwargs.update({"method": "delete"}) + _token_request(**kwargs) + + +def list_tokens(**kwargs): + """ + Lists existing API tokens + """ + kwargs.update({"method": "get"}) + return _token_request(**kwargs) + + +def add_token(**kwargs): + """ + Generates API token + """ + kwargs.update({"method": "post"}) + return _token_request(**kwargs) + + +def _path_request(**kwargs): + """ + Sends requests to path jupyter server API + """ + user_output = _get_full_user_output(**kwargs) + if user_output is None: + return None + + server_url = _get_server_url( + kwargs["hub_api_endpoint"], kwargs["server"], user_output + ) + if not server_url: + return None + + # Switching Hub API url for user server API url + kwargs.update({"hub_api_endpoint": urllib.parse.urljoin(server_url, "api")}) + + content_path = "" + # PUT method indicates file upload request + if kwargs["method"] == "put": + content_path = kwargs["data"]["path"] + else: + # Because get_path is called from upload_file, 'destination' + # is used, on other cases 'path' param + content_path = kwargs["path"] if "path" in kwargs else kwargs["destination"] + + if content_path.startswith("/"): + content_path = content_path[1:] + kwargs.update({"api_request_endpoint": f"contents/{content_path}"}) + response = _make_request(**kwargs) + + if response is not None and kwargs["method"] != "delete": + return _decode_response(response) + return None + + +def get_path(**kwargs): + """ + Gets path on the running server + """ + kwargs.update({"method": "get"}) + kwargs.update( + {"params": {"content": 1 if kwargs.get("show_content", False) else 0}} + ) + return _path_request(**kwargs) + + +def add_path(**kwargs): + """ + Adds path on the running server + """ + data = {} + + if "copy_from" in kwargs: + data.update({"type": kwargs["type"], "copy_from": kwargs["copy_from"]}) + + kwargs.update({"method": "post"}) + data.update({"type": kwargs["type"]}) + kwargs.update({"data": data}) + add_path_response = _path_request(**kwargs) + + if add_path_response is None: + return None + + if "name" in kwargs and kwargs["name"] is not None: + kwargs.update({"method": "patch"}) + # New destination for API request and new destination + # with new file/directory name must be set for renaming + kwargs.update({"destination": add_path_response["path"]}) + kwargs.update( + { + "data": { + "path": os.path.join( + os.path.dirname(add_path_response["path"]), kwargs["name"] + ) + } + } + ) + return _path_request(**kwargs) + + return add_path_response + + +def delete_path(**kwargs): + """ + Deletes file/directory on the running server + """ + kwargs.update({"method": "delete"}) + return _path_request(**kwargs) + + +def upload_file(**kwargs): + """ + Uploads file to the server + """ + server_destination = get_path(**kwargs) + + if server_destination is None: + return None + + try: + if server_destination["type"] == "file": + jupyter_file_dest = kwargs["destination"] + + else: + jupyter_file_dest = os.path.join( + kwargs["destination"], os.path.basename(kwargs["file"]) + ) + + except KeyError: + print("Error - failed to decode Jupyterhub response") + return None + + data = {} + + try: + with open(kwargs["file"], "rb") as f: + content = base64.standard_b64encode(f.read()).decode("utf-8") + data.update( + { + "content": content, + "format": "base64", + "name": os.path.basename(jupyter_file_dest), + "type": "file", + "path": jupyter_file_dest, + } + ) + + except IOError: + print("Error - Failed to read specified file") + return None + + # Setting new data + kwargs.update({"data": data}) + kwargs.update({"method": "put"}) + + return _path_request(**kwargs) + + +def exec_command(**kwargs): + """ + Sends command to extension endpoint for execution + """ + kwargs.update({"method": "post"}) + kwargs.update({"api_request_endpoint": "jlab-control/exec"}) + + user_output = _get_full_user_output(**kwargs) + if user_output is None: + return None + + server_url = _get_server_url( + kwargs["hub_api_endpoint"], kwargs["server"], user_output + ) + + # Switching Hub API url for user server url + kwargs.update({"hub_api_endpoint": server_url}) + + if len(kwargs["command"]) > 0: + kwargs.update({"data": {"command": kwargs["command"]}}) + response = _make_request(**kwargs) + + if response is not None: + decoded_response = _decode_response(response) + if decoded_response is not None: + if kwargs["output"] == "text": + return decoded_response["output"] + return decoded_response + return None + print("No command provided, nothing to do") + return None