From b50af989ebeb8fcc98cd45b0e8ba5e656dc4152e Mon Sep 17 00:00:00 2001 From: Lukas Masuch Date: Fri, 4 Oct 2019 14:17:07 +0200 Subject: [PATCH 1/9] Update workspace version to 0.8.7 and add spark workspace flavor --- resources/jupyterhub_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/jupyterhub_config.py b/resources/jupyterhub_config.py index 1074eba..36e6f2f 100644 --- a/resources/jupyterhub_config.py +++ b/resources/jupyterhub_config.py @@ -34,8 +34,8 @@ # --- Spawner-specific ---- c.JupyterHub.spawner_class = 'mlhubspawner.MLHubDockerSpawner' # override in your config if you want to have a different spawner. If it is the or inherits from DockerSpawner, the c.DockerSpawner config can have an effect. -c.Spawner.image = "mltooling/ml-workspace:0.8.6" -c.Spawner.workspace_images = [c.Spawner.image, "mltooling/ml-workspace-gpu:0.8.6", "mltooling/ml-workspace-r:0.8.6"] +c.Spawner.image = "mltooling/ml-workspace:0.8.7" +c.Spawner.workspace_images = [c.Spawner.image, "mltooling/ml-workspace-gpu:0.8.7", "mltooling/ml-workspace-r:0.8.7", "mltooling/ml-workspace-spark:0.8.7"] c.Spawner.notebook_dir = '/workspace' # Connect containers to this Docker network From f8107bdeffb27e08ddba6c2196f3376e10a621d6 Mon Sep 17 00:00:00 2001 From: Benjamin Raethlein Date: Thu, 10 Oct 2019 16:12:42 +0200 Subject: [PATCH 2/9] Install oauthenticator package to allow authenticators such as azure within the hub --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index ef8779d..b19d953 100644 --- a/Dockerfile +++ b/Dockerfile @@ -119,6 +119,8 @@ RUN PYCURL_SSL_LIBRARY=openssl pip3 install --no-cache-dir \ # Cleanup clean-layer.sh +RUN pip3 install oauthenticator + ### END INCUBATION ZONE ### ### CONFIGURATION ### @@ -206,3 +208,4 @@ CMD ["/bin/bash", "/resources/docker-entrypoint.sh"] # The port on which nginx listens and checks whether it's http(s) or ssh traffic EXPOSE 8080 + \ No newline at end of file From b34a4f2c2533a189da3613135bfb1afd6f262339 Mon Sep 17 00:00:00 2001 From: Benjamin Raethlein Date: Thu, 10 Oct 2019 16:13:38 +0200 Subject: [PATCH 3/9] Override the default Jupyterhub `normalize_username` function to remove characters that break our nginx (tool) routing --- resources/jupyterhub_config.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/resources/jupyterhub_config.py b/resources/jupyterhub_config.py index 36e6f2f..a572770 100644 --- a/resources/jupyterhub_config.py +++ b/resources/jupyterhub_config.py @@ -6,6 +6,19 @@ c = get_config() +# Override the Jupyterhub `normalize_username` function to remove problematic characters from the username - independent from the used authenticator. +# E.g. when the username is "lastname, firstname" and the comma and whitespace are not removed, they are encoded by the browser, which can lead to broken routing in our nginx proxy, +# especially for the tools-part. +# Everybody who starts the hub can override this behavior the same way we do in a mounted `jupyterhub_user_config.py` (Docker local) or via the `hub.extraConfig` (Kubernetes) +from jupyterhub.auth import Authenticator +original_normalize_username = Authenticator.normalize_username +def custom_normalize_username(self, username): + username = original_normalize_username(self, username) + for forbidden_username_char in [" ", ",", ";", "."]: + username = username.replace(forbidden_username_char, "") + return username +Authenticator.normalize_username = custom_normalize_username + # User containers will access hub by container name on the Docker network c.JupyterHub.hub_ip = '0.0.0.0' #'research-hub' c.JupyterHub.port = 8000 From c33e103ca5f8f10333d3f7a7bc687a9eb9a82b30 Mon Sep 17 00:00:00 2001 From: Benjamin Raethlein Date: Mon, 14 Oct 2019 19:35:19 +0200 Subject: [PATCH 4/9] Add button to Jupyterhub to update a workspace if there is a newer version (only applicable for default servers as named-servers could have been started via custom images) --- resources/jupyterhub-mod/template-home.html | 9 +++++++++ resources/mlhubspawner/mlhubspawner/mlhubspawner.py | 9 ++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/resources/jupyterhub-mod/template-home.html b/resources/jupyterhub-mod/template-home.html index 248c9af..82d28f8 100644 --- a/resources/jupyterhub-mod/template-home.html +++ b/resources/jupyterhub-mod/template-home.html @@ -17,6 +17,15 @@ {% if not default_server.active %}Start{% endif %} My Server + {% if default_server.is_update_available is defined %} + {% if default_server.is_update_available() %} + + Update Workspace + + {% endif %} + {% endif %} {% if allow_named_servers %} diff --git a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py index fce9a3c..eb057df 100644 --- a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py +++ b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py @@ -148,6 +148,7 @@ def start(self) -> (str, int): Returns: (str, int): container's ip address or '127.0.0.1', container's port """ + if self.user_options.get('image'): self.image = self.user_options.get('image') @@ -198,10 +199,13 @@ def start(self) -> (str, int): # Delete existing container when it is created via the options_form UI (to make sure that not an existing container is re-used when you actually want to create a new one) # reset the flag afterwards to prevent the container from being removed when just stopped - if (hasattr(self, 'new_creating') and self.new_creating == True): + # Also make it deletable via the user_options (can be set via the POST API) + if ((hasattr(self, 'new_creating') and self.new_creating == True) + or self.user_options.get("update", False)): self.remove = True res = yield super().start() self.remove = False + self.new_creating = False return res @gen.coroutine @@ -295,6 +299,9 @@ def get_container_metadata(self) -> str: def get_lifetime_timestamp(self, labels: dict) -> float: return float(labels.get(utils.LABEL_EXPIRATION_TIMESTAMP, '0')) + def is_update_available(self): + return self.image != self.highlevel_docker_client.containers.get(self.container_id).image.tags[0] + def get_labels(self) -> dict: try: return self.highlevel_docker_client.containers.get(self.container_id).labels From 733004c5cb37d50456a3d47f7f069c520c565694 Mon Sep 17 00:00:00 2001 From: Benjamin Raethlein Date: Wed, 16 Oct 2019 17:44:49 +0200 Subject: [PATCH 5/9] The hub container name can now be set via an environment variable. Created resources (in Docker local created networks, started containers, ...) are now dependent on this name which allows to have multiple instances of MLHub running --- Dockerfile | 4 +- README.md | 11 ++++- resources/docker-entrypoint.sh | 32 +++++++------- resources/jupyterhub_config.py | 42 ++++++++++++++++--- .../mlhubspawner/mlhubkubernetesspawner.py | 2 +- .../mlhubspawner/mlhubspawner/mlhubspawner.py | 21 ++++------ resources/mlhubspawner/mlhubspawner/utils.py | 19 +++++++++ resources/nginx.conf | 2 +- resources/scripts/run_nginx.py | 2 + 9 files changed, 98 insertions(+), 37 deletions(-) diff --git a/Dockerfile b/Dockerfile index b19d953..0e9deb5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -120,6 +120,7 @@ RUN PYCURL_SSL_LIBRARY=openssl pip3 install --no-cache-dir \ clean-layer.sh RUN pip3 install oauthenticator +RUN apt-get update && apt-get install -y pcregrep && clean-layer.sh ### END INCUBATION ZONE ### @@ -152,7 +153,8 @@ ENV \ START_SSH=true \ START_JHUB=true \ START_CHP=false \ - EXECUTION_MODE="local" + EXECUTION_MODE="local" \ + HUB_NAME="mlhub" ### END CONFIGURATION ### diff --git a/README.md b/README.md index 3d4c370..e4804d2 100644 --- a/README.md +++ b/README.md @@ -49,14 +49,13 @@ Most parts will be identical to the configuration of Jupyterhub 1.0.0. One of th ```bash docker run \ -p 8080 \ - --name mlhub \ -v /var/run/docker.sock:/var/run/docker.sock \ -v jupyterhub_data:/data \ mltooling/ml-hub:latest ``` To persist the hub data, such as started workspaces and created users, mount a directory to `/data`. -A name (`--name`) must be set for the mlhub container, since we let the workspace container connect to the hub not via its docker id but its docker name. This way, the workspaces can still connect to the hub in case it was deleted and re-created (for example when the hub was updated). +Any given name (`--name`) will be overruled by the environment variable `HUB_NAME`. For Kubernetes deployment, we forked and modified [zero-to-jupyterhub-k8s](https://github.com/jupyterhub/zero-to-jupyterhub-k8s) which you can find [here](https://github.com/ml-tooling/zero-to-mlhub-k8s). @@ -74,6 +73,14 @@ Here are the additional environment variables for the hub: Description Default + + HUB_NAME + In Docker-local mode, the container will be (re-)named based on the value of this environment variable. All resources created by the hub will take this name into account. Hence, you can have multiple hub instances running without any naming conflicts. + Further, we let the workspace containers connect to the hub not via its docker id but its docker name. This way, the workspaces can still connect to the hub in case it was deleted and re-created (for example when the hub was updated). + The value must be DNS compliant and must be between 1 and 5 characters long. + + mlhub + SSL_ENABLED Enable SSL. If you don't provide an ssl certificate as described in Section "Enable SSL/HTTPS", certificates will be generated automatically. As this auto-generated certificate is not signed, you have to trust it in the browser. Without ssl enabled, ssh access won't work as the container uses a single port and has to tell https and ssh traffic apart. diff --git a/resources/docker-entrypoint.sh b/resources/docker-entrypoint.sh index ff21fa8..397aefc 100755 --- a/resources/docker-entrypoint.sh +++ b/resources/docker-entrypoint.sh @@ -14,25 +14,16 @@ if [ "$execution_mode" == "k8s" ]; then # Preserve Kubernetes-specific environment variables for sshd process echo "export KUBERNETES_SERVICE_HOST=$KUBERNETES_SERVICE_HOST" >> $SSHD_ENVIRONMENT_VARIABLES echo "export KUBERNETES_SERVICE_PORT=$KUBERNETES_SERVICE_PORT" >> $SSHD_ENVIRONMENT_VARIABLES -fi - -# It is possible to override the default sshd target with this command, -# e.g. if it runs in a different container -if [ ! -z "${SSHD_TARGET}" ]; then - sed -i "s/127.0.0.1:22/${SSHD_TARGET}/g" /etc/nginx/nginx.conf +else + if ! echo $HUB_NAME | pcregrep "^(?![0-9]+$)(?!-)[a-zA-Z0-9-]{1,5}(? /dev/null; then + echo "Container name for ml-hub is either too long or not DNS-compatible. Make sure that a DNS-compatible name (--env HUB_NAME) with 1 to 5 characters is provided for the ml-hub container." + exit 1 + fi fi # create / copy certificates $_RESOURCES_PATH/scripts/setup_certs.sh -if [ "${START_NGINX}" == true ]; then - # Configure and start nginx - # TODO: restart nginx - # TODO: make dependent on Kubernetes mode - - python $_RESOURCES_PATH/scripts/run_nginx.py -fi - function start_ssh { echo "Start SSH Daemon service" # Run ssh-bastion image entrypoint @@ -62,6 +53,19 @@ if [ "${START_CHP}" == true ]; then start_http_proxy fi +if [ "${START_NGINX}" == true ]; then + # It is possible to override the default sshd target with this command, + # e.g. if it runs in a different container + if [ ! -z "${SSHD_TARGET}" ]; then + sed -i "s/127.0.0.1:22/${SSHD_TARGET}/g" /etc/nginx/nginx.conf + fi + # Configure and start nginx + # TODO: restart nginx + # TODO: make dependent on Kubernetes mode + + python $_RESOURCES_PATH/scripts/run_nginx.py +fi + # Copied from: https://docs.docker.com/config/containers/multi-service_container/ # Naive check runs checks once a minute to see if either of the processes exited. # This illustrates part of the heavy lifting you need to do if you want to run diff --git a/resources/jupyterhub_config.py b/resources/jupyterhub_config.py index a572770..778e61f 100644 --- a/resources/jupyterhub_config.py +++ b/resources/jupyterhub_config.py @@ -3,6 +3,10 @@ """ import os +import socket + +from mlhubspawner import utils +from subprocess import call c = get_config() @@ -19,6 +23,25 @@ def custom_normalize_username(self, username): return username Authenticator.normalize_username = custom_normalize_username +### Helper Functions ### + +def get_or_init(config: object, config_type: type) -> object: + if not isinstance(config, config_type): + return config_type() + return config + +def combine_config_dicts(*configs) -> dict: + combined_config = {} + for config in configs: + if not isinstance(config, dict): + config = {} + combined_config.update(config) + return combined_config + +### END HELPER FUNCTIONS### + +ENV_HUB_NAME = os.environ['HUB_NAME'] + # User containers will access hub by container name on the Docker network c.JupyterHub.hub_ip = '0.0.0.0' #'research-hub' c.JupyterHub.port = 8000 @@ -56,7 +79,7 @@ def custom_normalize_username(self, username): c.Spawner.extra_host_config = { 'shm_size': '256m' } c.Spawner.prefix = 'ws' -c.Spawner.name_template = c.Spawner.prefix + '-{username}-hub{servername}' # override in your config when you want to have a different name schema. Also consider changing c.Authenticator.username_pattern and check the environment variables to permit ssh connection +c.Spawner.name_template = c.Spawner.prefix + '-{username}-' + ENV_HUB_NAME + '{servername}' # override in your config when you want to have a different name schema. Also consider changing c.Authenticator.username_pattern and check the environment variables to permit ssh connection # Don't remove containers once they are stopped - persist state c.Spawner.remove_containers = False @@ -94,14 +117,23 @@ def custom_normalize_username(self, username): from z2jh import set_config_if_not_none set_config_if_not_none(c.KubeSpawner, 'workspace_images', 'singleuser.workspaceImages') - if not isinstance(c.KubeSpawner.environment, dict): - c.KubeSpawner.environment = {} + c.KubeSpawner.environment = get_or_init(c.KubeSpawner.environment, dict) + # if not isinstance(c.KubeSpawner.environment, dict): + # c.KubeSpawner.environment = {} c.KubeSpawner.environment.update(default_env) +else: + client_kwargs = {**get_or_init(c.DockerSpawner.client_kwargs, dict), **get_or_init(c.MLHubDockerSpawner.client_kwargs, dict)} + tls_config = {**get_or_init(c.DockerSpawner.tls_config, dict), **get_or_init(c.MLHubDockerSpawner.tls_config, dict)} + + docker_client = utils.init_docker_client(client_kwargs, tls_config) + docker_client.containers.list(filters={"id": socket.gethostname()})[0].rename(ENV_HUB_NAME) + c.MLHubDockerSpawner.hub_name = ENV_HUB_NAME # Add nativeauthenticator-specific templates if c.JupyterHub.authenticator_class == NATIVE_AUTHENTICATOR_CLASS: import nativeauthenticator # if template_paths is not set yet in user_config, it is of type traitlets.config.loader.LazyConfigValue; in other words, it was not initialized yet - if not isinstance(c.JupyterHub.template_paths, list): - c.JupyterHub.template_paths = [] + c.JupyterHub.template_paths = get_or_init(c.JupyterHub.template_paths, list) + # if not isinstance(c.JupyterHub.template_paths, list): + # c.JupyterHub.template_paths = [] c.JupyterHub.template_paths.append("{}/templates/".format(os.path.dirname(nativeauthenticator.__file__))) diff --git a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py index b17fede..909d841 100644 --- a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py +++ b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py @@ -30,7 +30,7 @@ class MLHubKubernetesSpawner(KubeSpawner): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.hub_name = os.getenv("MLHUB_NAME", "mlhub") + self.hub_name = os.getenv("HUB_NAME", "mlhub") self.default_label = {"origin": self.hub_name} @default('options_form') diff --git a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py index eb057df..7a51d5e 100644 --- a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py +++ b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py @@ -46,6 +46,8 @@ def has_complete_network_information(network): class MLHubDockerSpawner(DockerSpawner): """Provides the possibility to spawn docker containers with specific options, such as resource limits (CPU and Memory), Environment Variables, ...""" + hub_name = Unicode(config=True, help="Name of the hub container.") + workspace_images = List( trait = Unicode(), default_value = [], @@ -59,12 +61,8 @@ def __init__(self, *args, **kwargs): # Get the MLHub container name to be used as the DNS name for the spawned workspaces, so they can connect to the Hub even if the container is # removed and restarted client = self.highlevel_docker_client - self.hub_name = client.containers.list(filters={"id": socket.gethostname()})[0].name # TODO: set default to mlhub? self.default_label = {"origin": self.hub_name} - if re.compile('^(?![0-9]+$)(?!-)[a-zA-Z0-9-]{,63}(? float: return float(labels.get(utils.LABEL_EXPIRATION_TIMESTAMP, '0')) def is_update_available(self): - return self.image != self.highlevel_docker_client.containers.get(self.container_id).image.tags[0] + try: + return self.image != self.highlevel_docker_client.containers.get(self.container_id).image.tags[0] + except (docker.errors.NotFound, docker.errors.NullResource): + return False def get_labels(self) -> dict: try: diff --git a/resources/mlhubspawner/mlhubspawner/utils.py b/resources/mlhubspawner/mlhubspawner/utils.py index 7b6e1be..75c2775 100644 --- a/resources/mlhubspawner/mlhubspawner/utils.py +++ b/resources/mlhubspawner/mlhubspawner/utils.py @@ -5,6 +5,9 @@ import math import time +import docker +from docker.utils import kwargs_from_env + LABEL_NVIDIA_VISIBLE_DEVICES = 'nvidia_visible_devices' LABEL_EXPIRATION_TIMESTAMP = 'expiration_timestamp_seconds' @@ -24,3 +27,19 @@ def get_container_metadata(spawner): return "" return "({})".format(", ".join(meta_information)) + +def init_docker_client(client_kwargs: dict, tls_config: dict) -> docker.DockerClient: + """Create a docker client. + The configuration is done the same way DockerSpawner initializes the low-level API client. + + Returns: + docker.DockerClient + """ + + kwargs = {"version": "auto"} + if tls_config: + kwargs["tls"] = docker.tls.TLSConfig(**tls_config) + kwargs.update(kwargs_from_env()) + if client_kwargs: + kwargs.update(client_kwargs) + return docker.DockerClient(**kwargs) diff --git a/resources/nginx.conf b/resources/nginx.conf index 2a53867..47efbc9 100755 --- a/resources/nginx.conf +++ b/resources/nginx.conf @@ -117,7 +117,7 @@ http { set $user_server "-${user_server}"; } - proxy_pass http://ws-$user-hub$user_server$service_suffix:{DEFAULT_WORKSPACE_PORT}$request_uri; + proxy_pass http://ws-$user-{HUB_NAME}$user_server$service_suffix:{DEFAULT_WORKSPACE_PORT}$request_uri; } } } diff --git a/resources/scripts/run_nginx.py b/resources/scripts/run_nginx.py index 0c156dd..6f2242d 100644 --- a/resources/scripts/run_nginx.py +++ b/resources/scripts/run_nginx.py @@ -10,6 +10,7 @@ ENV_RESOURCES_PATH = os.environ["_RESOURCES_PATH"] ENV_DEFAULT_WORKSPACE_PORT = os.environ["DEFAULT_WORKSPACE_PORT"] +ENV_HUB_NAME = os.environ.get("HUB_NAME", "hub") NGINX_FILE = "/etc/nginx/nginx.conf" @@ -37,6 +38,7 @@ call("sed -i 's/{UPSTREAM}/" + UPSTREAM + "/g' " + NGINX_FILE, shell=True) call("sed -i 's/{SSL}/" + SSL + "/g' " + NGINX_FILE, shell=True) call("sed -i 's@{DEFAULT_WORKSPACE_PORT}@" + ENV_DEFAULT_WORKSPACE_PORT + "@g' " + NGINX_FILE, shell=True) +call("sed -i 's@{HUB_NAME}@" + ENV_HUB_NAME + "@g' " + NGINX_FILE, shell=True) # start nginx print("Start nginx") From cae5c0f44b92957f9bd7392a236bdf0f7da71cdc Mon Sep 17 00:00:00 2001 From: Benjamin Raethlein Date: Wed, 16 Oct 2019 17:47:22 +0200 Subject: [PATCH 6/9] Add subjectAltName to self-generated certificate as it should be set --- resources/scripts/setup_certs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resources/scripts/setup_certs.sh b/resources/scripts/setup_certs.sh index d4dd749..b015d43 100755 --- a/resources/scripts/setup_certs.sh +++ b/resources/scripts/setup_certs.sh @@ -9,7 +9,8 @@ if [ ! -f ${_SSL_RESOURCES_PATH}/$SSLNAME.crt ]; then echo "Generate self-signed certificate for SSL/HTTPS." SSLDAYS=365 - openssl req -x509 -nodes -newkey rsa:2048 -keyout $SSLNAME.key -out $SSLNAME.crt -days $SSLDAYS -subj '/C=DE/ST=Berlin/L=Berlin/CN=localhost' > /dev/null 2>&1 + touch /root/.rnd + openssl req -x509 -nodes -newkey rsa:2048 -keyout $SSLNAME.key -out $SSLNAME.crt -days $SSLDAYS -subj '/C=DE/ST=Berlin/L=Berlin/CN=localhost' -reqexts SAN -extensions SAN -config <(cat /etc/ssl/openssl.cnf <(printf "\n[SAN]\nsubjectAltName=DNS:localhost,DNS:127.0.0.1\n")) > /dev/null 2>&1 mv $SSLNAME.crt ${_SSL_RESOURCES_PATH}/$SSLNAME.crt mv $SSLNAME.key ${_SSL_RESOURCES_PATH}/$SSLNAME.key From 3554e8075d7a16261b0d752fc6751c55ffce0cbc Mon Sep 17 00:00:00 2001 From: Benjamin Raethlein Date: Thu, 17 Oct 2019 13:35:33 +0200 Subject: [PATCH 7/9] Remove patch as our PR was merged into the original repository --- Dockerfile | 2 +- .../mlhubspawner/mlhubspawner/mlhubspawner.py | 41 ------------------- 2 files changed, 1 insertion(+), 42 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0e9deb5..8c1185c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -92,7 +92,7 @@ RUN \ COPY resources/mlhubspawner /mlhubspawner RUN \ - pip install --no-cache dockerspawner && \ + pip install --no-cache git+https://github.com/jupyterhub/dockerspawner@d1f27e2855d2cefbdb25b29cc069b9ca69d564e3 && \ pip install --no-cache git+https://github.com/ml-tooling/nativeauthenticator@983b203069ca797ff5c595f985075c11ae17656c && \ pip install --no-cache git+https://github.com/ryanlovett/imagespawner && \ pip install --no-cache /mlhubspawner && \ diff --git a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py index 7a51d5e..4fc0813 100644 --- a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py +++ b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py @@ -310,44 +310,3 @@ def template_namespace(self): template["servername"] = "-" + template["servername"] return template - - # NOTE: overwrite method to fix an issue with the image splitting. - # We create a PR with the fix for Dockerspawner and, if fixed, we can - # remove this one here again - @gen.coroutine - def pull_image(self, image): - """Pull the image, if needed - - pulls it unconditionally if pull_policy == 'always' - - otherwise, checks if it exists, and - - raises if pull_policy == 'never' - - pulls if pull_policy == 'ifnotpresent' - """ - # docker wants to split repo:tag - - # the part split("/")[-1] allows having an image from a custom repo - # with port but without tag. For example: my.docker.repo:51150/foo would not - # pass this test, resulting in image=my.docker.repo:51150/foo and tag=latest - if ':' in image.split("/")[-1]: - # rsplit splits from right to left, allowing to have a custom image repo with port - repo, tag = image.rsplit(':', 1) - else: - repo = image - tag = 'latest' - - if self.pull_policy.lower() == 'always': - # always pull - self.log.info("pulling %s", image) - yield self.docker('pull', repo, tag) - # done - return - try: - # check if the image is present - yield self.docker('inspect_image', image) - except docker.errors.NotFound: - if self.pull_policy == "never": - # never pull, raise because there is no such image - raise - elif self.pull_policy == "ifnotpresent": - # not present, pull it for the first time - self.log.info("pulling image %s", image) - yield self.docker('pull', repo, tag) \ No newline at end of file From dd0f5fc3a96b0ae8208b99862239708863fe0bdb Mon Sep 17 00:00:00 2001 From: Benjamin Raethlein Date: Thu, 17 Oct 2019 14:59:11 +0200 Subject: [PATCH 8/9] Show cpu and memory information in Docker-mode options form --- Dockerfile | 2 +- .../mlhubspawner/mlhubspawner/mlhubspawner.py | 10 +++++-- .../mlhubspawner/spawner_options.py | 26 ++++++++++++++----- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8c1185c..b81e9cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -119,7 +119,7 @@ RUN PYCURL_SSL_LIBRARY=openssl pip3 install --no-cache-dir \ # Cleanup clean-layer.sh -RUN pip3 install oauthenticator +RUN pip3 install oauthenticator psutil RUN apt-get update && apt-get install -y pcregrep && clean-layer.sh ### END INCUBATION ZONE ### diff --git a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py index 4fc0813..6420ef7 100644 --- a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py +++ b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py @@ -17,7 +17,7 @@ import ipaddress from traitlets import default, Unicode, List from tornado import gen -import multiprocessing +import psutil import time import re @@ -70,6 +70,12 @@ def __init__(self, *args, **kwargs): self.connect_hub_to_network(network) except: pass + + # Get available resource information + self.resource_information = { + "cpu_count": psutil.cpu_count(), + "memory_count_in_gb": round(psutil.virtual_memory().total/1024/1024/1024, 1) + } @property def highlevel_docker_client(self): @@ -147,7 +153,7 @@ def start(self) -> (str, int): extra_host_config = {} if self.user_options.get('cpu_limit'): # nano_cpus cannot be bigger than the number of CPUs of the machine (this method would currently not work in a cluster, as machines could be different than the machine where the runtime-manager and this code run. - max_available_cpus = multiprocessing.cpu_count() + max_available_cpus = self.resource_information.cpu_count limited_cpus = min( int(self.user_options.get('cpu_limit')), max_available_cpus) diff --git a/resources/mlhubspawner/mlhubspawner/spawner_options.py b/resources/mlhubspawner/mlhubspawner/spawner_options.py index 77f350c..644d241 100644 --- a/resources/mlhubspawner/mlhubspawner/spawner_options.py +++ b/resources/mlhubspawner/mlhubspawner/spawner_options.py @@ -5,9 +5,10 @@ label_style = "width: 25%" input_style = "width: 75%" div_style = "margin-bottom: 16px" +additional_info_style="margin-top: 4px; color: rgb(165,165,165); font-size: 12px;" optional_label = "(optional)" -def get_options_form(spawner): +def get_options_form(spawner, additional_cpu_info="", additional_memory_info="") -> str: """Return the spawner options screen""" # Only show spawner options for named servers (the default server should start with default values) @@ -15,7 +16,7 @@ def get_options_form(spawner): return '' description_memory_limit = 'Minimum limit must be 4mb as required by Docker.' - description_env = 'In the form env=value (one per line)' + description_env = 'One name=value pair per line, without quotes' description_days_to_live = 'Number of days the container should live' default_image = getattr(spawner, "image", "mltooling/ml-workspace:latest") @@ -51,14 +52,17 @@ def get_options_form(spawner):
+
{additional_cpu_info}
+
{additional_memory_info}
- + +
{description_env}
@@ -69,18 +73,26 @@ def get_options_form(spawner): div_style=div_style, label_style=label_style, input_style=input_style, + additional_info_style=additional_info_style, default_image=default_image, images_template=images_template, custom_image_listener=custom_image_listener, optional_label=optional_label, description_memory_limit=description_memory_limit, description_env=description_env, - description_days_to_live=description_days_to_live + description_days_to_live=description_days_to_live, + additional_cpu_info=additional_cpu_info, + additional_memory_info=additional_memory_info ) def get_options_form_docker(spawner): - options_form = get_options_form(spawner) - + print(spawner.resource_information) + additional_info = { + "additional_cpu_info": "Number between 1 and {cpu_count}".format(cpu_count=spawner.resource_information['cpu_count']), + "additional_memory_info": "Number between 200mb and {memory_count_in_gb}gb".format(memory_count_in_gb=spawner.resource_information['memory_count_in_gb']) + } + options_form = get_options_form(spawner, **additional_info) + # When GPus shall be used, change the default image to the default gpu image (if the user entered a different image, it is not changed), and show an info box # reminding the user of inserting a GPU-leveraging docker image @@ -92,7 +104,7 @@ def get_options_form_docker(spawner): show_gpu_info_box=show_gpu_info_box, hide_gpu_info_box=hide_gpu_info_box ) - + options_form_docker = \ """
From 48d9999c06534626ff9c5b6270e861122172c35d Mon Sep 17 00:00:00 2001 From: Benjamin Raethlein Date: Thu, 17 Oct 2019 17:29:13 +0200 Subject: [PATCH 9/9] - Add information about GPUs - Memory limit has to be given in GB now --- .../mlhubspawner/mlhubkubernetesspawner.py | 2 +- .../mlhubspawner/mlhubspawner/mlhubspawner.py | 32 +++++++++++++++-- .../mlhubspawner/spawner_options.py | 36 ++++++++++++------- 3 files changed, 54 insertions(+), 16 deletions(-) diff --git a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py index 909d841..8448eac 100644 --- a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py +++ b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py @@ -79,7 +79,7 @@ def start(self): self.cpu_limit = float(self.user_options.get('cpu_limit')) if self.user_options.get('mem_limit'): - memory = str(self.user_options.get('mem_limit')) + memory = str(self.user_options.get('mem_limit')) + "G" self.mem_limit = memory.upper().replace("GB", "G").replace("KB", "K").replace("MB", "M").replace("TB", "T") #if self.user_options.get('is_mount_volume') == 'on': diff --git a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py index 6420ef7..3f7ffd6 100644 --- a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py +++ b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py @@ -13,6 +13,7 @@ from docker.utils import kwargs_from_env import os +import subprocess import socket import ipaddress from traitlets import default, Unicode, List @@ -74,7 +75,8 @@ def __init__(self, *args, **kwargs): # Get available resource information self.resource_information = { "cpu_count": psutil.cpu_count(), - "memory_count_in_gb": round(psutil.virtual_memory().total/1024/1024/1024, 1) + "memory_count_in_gb": round(psutil.virtual_memory().total/1024/1024/1024, 1), + "gpu_count": self.get_gpu_info() } @property @@ -161,8 +163,8 @@ def start(self) -> (str, int): nano_cpus = int(limited_cpus * 1e9) extra_host_config['nano_cpus'] = nano_cpus if self.user_options.get('mem_limit'): - extra_host_config['mem_limit'] = self.user_options.get( - 'mem_limit') + extra_host_config['mem_limit'] = str(self.user_options.get( + 'mem_limit')) + "gb" if self.user_options.get('is_mount_volume') == 'on': # {username} and {servername} will be automatically replaced by DockerSpawner with the right values as in template_namespace @@ -316,3 +318,27 @@ def template_namespace(self): template["servername"] = "-" + template["servername"] return template + + def get_gpu_info(self) -> list: + count_gpu = 0 + try: + sp = subprocess.Popen(['nvidia-smi', '-q'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out_str = sp.communicate() + out_list = out_str[0].decode("utf-8").split('\n') + + # out_dict = {} + + for item in out_list: + try: + key, val = item.split(':') + key, val = key.strip(), val.strip() + if key == 'Product Name': + count_gpu += 1 + # gpus.append(val) + #out_dict[key + "_" + str(count_gpu)] = val + except: + pass + except: + pass + + return count_gpu diff --git a/resources/mlhubspawner/mlhubspawner/spawner_options.py b/resources/mlhubspawner/mlhubspawner/spawner_options.py index 644d241..014942c 100644 --- a/resources/mlhubspawner/mlhubspawner/spawner_options.py +++ b/resources/mlhubspawner/mlhubspawner/spawner_options.py @@ -8,14 +8,14 @@ additional_info_style="margin-top: 4px; color: rgb(165,165,165); font-size: 12px;" optional_label = "(optional)" -def get_options_form(spawner, additional_cpu_info="", additional_memory_info="") -> str: +def get_options_form(spawner, additional_cpu_info="", additional_memory_info="", additional_gpu_info="") -> str: """Return the spawner options screen""" # Only show spawner options for named servers (the default server should start with default values) if getattr(spawner, "name", "") == "": return '' - description_memory_limit = 'Minimum limit must be 4mb as required by Docker.' + description_memory_limit = 'Memory Limit in GB.' description_env = 'One name=value pair per line, without quotes' description_days_to_live = 'Number of days the container should live' @@ -23,6 +23,9 @@ def get_options_form(spawner, additional_cpu_info="", additional_memory_info="") # Show / hide custom image input field when checkbox is clicked custom_image_listener = "if(event.target.checked){ $('#image-name').css('display', 'block'); $('.defined-images').css('display', 'none'); }else{ $('#image-name').css('display', 'none'); $('.defined-images').css('display', 'block'); }" + + # Indicate a wrong input value (not a number) by changing the color to red + memory_input_listener = "if(isNaN(event.srcElement.value)){ $('#mem-limit').css('color', 'red'); }else{ $('#mem-limit').css('color', 'black'); }" # Create drop down menu with pre-defined custom images image_option_template = """ @@ -55,8 +58,8 @@ def get_options_form(spawner, additional_cpu_info="", additional_memory_info="")
{additional_cpu_info}
- - + +
{additional_memory_info}
@@ -64,7 +67,6 @@ def get_options_form(spawner, additional_cpu_info="", additional_memory_info="")
{description_env}
-
@@ -79,17 +81,20 @@ def get_options_form(spawner, additional_cpu_info="", additional_memory_info="") custom_image_listener=custom_image_listener, optional_label=optional_label, description_memory_limit=description_memory_limit, + memory_input_listener=memory_input_listener, description_env=description_env, description_days_to_live=description_days_to_live, additional_cpu_info=additional_cpu_info, - additional_memory_info=additional_memory_info + additional_memory_info=additional_memory_info, ) def get_options_form_docker(spawner): - print(spawner.resource_information) + description_gpus = 'Empty for no GPU, "all" for all GPUs, or a comma-separated list of indices of the GPUs (e.g 0,2).' additional_info = { "additional_cpu_info": "Number between 1 and {cpu_count}".format(cpu_count=spawner.resource_information['cpu_count']), - "additional_memory_info": "Number between 200mb and {memory_count_in_gb}gb".format(memory_count_in_gb=spawner.resource_information['memory_count_in_gb']) + "additional_memory_info": "Number between 1 and {memory_count_in_gb}".format(memory_count_in_gb=spawner.resource_information['memory_count_in_gb']), + "additional_gpu_info": "
Available GPUs: {gpu_count}
{description_gpus}
".format(gpu_count=spawner.resource_information['gpu_count'], description_gpus=description_gpus) + } options_form = get_options_form(spawner, **additional_info) @@ -98,13 +103,16 @@ def get_options_form_docker(spawner): # reminding the user of inserting a GPU-leveraging docker image show_gpu_info_box = "$('#gpu-info-box').css('display', 'block');" hide_gpu_info_box = "$('#gpu-info-box').css('display', 'none');" - description_gpus = 'Empty for no GPU-acess. A comma-separted list of numbers describe the indices of the accessible GPUs.' gpu_input_listener = "if(event.srcElement.value !== ''){{ {show_gpu_info_box} }}else{{ {hide_gpu_info_box} }}" \ .format( show_gpu_info_box=show_gpu_info_box, hide_gpu_info_box=hide_gpu_info_box ) - + + gpu_disabled = "" + if spawner.resource_information['gpu_count'] < 1: + gpu_disabled = "disabled" + options_form_docker = \ """
@@ -113,16 +121,20 @@ def get_options_form_docker(spawner):
- + +
{additional_gpu_info}
""".format( div_style=div_style, label_style=label_style, input_style=input_style, + additional_info_style=additional_info_style, optional_label=optional_label, gpu_input_listener=gpu_input_listener, - description_gpus=description_gpus + gpu_disabled=gpu_disabled, + description_gpus=description_gpus, + additional_gpu_info=additional_info['additional_gpu_info'] ) return options_form + options_form_docker