diff --git a/Dockerfile b/Dockerfile
index ef8779d..b81e9cc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -92,7 +92,7 @@ RUN \
COPY resources/mlhubspawner /mlhubspawner
RUN \
- pip install --no-cache dockerspawner && \
+ pip install --no-cache git+https://github.com/jupyterhub/dockerspawner@d1f27e2855d2cefbdb25b29cc069b9ca69d564e3 && \
pip install --no-cache git+https://github.com/ml-tooling/nativeauthenticator@983b203069ca797ff5c595f985075c11ae17656c && \
pip install --no-cache git+https://github.com/ryanlovett/imagespawner && \
pip install --no-cache /mlhubspawner && \
@@ -119,6 +119,9 @@ RUN PYCURL_SSL_LIBRARY=openssl pip3 install --no-cache-dir \
# Cleanup
clean-layer.sh
+RUN pip3 install oauthenticator psutil
+RUN apt-get update && apt-get install -y pcregrep && clean-layer.sh
+
### END INCUBATION ZONE ###
### CONFIGURATION ###
@@ -150,7 +153,8 @@ ENV \
START_SSH=true \
START_JHUB=true \
START_CHP=false \
- EXECUTION_MODE="local"
+ EXECUTION_MODE="local" \
+ HUB_NAME="mlhub"
### END CONFIGURATION ###
@@ -206,3 +210,4 @@ CMD ["/bin/bash", "/resources/docker-entrypoint.sh"]
# The port on which nginx listens and checks whether it's http(s) or ssh traffic
EXPOSE 8080
+
\ No newline at end of file
diff --git a/README.md b/README.md
index 3d4c370..e4804d2 100644
--- a/README.md
+++ b/README.md
@@ -49,14 +49,13 @@ Most parts will be identical to the configuration of Jupyterhub 1.0.0. One of th
```bash
docker run \
-p 8080 \
- --name mlhub \
-v /var/run/docker.sock:/var/run/docker.sock \
-v jupyterhub_data:/data \
mltooling/ml-hub:latest
```
To persist the hub data, such as started workspaces and created users, mount a directory to `/data`.
-A name (`--name`) must be set for the mlhub container, since we let the workspace container connect to the hub not via its docker id but its docker name. This way, the workspaces can still connect to the hub in case it was deleted and re-created (for example when the hub was updated).
+Any given name (`--name`) will be overruled by the environment variable `HUB_NAME`.
For Kubernetes deployment, we forked and modified [zero-to-jupyterhub-k8s](https://github.com/jupyterhub/zero-to-jupyterhub-k8s) which you can find [here](https://github.com/ml-tooling/zero-to-mlhub-k8s).
@@ -74,6 +73,14 @@ Here are the additional environment variables for the hub:
Description
Default
+
+
HUB_NAME
+
In Docker-local mode, the container will be (re-)named based on the value of this environment variable. All resources created by the hub will take this name into account. Hence, you can have multiple hub instances running without any naming conflicts.
+ Further, we let the workspace containers connect to the hub not via its docker id but its docker name. This way, the workspaces can still connect to the hub in case it was deleted and re-created (for example when the hub was updated).
+ The value must be DNS compliant and must be between 1 and 5 characters long.
+
+
mlhub
+
SSL_ENABLED
Enable SSL. If you don't provide an ssl certificate as described in Section "Enable SSL/HTTPS", certificates will be generated automatically. As this auto-generated certificate is not signed, you have to trust it in the browser. Without ssl enabled, ssh access won't work as the container uses a single port and has to tell https and ssh traffic apart.
diff --git a/resources/docker-entrypoint.sh b/resources/docker-entrypoint.sh
index ff21fa8..397aefc 100755
--- a/resources/docker-entrypoint.sh
+++ b/resources/docker-entrypoint.sh
@@ -14,25 +14,16 @@ if [ "$execution_mode" == "k8s" ]; then
# Preserve Kubernetes-specific environment variables for sshd process
echo "export KUBERNETES_SERVICE_HOST=$KUBERNETES_SERVICE_HOST" >> $SSHD_ENVIRONMENT_VARIABLES
echo "export KUBERNETES_SERVICE_PORT=$KUBERNETES_SERVICE_PORT" >> $SSHD_ENVIRONMENT_VARIABLES
-fi
-
-# It is possible to override the default sshd target with this command,
-# e.g. if it runs in a different container
-if [ ! -z "${SSHD_TARGET}" ]; then
- sed -i "s/127.0.0.1:22/${SSHD_TARGET}/g" /etc/nginx/nginx.conf
+else
+ if ! echo $HUB_NAME | pcregrep "^(?![0-9]+$)(?!-)[a-zA-Z0-9-]{1,5}(? /dev/null; then
+ echo "Container name for ml-hub is either too long or not DNS-compatible. Make sure that a DNS-compatible name (--env HUB_NAME) with 1 to 5 characters is provided for the ml-hub container."
+ exit 1
+ fi
fi
# create / copy certificates
$_RESOURCES_PATH/scripts/setup_certs.sh
-if [ "${START_NGINX}" == true ]; then
- # Configure and start nginx
- # TODO: restart nginx
- # TODO: make dependent on Kubernetes mode
-
- python $_RESOURCES_PATH/scripts/run_nginx.py
-fi
-
function start_ssh {
echo "Start SSH Daemon service"
# Run ssh-bastion image entrypoint
@@ -62,6 +53,19 @@ if [ "${START_CHP}" == true ]; then
start_http_proxy
fi
+if [ "${START_NGINX}" == true ]; then
+ # It is possible to override the default sshd target with this command,
+ # e.g. if it runs in a different container
+ if [ ! -z "${SSHD_TARGET}" ]; then
+ sed -i "s/127.0.0.1:22/${SSHD_TARGET}/g" /etc/nginx/nginx.conf
+ fi
+ # Configure and start nginx
+ # TODO: restart nginx
+ # TODO: make dependent on Kubernetes mode
+
+ python $_RESOURCES_PATH/scripts/run_nginx.py
+fi
+
# Copied from: https://docs.docker.com/config/containers/multi-service_container/
# Naive check runs checks once a minute to see if either of the processes exited.
# This illustrates part of the heavy lifting you need to do if you want to run
diff --git a/resources/jupyterhub-mod/template-home.html b/resources/jupyterhub-mod/template-home.html
index 248c9af..82d28f8 100644
--- a/resources/jupyterhub-mod/template-home.html
+++ b/resources/jupyterhub-mod/template-home.html
@@ -17,6 +17,15 @@
{% if not default_server.active %}Start{% endif %}
My Server
+ {% if default_server.is_update_available is defined %}
+ {% if default_server.is_update_available() %}
+
+ Update Workspace
+
+ {% endif %}
+ {% endif %}
{% if allow_named_servers %}
diff --git a/resources/jupyterhub_config.py b/resources/jupyterhub_config.py
index 1074eba..778e61f 100644
--- a/resources/jupyterhub_config.py
+++ b/resources/jupyterhub_config.py
@@ -3,9 +3,45 @@
"""
import os
+import socket
+
+from mlhubspawner import utils
+from subprocess import call
c = get_config()
+# Override the Jupyterhub `normalize_username` function to remove problematic characters from the username - independent from the used authenticator.
+# E.g. when the username is "lastname, firstname" and the comma and whitespace are not removed, they are encoded by the browser, which can lead to broken routing in our nginx proxy,
+# especially for the tools-part.
+# Everybody who starts the hub can override this behavior the same way we do in a mounted `jupyterhub_user_config.py` (Docker local) or via the `hub.extraConfig` (Kubernetes)
+from jupyterhub.auth import Authenticator
+original_normalize_username = Authenticator.normalize_username
+def custom_normalize_username(self, username):
+ username = original_normalize_username(self, username)
+ for forbidden_username_char in [" ", ",", ";", "."]:
+ username = username.replace(forbidden_username_char, "")
+ return username
+Authenticator.normalize_username = custom_normalize_username
+
+### Helper Functions ###
+
+def get_or_init(config: object, config_type: type) -> object:
+ if not isinstance(config, config_type):
+ return config_type()
+ return config
+
+def combine_config_dicts(*configs) -> dict:
+ combined_config = {}
+ for config in configs:
+ if not isinstance(config, dict):
+ config = {}
+ combined_config.update(config)
+ return combined_config
+
+### END HELPER FUNCTIONS###
+
+ENV_HUB_NAME = os.environ['HUB_NAME']
+
# User containers will access hub by container name on the Docker network
c.JupyterHub.hub_ip = '0.0.0.0' #'research-hub'
c.JupyterHub.port = 8000
@@ -34,8 +70,8 @@
# --- Spawner-specific ----
c.JupyterHub.spawner_class = 'mlhubspawner.MLHubDockerSpawner' # override in your config if you want to have a different spawner. If it is the or inherits from DockerSpawner, the c.DockerSpawner config can have an effect.
-c.Spawner.image = "mltooling/ml-workspace:0.8.6"
-c.Spawner.workspace_images = [c.Spawner.image, "mltooling/ml-workspace-gpu:0.8.6", "mltooling/ml-workspace-r:0.8.6"]
+c.Spawner.image = "mltooling/ml-workspace:0.8.7"
+c.Spawner.workspace_images = [c.Spawner.image, "mltooling/ml-workspace-gpu:0.8.7", "mltooling/ml-workspace-r:0.8.7", "mltooling/ml-workspace-spark:0.8.7"]
c.Spawner.notebook_dir = '/workspace'
# Connect containers to this Docker network
@@ -43,7 +79,7 @@
c.Spawner.extra_host_config = { 'shm_size': '256m' }
c.Spawner.prefix = 'ws'
-c.Spawner.name_template = c.Spawner.prefix + '-{username}-hub{servername}' # override in your config when you want to have a different name schema. Also consider changing c.Authenticator.username_pattern and check the environment variables to permit ssh connection
+c.Spawner.name_template = c.Spawner.prefix + '-{username}-' + ENV_HUB_NAME + '{servername}' # override in your config when you want to have a different name schema. Also consider changing c.Authenticator.username_pattern and check the environment variables to permit ssh connection
# Don't remove containers once they are stopped - persist state
c.Spawner.remove_containers = False
@@ -81,14 +117,23 @@
from z2jh import set_config_if_not_none
set_config_if_not_none(c.KubeSpawner, 'workspace_images', 'singleuser.workspaceImages')
- if not isinstance(c.KubeSpawner.environment, dict):
- c.KubeSpawner.environment = {}
+ c.KubeSpawner.environment = get_or_init(c.KubeSpawner.environment, dict)
+ # if not isinstance(c.KubeSpawner.environment, dict):
+ # c.KubeSpawner.environment = {}
c.KubeSpawner.environment.update(default_env)
+else:
+ client_kwargs = {**get_or_init(c.DockerSpawner.client_kwargs, dict), **get_or_init(c.MLHubDockerSpawner.client_kwargs, dict)}
+ tls_config = {**get_or_init(c.DockerSpawner.tls_config, dict), **get_or_init(c.MLHubDockerSpawner.tls_config, dict)}
+
+ docker_client = utils.init_docker_client(client_kwargs, tls_config)
+ docker_client.containers.list(filters={"id": socket.gethostname()})[0].rename(ENV_HUB_NAME)
+ c.MLHubDockerSpawner.hub_name = ENV_HUB_NAME
# Add nativeauthenticator-specific templates
if c.JupyterHub.authenticator_class == NATIVE_AUTHENTICATOR_CLASS:
import nativeauthenticator
# if template_paths is not set yet in user_config, it is of type traitlets.config.loader.LazyConfigValue; in other words, it was not initialized yet
- if not isinstance(c.JupyterHub.template_paths, list):
- c.JupyterHub.template_paths = []
+ c.JupyterHub.template_paths = get_or_init(c.JupyterHub.template_paths, list)
+ # if not isinstance(c.JupyterHub.template_paths, list):
+ # c.JupyterHub.template_paths = []
c.JupyterHub.template_paths.append("{}/templates/".format(os.path.dirname(nativeauthenticator.__file__)))
diff --git a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
index b17fede..8448eac 100644
--- a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
+++ b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
@@ -30,7 +30,7 @@ class MLHubKubernetesSpawner(KubeSpawner):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- self.hub_name = os.getenv("MLHUB_NAME", "mlhub")
+ self.hub_name = os.getenv("HUB_NAME", "mlhub")
self.default_label = {"origin": self.hub_name}
@default('options_form')
@@ -79,7 +79,7 @@ def start(self):
self.cpu_limit = float(self.user_options.get('cpu_limit'))
if self.user_options.get('mem_limit'):
- memory = str(self.user_options.get('mem_limit'))
+ memory = str(self.user_options.get('mem_limit')) + "G"
self.mem_limit = memory.upper().replace("GB", "G").replace("KB", "K").replace("MB", "M").replace("TB", "T")
#if self.user_options.get('is_mount_volume') == 'on':
diff --git a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
index fce9a3c..3f7ffd6 100644
--- a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
+++ b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
@@ -13,11 +13,12 @@
from docker.utils import kwargs_from_env
import os
+import subprocess
import socket
import ipaddress
from traitlets import default, Unicode, List
from tornado import gen
-import multiprocessing
+import psutil
import time
import re
@@ -46,6 +47,8 @@ def has_complete_network_information(network):
class MLHubDockerSpawner(DockerSpawner):
"""Provides the possibility to spawn docker containers with specific options, such as resource limits (CPU and Memory), Environment Variables, ..."""
+ hub_name = Unicode(config=True, help="Name of the hub container.")
+
workspace_images = List(
trait = Unicode(),
default_value = [],
@@ -59,12 +62,8 @@ def __init__(self, *args, **kwargs):
# Get the MLHub container name to be used as the DNS name for the spawned workspaces, so they can connect to the Hub even if the container is
# removed and restarted
client = self.highlevel_docker_client
- self.hub_name = client.containers.list(filters={"id": socket.gethostname()})[0].name # TODO: set default to mlhub?
self.default_label = {"origin": self.hub_name}
- if re.compile('^(?![0-9]+$)(?!-)[a-zA-Z0-9-]{,63}(? (str, int):
Returns:
(str, int): container's ip address or '127.0.0.1', container's port
"""
+
if self.user_options.get('image'):
self.image = self.user_options.get('image')
extra_host_config = {}
if self.user_options.get('cpu_limit'):
# nano_cpus cannot be bigger than the number of CPUs of the machine (this method would currently not work in a cluster, as machines could be different than the machine where the runtime-manager and this code run.
- max_available_cpus = multiprocessing.cpu_count()
+ max_available_cpus = self.resource_information.cpu_count
limited_cpus = min(
int(self.user_options.get('cpu_limit')), max_available_cpus)
@@ -162,8 +163,8 @@ def start(self) -> (str, int):
nano_cpus = int(limited_cpus * 1e9)
extra_host_config['nano_cpus'] = nano_cpus
if self.user_options.get('mem_limit'):
- extra_host_config['mem_limit'] = self.user_options.get(
- 'mem_limit')
+ extra_host_config['mem_limit'] = str(self.user_options.get(
+ 'mem_limit')) + "gb"
if self.user_options.get('is_mount_volume') == 'on':
# {username} and {servername} will be automatically replaced by DockerSpawner with the right values as in template_namespace
@@ -198,10 +199,13 @@ def start(self) -> (str, int):
# Delete existing container when it is created via the options_form UI (to make sure that not an existing container is re-used when you actually want to create a new one)
# reset the flag afterwards to prevent the container from being removed when just stopped
- if (hasattr(self, 'new_creating') and self.new_creating == True):
+ # Also make it deletable via the user_options (can be set via the POST API)
+ if ((hasattr(self, 'new_creating') and self.new_creating == True)
+ or self.user_options.get("update", False)):
self.remove = True
res = yield super().start()
self.remove = False
+ self.new_creating = False
return res
@gen.coroutine
@@ -295,6 +299,12 @@ def get_container_metadata(self) -> str:
def get_lifetime_timestamp(self, labels: dict) -> float:
return float(labels.get(utils.LABEL_EXPIRATION_TIMESTAMP, '0'))
+ def is_update_available(self):
+ try:
+ return self.image != self.highlevel_docker_client.containers.get(self.container_id).image.tags[0]
+ except (docker.errors.NotFound, docker.errors.NullResource):
+ return False
+
def get_labels(self) -> dict:
try:
return self.highlevel_docker_client.containers.get(self.container_id).labels
@@ -309,43 +319,26 @@ def template_namespace(self):
return template
- # NOTE: overwrite method to fix an issue with the image splitting.
- # We create a PR with the fix for Dockerspawner and, if fixed, we can
- # remove this one here again
- @gen.coroutine
- def pull_image(self, image):
- """Pull the image, if needed
- - pulls it unconditionally if pull_policy == 'always'
- - otherwise, checks if it exists, and
- - raises if pull_policy == 'never'
- - pulls if pull_policy == 'ifnotpresent'
- """
- # docker wants to split repo:tag
-
- # the part split("/")[-1] allows having an image from a custom repo
- # with port but without tag. For example: my.docker.repo:51150/foo would not
- # pass this test, resulting in image=my.docker.repo:51150/foo and tag=latest
- if ':' in image.split("/")[-1]:
- # rsplit splits from right to left, allowing to have a custom image repo with port
- repo, tag = image.rsplit(':', 1)
- else:
- repo = image
- tag = 'latest'
-
- if self.pull_policy.lower() == 'always':
- # always pull
- self.log.info("pulling %s", image)
- yield self.docker('pull', repo, tag)
- # done
- return
+ def get_gpu_info(self) -> list:
+ count_gpu = 0
try:
- # check if the image is present
- yield self.docker('inspect_image', image)
- except docker.errors.NotFound:
- if self.pull_policy == "never":
- # never pull, raise because there is no such image
- raise
- elif self.pull_policy == "ifnotpresent":
- # not present, pull it for the first time
- self.log.info("pulling image %s", image)
- yield self.docker('pull', repo, tag)
\ No newline at end of file
+ sp = subprocess.Popen(['nvidia-smi', '-q'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out_str = sp.communicate()
+ out_list = out_str[0].decode("utf-8").split('\n')
+
+ # out_dict = {}
+
+ for item in out_list:
+ try:
+ key, val = item.split(':')
+ key, val = key.strip(), val.strip()
+ if key == 'Product Name':
+ count_gpu += 1
+ # gpus.append(val)
+ #out_dict[key + "_" + str(count_gpu)] = val
+ except:
+ pass
+ except:
+ pass
+
+ return count_gpu
diff --git a/resources/mlhubspawner/mlhubspawner/spawner_options.py b/resources/mlhubspawner/mlhubspawner/spawner_options.py
index 77f350c..014942c 100644
--- a/resources/mlhubspawner/mlhubspawner/spawner_options.py
+++ b/resources/mlhubspawner/mlhubspawner/spawner_options.py
@@ -5,23 +5,27 @@
label_style = "width: 25%"
input_style = "width: 75%"
div_style = "margin-bottom: 16px"
+additional_info_style="margin-top: 4px; color: rgb(165,165,165); font-size: 12px;"
optional_label = "(optional)"
-def get_options_form(spawner):
+def get_options_form(spawner, additional_cpu_info="", additional_memory_info="", additional_gpu_info="") -> str:
"""Return the spawner options screen"""
# Only show spawner options for named servers (the default server should start with default values)
if getattr(spawner, "name", "") == "":
return ''
- description_memory_limit = 'Minimum limit must be 4mb as required by Docker.'
- description_env = 'In the form env=value (one per line)'
+ description_memory_limit = 'Memory Limit in GB.'
+ description_env = 'One name=value pair per line, without quotes'
description_days_to_live = 'Number of days the container should live'
default_image = getattr(spawner, "image", "mltooling/ml-workspace:latest")
# Show / hide custom image input field when checkbox is clicked
custom_image_listener = "if(event.target.checked){ $('#image-name').css('display', 'block'); $('.defined-images').css('display', 'none'); }else{ $('#image-name').css('display', 'none'); $('.defined-images').css('display', 'block'); }"
+
+ # Indicate a wrong input value (not a number) by changing the color to red
+ memory_input_listener = "if(isNaN(event.srcElement.value)){ $('#mem-limit').css('color', 'red'); }else{ $('#mem-limit').css('color', 'black'); }"
# Create drop down menu with pre-defined custom images
image_option_template = """
@@ -51,16 +55,18 @@ def get_options_form(spawner):
+
{additional_cpu_info}
-
-
+
+
+
{additional_memory_info}
-
+
+
{description_env}
-
@@ -69,30 +75,44 @@ def get_options_form(spawner):
div_style=div_style,
label_style=label_style,
input_style=input_style,
+ additional_info_style=additional_info_style,
default_image=default_image,
images_template=images_template,
custom_image_listener=custom_image_listener,
optional_label=optional_label,
description_memory_limit=description_memory_limit,
+ memory_input_listener=memory_input_listener,
description_env=description_env,
- description_days_to_live=description_days_to_live
+ description_days_to_live=description_days_to_live,
+ additional_cpu_info=additional_cpu_info,
+ additional_memory_info=additional_memory_info,
)
def get_options_form_docker(spawner):
- options_form = get_options_form(spawner)
-
+ description_gpus = 'Empty for no GPU, "all" for all GPUs, or a comma-separated list of indices of the GPUs (e.g 0,2).'
+ additional_info = {
+ "additional_cpu_info": "Number between 1 and {cpu_count}".format(cpu_count=spawner.resource_information['cpu_count']),
+ "additional_memory_info": "Number between 1 and {memory_count_in_gb}".format(memory_count_in_gb=spawner.resource_information['memory_count_in_gb']),
+ "additional_gpu_info": "
Available GPUs: {gpu_count}
{description_gpus}
".format(gpu_count=spawner.resource_information['gpu_count'], description_gpus=description_gpus)
+
+ }
+ options_form = get_options_form(spawner, **additional_info)
+
# When GPus shall be used, change the default image to the default gpu image (if the user entered a different image, it is not changed), and show an info box
# reminding the user of inserting a GPU-leveraging docker image
show_gpu_info_box = "$('#gpu-info-box').css('display', 'block');"
hide_gpu_info_box = "$('#gpu-info-box').css('display', 'none');"
- description_gpus = 'Empty for no GPU-acess. A comma-separted list of numbers describe the indices of the accessible GPUs.'
gpu_input_listener = "if(event.srcElement.value !== ''){{ {show_gpu_info_box} }}else{{ {hide_gpu_info_box} }}" \
.format(
show_gpu_info_box=show_gpu_info_box,
hide_gpu_info_box=hide_gpu_info_box
)
+ gpu_disabled = ""
+ if spawner.resource_information['gpu_count'] < 1:
+ gpu_disabled = "disabled"
+
options_form_docker = \
"""