From 10eff9a8a90755507dca08c9dc4382aa69dadf5e Mon Sep 17 00:00:00 2001 From: Benjamin Raethlein Date: Wed, 30 Oct 2019 12:34:31 +0100 Subject: [PATCH 01/21] Update native authenticator version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b81e9cc..8e5f014 100644 --- a/Dockerfile +++ b/Dockerfile @@ -93,7 +93,7 @@ COPY resources/mlhubspawner /mlhubspawner RUN \ pip install --no-cache git+https://github.com/jupyterhub/dockerspawner@d1f27e2855d2cefbdb25b29cc069b9ca69d564e3 && \ - pip install --no-cache git+https://github.com/ml-tooling/nativeauthenticator@983b203069ca797ff5c595f985075c11ae17656c && \ + pip install --no-cache git+https://github.com/ml-tooling/nativeauthenticator@9859a69dcc9d2ae8d827f192a1580d86f897e9f1 && \ pip install --no-cache git+https://github.com/ryanlovett/imagespawner && \ pip install --no-cache /mlhubspawner && \ rm -r /mlhubspawner && \ From 0e582d13857286060fe17c14da503a0bc833acfb Mon Sep 17 00:00:00 2001 From: Benjamin Raethlein Date: Wed, 30 Oct 2019 12:34:48 +0100 Subject: [PATCH 02/21] Update Readme with regards to Default Login information --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e4804d2..8e0e91a 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Contribution

-MLHub is based on [Jupyterhub](https://github.com/jupyterhub/jupyterhub). MLHub allows to create and manage multiple [workspaces](https://github.com/ml-tooling/ml-workspace), for example to distribute them to a group of people or within a team. +MLHub is based on [Jupyterhub](https://github.com/jupyterhub/jupyterhub) with complete focus on Docker and Kubernetes. MLHub allows to create and manage multiple [workspaces](https://github.com/ml-tooling/ml-workspace), for example to distribute them to a group of people or within a team. ## Highlights @@ -61,7 +61,10 @@ For Kubernetes deployment, we forked and modified [zero-to-jupyterhub-k8s](https ### Configuration -In the default config, a user named `admin` can register and access the hub. If you use a different authenticator, you might want to set a different user as initial admin user as well. +#### Default Login + +When using the default config - so leaving the Jupyterhub config `c.Authenticator.admin_users` as it is -, a user named `admin` can access the hub with admin rights. If you use the default `NativeAuthenticator` as authenticator, youc must register the user `admin` with a password of your choice first before login in. +If you use a different authenticator, you might want to set a different user as initial admin user as well, for example in case of using oauth you want to set `c.Authenticator.admin_users` to a username returned by the oauth login. #### Environment Variables From 40ecbdcdf636ba2dfbafebcf732710cb7534bb19 Mon Sep 17 00:00:00 2001 From: Benjamin Raethlein Date: Wed, 30 Oct 2019 16:49:07 +0100 Subject: [PATCH 03/21] Refine host resource in spawner options view [Docker-local only] --- resources/mlhubspawner/mlhubspawner/spawner_options.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/resources/mlhubspawner/mlhubspawner/spawner_options.py b/resources/mlhubspawner/mlhubspawner/spawner_options.py index 014942c..c26dcf2 100644 --- a/resources/mlhubspawner/mlhubspawner/spawner_options.py +++ b/resources/mlhubspawner/mlhubspawner/spawner_options.py @@ -53,7 +53,7 @@ def get_options_form(spawner, additional_cpu_info="", additional_memory_info="",
- +
{additional_cpu_info}
@@ -89,11 +89,11 @@ def get_options_form(spawner, additional_cpu_info="", additional_memory_info="", ) def get_options_form_docker(spawner): - description_gpus = 'Empty for no GPU, "all" for all GPUs, or a comma-separated list of indices of the GPUs (e.g 0,2).' + description_gpus = 'Leave empty for no GPU, "all" for all GPUs, or a comma-separated list of indices of the GPUs (e.g 0,2).' additional_info = { - "additional_cpu_info": "Number between 1 and {cpu_count}".format(cpu_count=spawner.resource_information['cpu_count']), - "additional_memory_info": "Number between 1 and {memory_count_in_gb}".format(memory_count_in_gb=spawner.resource_information['memory_count_in_gb']), - "additional_gpu_info": "
Available GPUs: {gpu_count}
{description_gpus}
".format(gpu_count=spawner.resource_information['gpu_count'], description_gpus=description_gpus) + "additional_cpu_info": "Host has {cpu_count} CPUs".format(cpu_count=spawner.resource_information['cpu_count']), + "additional_memory_info": "Host has {memory_count_in_gb}GB memory".format(memory_count_in_gb=spawner.resource_information['memory_count_in_gb']), + "additional_gpu_info": "
Host has {gpu_count} GPUs
{description_gpus}
".format(gpu_count=spawner.resource_information['gpu_count'], description_gpus=description_gpus) } options_form = get_options_form(spawner, **additional_info) From 312fb19de71061d905bf2ca1dd519976ac519284 Mon Sep 17 00:00:00 2001 From: Benjamin Raethlein Date: Thu, 31 Oct 2019 10:45:59 +0100 Subject: [PATCH 04/21] Catch error when trying to rename a the hub container to make it more human-friendly --- resources/jupyterhub_config.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/resources/jupyterhub_config.py b/resources/jupyterhub_config.py index 778e61f..d20bb86 100644 --- a/resources/jupyterhub_config.py +++ b/resources/jupyterhub_config.py @@ -3,7 +3,9 @@ """ import os +import signal import socket +import docker.errors from mlhubspawner import utils from subprocess import call @@ -126,7 +128,12 @@ def combine_config_dicts(*configs) -> dict: tls_config = {**get_or_init(c.DockerSpawner.tls_config, dict), **get_or_init(c.MLHubDockerSpawner.tls_config, dict)} docker_client = utils.init_docker_client(client_kwargs, tls_config) - docker_client.containers.list(filters={"id": socket.gethostname()})[0].rename(ENV_HUB_NAME) + try: + docker_client.containers.list(filters={"id": socket.gethostname()})[0].rename(ENV_HUB_NAME) + except docker.errors.APIError as e: + print("Could not correctly start MLHub container. " + str(e)) + os.kill(os.getpid(), signal.SIGTERM) + c.MLHubDockerSpawner.hub_name = ENV_HUB_NAME # Add nativeauthenticator-specific templates From 2fe35f73a4064729c938a8a1ce372f2ccd67317c Mon Sep 17 00:00:00 2001 From: Benjamin Raethlein Date: Mon, 4 Nov 2019 17:10:22 +0100 Subject: [PATCH 05/21] Add button to show workspace config information --- Dockerfile | 2 + .../jupyterhub-mod/info-dialog-snippet.html | 36 ++ .../jsonpresenter/jquery.jsonPresenter.css | 59 +++ .../jsonpresenter/jquery.jsonPresenter.js | 347 ++++++++++++++++++ resources/jupyterhub-mod/template-admin.html | 14 +- resources/jupyterhub-mod/template-home.html | 6 + .../mlhubspawner/mlhubkubernetesspawner.py | 18 + .../mlhubspawner/mlhubspawner/mlhubspawner.py | 23 +- resources/mlhubspawner/mlhubspawner/utils.py | 18 + 9 files changed, 521 insertions(+), 2 deletions(-) create mode 100644 resources/jupyterhub-mod/info-dialog-snippet.html create mode 100644 resources/jupyterhub-mod/jsonpresenter/jquery.jsonPresenter.css create mode 100644 resources/jupyterhub-mod/jsonpresenter/jquery.jsonPresenter.js diff --git a/Dockerfile b/Dockerfile index 8e5f014..c08b7a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -133,6 +133,8 @@ COPY resources/logo.png /usr/local/share/jupyterhub/static/images/jupyter.png COPY resources/jupyterhub_config.py $_RESOURCES_PATH/jupyterhub_config.py COPY resources/jupyterhub-mod/template-home.html /usr/local/share/jupyterhub/templates/home.html COPY resources/jupyterhub-mod/template-admin.html /usr/local/share/jupyterhub/templates/admin.html +COPY resources/jupyterhub-mod/info-dialog-snippet.html /usr/local/share/jupyterhub/templates/info-dialog-snippet.html +COPY resources/jupyterhub-mod/jsonpresenter /usr/local/share/jupyterhub/static/components/jsonpresenter/ RUN \ touch $_RESOURCES_PATH/jupyterhub_user_config.py && \ diff --git a/resources/jupyterhub-mod/info-dialog-snippet.html b/resources/jupyterhub-mod/info-dialog-snippet.html new file mode 100644 index 0000000..2770d71 --- /dev/null +++ b/resources/jupyterhub-mod/info-dialog-snippet.html @@ -0,0 +1,36 @@ + + + + + + + + +{% call modal('Workspace Info', btn_label='', btn_class='hidden') %} + Workspace Info for : +
+ 

+  

+{% endcall %}
diff --git a/resources/jupyterhub-mod/jsonpresenter/jquery.jsonPresenter.css b/resources/jupyterhub-mod/jsonpresenter/jquery.jsonPresenter.css
new file mode 100644
index 0000000..2098418
--- /dev/null
+++ b/resources/jupyterhub-mod/jsonpresenter/jquery.jsonPresenter.css
@@ -0,0 +1,59 @@
+/**
+ * jQuery Json Presenter Plugin v1.0.0
+ *
+ * Copyright 2014 Steven Pease
+ * Released under the MIT license:
+ *   http://www.opensource.org/licenses/mit-license.php
+ */
+.parsed-json {
+	font-size: 11px !important;
+	font-family: Menlo, monospace;
+	margin-top: 0px;
+	margin-bottom: 0px;
+}
+
+.parsed-json-expandable-ellipsis:hover,
+.parsed-json-property-expandable:hover,
+.parsed-json-property-toggleable:hover,
+.parsed-json-has-alternate-value:hover {
+	cursor: hand;
+	cursor: pointer;
+}
+
+.parsed-json-property-name {
+	color: rgb(136, 19, 145);
+}
+
+.parsed-json-object-comma {
+	color: #333;
+}
+
+.parsed-json-value-boolean {
+	color: #0000ff;
+}
+
+.parsed-json-value-function,
+.parsed-json-array-bracket,
+.parsed-json-object-bracket {
+	color: #333;
+}
+
+.parsed-json-value-null,
+.parsed-json-value-undefined {
+	color: rgb(128, 128, 128);
+}
+
+.parsed-json-value-number {
+	color: #5FB526;
+}
+
+.parsed-json-value-regexp,
+.parsed-json-value-string {
+	color: rgb(196, 26, 22);
+	white-space: pre;
+	unicode-bidi: -webkit-isolate;
+}
+
+.parsed-json .hidden {
+	display: none;
+}
diff --git a/resources/jupyterhub-mod/jsonpresenter/jquery.jsonPresenter.js b/resources/jupyterhub-mod/jsonpresenter/jquery.jsonPresenter.js
new file mode 100644
index 0000000..a3837bc
--- /dev/null
+++ b/resources/jupyterhub-mod/jsonpresenter/jquery.jsonPresenter.js
@@ -0,0 +1,347 @@
+/**
+ * jQuery Json Presenter Plugin v1.0.0
+ *
+ * Copyright 2014 Steven Pease
+ * Released under the MIT license:
+ *   http://www.opensource.org/licenses/mit-license.php
+ */
+( function( $ ) {
+	/**
+	 * @param numberOfIndents
+	 */
+	function getIndentString( numberOfIndents ) {
+		if ( typeof numberOfIndents === "undefined" ) {
+			numberOfIndents = 1;
+		}
+
+		var result = '';
+		for ( var i = 0; i < numberOfIndents; i++ ) {
+
+			// Use two spaces to represent an indentation
+			result += '  ';
+		}
+		return result;
+	}
+
+	function isJsonArray( jsonValue ) {
+		return jsonValue && typeof jsonValue === 'object' && typeof jsonValue.length === 'number' && !jsonValue.propertyIsEnumerable( 'length' );
+	}
+
+	/**
+	 * @param {unknown_type} jsonValue The JSON value to test
+	 * @return {Boolean} Whether the provided JSON value is a Date object
+	 */
+	var isJsonDate = ( function() {
+		var dateObject = new Date();
+
+		return function( jsonValue ) {
+			return jsonValue && jsonValue.constructor === dateObject.constructor;
+		};
+	} )();
+
+	/**
+	 * @param {unknown_type} jsonValue The JSON value to test
+	 * @return {Boolean} Whether the provided JSON value is a NULL value
+	 */
+	var isJsonNull = function( jsonValue ) {
+		return jsonValue === null;
+	};
+
+	/**
+	 * @param {unknown_type} jsonValue The JSON value to test
+	 * @return {Boolean} Whether the provided JSON value is a RegExp object
+	 */
+	var isJsonRegExp = ( function() {
+		var regExpObject = new RegExp();
+
+		return function( jsonValue ) {
+			return jsonValue && jsonValue.constructor === regExpObject.constructor;
+		};
+	} )();
+
+	function processJsonPrimitive( className, value, alternateDisplayValue ) {
+		var cleanValue = function( value ) {
+			var result = value;
+
+			// Remove any "<" or ">" characters that could be interpretted as HTML
+			if ( typeof result === 'string' ) {
+				result = result.replace( //g, '>' );
+			}
+
+			return result;
+		};
+
+		if ( alternateDisplayValue ) {
+			value = '' + cleanValue( value ) + '';
+		} else {
+			value = cleanValue( value );
+		}
+
+		return '' + value + '';
+	}
+
+	function processJsonValue( settings, jsonValue, indentLevel, propertyName ) {
+		if ( typeof indentLevel === 'undefined' ) {
+			indentLevel = 0;
+		}
+
+		var isExpandable = false,
+			isToggleable = false,
+			result = '';
+
+		if ( isJsonArray( jsonValue ) ) {
+			if ( jsonValue.length ) {
+				result += '[';
+
+				for ( var i = 0; i < jsonValue.length; i++ ) {
+					result += processJsonValue( settings, jsonValue[ i ], indentLevel + 1 );
+
+					if ( i < jsonValue.length - 1 ) {
+						result += ',';
+					}
+				}
+
+				result += "\n" + getIndentString( indentLevel ) + ']';
+				isExpandable = true;
+			} else {
+				result += '[]';
+			}
+		} else {
+			var valueType = typeof jsonValue;
+
+			switch ( valueType ) {
+				case 'object':
+					if ( isJsonNull( jsonValue ) ) {
+						result += processJsonPrimitive( 'null', null );
+					} else if ( isJsonDate( jsonValue ) ) {
+						result += processJsonPrimitive( 'date', 'new Date(' + jsonValue.getTime() + ')', jsonValue.toString() );
+						isToggleable = true;
+					} else if ( isJsonRegExp( jsonValue ) ) {
+						result += processJsonPrimitive( 'regexp', jsonValue );
+					} else {
+
+						// Determine the number of properties this object has
+						var propertyCount = ( function() {
+							var result = 0;
+							for ( var i in jsonValue ) { // jshint ignore:line
+								result++;
+							}
+							return result;
+						} )();
+
+						if ( propertyCount > 0 ) {
+							result += '{';
+							( function() {
+								var propertyCounter = 0;
+								for ( var propertyName in jsonValue ) {
+									result += processJsonValue( settings, jsonValue[ propertyName ], indentLevel + 1, propertyName );
+
+									if ( ++propertyCounter < propertyCount ) {
+										result += ',';
+									}
+								}
+							} )();
+							result += "\n" + getIndentString( indentLevel ) + '}';
+							isExpandable = true;
+						} else {
+							result += '{}';
+						}
+					}
+				break;
+				case 'number':
+					result += processJsonPrimitive( 'number', jsonValue );
+				break;
+				case 'boolean':
+					result += processJsonPrimitive( 'boolean', jsonValue );
+				break;
+				case 'function':
+					var expandedFunction = ( jsonValue.toString() ).replace( /\n/g, "\n" + getIndentString( indentLevel ) ),
+						nonExpandedFunction = ( jsonValue.toString() ).replace( /\s+/g, ' ' );
+
+					if ( expandedFunction !== nonExpandedFunction ) {
+						result += processJsonPrimitive( 'function', nonExpandedFunction, expandedFunction );
+						isToggleable = true;
+					} else {
+						result += processJsonPrimitive( 'function', nonExpandedFunction );
+					}
+				break;
+				case 'undefined':
+					result += processJsonPrimitive( 'undefined', jsonValue );
+				break;
+				default:
+					var displayValue = '"' + jsonValue.replace( /\n/g, "\\n" ) + '"',
+						alternateDisplayValue = '"' + jsonValue + '"';
+
+					if ( displayValue !== alternateDisplayValue ) {
+						result += processJsonPrimitive( 'string', displayValue, alternateDisplayValue );
+						isToggleable = true;
+					} else {
+						result += processJsonPrimitive( 'string', displayValue );
+					}
+
+				break;
+			}
+		}
+
+		var resultPrefix = ( indentLevel !== 0 ? "\n" : '' ) + getIndentString( indentLevel );
+		if ( typeof propertyName !== 'undefined' ) {
+			var propertyNameLabel = settings.wrapPropertiesInQuotes ? '"' + propertyName + '"' : propertyName;
+			resultPrefix += '' + propertyNameLabel + ': ';
+		}
+
+		result = resultPrefix + result;
+
+		if ( isExpandable || isToggleable ) {
+			return '' + result + '';
+		} else {
+			return result;
+		}
+	}
+
+	function expandNode( expandableNodeElement ) {
+		if ( expandableNodeElement.children( '.parsed-json-expandable' ).is( ':not(:visible)' ) ) {
+			toggleExpandNode( expandableNodeElement );
+		}
+	}
+
+	function collapseNode( expandableNodeElement ) {
+		if ( expandableNodeElement.children( '.parsed-json-expandable' ).is( ':visible' ) ) {
+			toggleExpandNode( expandableNodeElement );
+		}
+	}
+
+	function toggleExpandNode( expandableNodeElement ) {
+		expandableNodeElement.children( '.parsed-json-expandable,.parsed-json-expandable-ellipsis' ).toggleClass( 'hidden' );
+	}
+
+	function togglePresentationNode( toggleableNodeElement ) {
+		toggleableNodeElement.children( '.parsed-json-has-alternate-value' ).find( 'span' ).toggleClass( 'hidden' );
+	}
+
+	function getExpandableChildNodes( expandableNodeElement ) {
+		return expandableNodeElement.find( '> .parsed-json-expandable > .parsed-json-node-expandable' );
+	}
+
+	function expandAll( expandableElement ) {
+		expand( expandableElement );
+	}
+
+	function collapseAll( expandableElement ) {
+		collapse( expandableElement );
+	}
+
+	function expand( expandableNodeElement, depth ) {
+		expandNode( expandableNodeElement );
+		if ( !( typeof depth === 'number' && depth <= 0 ) ) {
+			getExpandableChildNodes( expandableNodeElement ).each( function() {
+				expandNode( $( this ) );
+				expand( $( this ), typeof depth !== 'undefined' ? depth - 1 : depth );
+			} );
+		}
+	}
+
+	function collapse( expandableNodeElement, depth ) {
+		if ( !( typeof depth === 'number' && depth <= 0 ) ) {
+			getExpandableChildNodes( expandableNodeElement ).each( function() {
+				collapse( $( this ), typeof depth !== 'undefined' ? depth - 1 : depth );
+				collapseNode( $( this ) );
+			} );
+		}
+		collapseNode( expandableNodeElement );
+	}
+
+	function getRootNode( containerElement ) {
+		return containerElement.find( '> .parsed-json > .parsed-json-node-expandable' );
+	}
+
+	function onToggleableValueClick( event ) {
+		togglePresentationNode( $( $( event.currentTarget ).parents( '.parsed-json-node-toggleable' ).get( 0 ) ) );
+		event.stopPropagation();
+	}
+
+	function onExpandableValueClick( event ) {
+		toggleExpandNode( $( $( event.currentTarget ).parents( '.parsed-json-node-expandable' ).get( 0 ) ) );
+		event.stopPropagation();
+	}
+
+	function onExpandablePropertyClick( event ) {
+		toggleExpandNode( $( event.currentTarget ).parent() );
+		event.stopPropagation();
+	}
+
+	function onToggleablePropertyClick( event ) {
+		togglePresentationNode( $( event.currentTarget ).parent() );
+		event.stopPropagation();
+	}
+
+	function destroy( containerElement ) {
+		containerElement
+		.off( 'click', '.parsed-json-has-alternate-value span', onToggleableValueClick )
+		.off( 'click', '.parsed-json-property-expandable', onExpandablePropertyClick )
+		.off( 'click', '.parsed-json-expandable-ellipsis', onExpandableValueClick )
+		.off( 'click', '.parsed-json-property-toggleable', onToggleablePropertyClick );
+
+		containerElement.html( '' );
+	}
+
+	function create( containerElement, settings ) {
+
+		// Make sure that the JSON Presenter is not stacking event listeners on top of existing ones
+		if ( isAlreadyPresentingJson( containerElement ) ) {
+			destroy( containerElement );
+		}
+
+		containerElement
+		.on( 'click', '.parsed-json-has-alternate-value span', onToggleableValueClick )
+		.on( 'click', '.parsed-json-property-expandable', onExpandablePropertyClick )
+		.on( 'click', '.parsed-json-expandable-ellipsis', onExpandableValueClick )
+		.on( 'click', '.parsed-json-property-toggleable', onToggleablePropertyClick );
+
+		containerElement.html( '
' + processJsonValue( settings, settings.json ) + '
' );
+	}
+
+	/**
+	 * @param {DOMNode} containerElement The container element to check whether it already has JSON being presented within it
+	 * @return {Boolean} Whether the provided container element already has JSON being presented within it
+	 */
+	function isAlreadyPresentingJson( containerElement ) {
+		return !!containerElement.find( '> pre.parsed-json' ).length;
+	}
+
+	$.fn.jsonPresenter = function( options ) {
+		if ( options && typeof options === 'object' ) {
+			var defaults = {
+				json: {},
+				wrapPropertiesInQuotes: false
+			};
+
+			var settings = $.extend( {}, defaults, options );
+
+			return this.each( function() {
+				create( $( this ), settings );
+
+				if ( typeof settings.expand !== 'undefined' ) {
+					$( this ).jsonPresenter( 'expand', settings.expand );
+				}
+			} );
+		} else if ( arguments[ 0 ] === 'destroy' ) {
+			return this.each( function() {
+				destroy( $( this ) );
+			} );
+		} else if ( arguments[ 0 ] === 'expandAll' ) {
+			return this.each( function() {
+				expandAll( getRootNode( $( this ) ) );
+			} );
+		} else if ( arguments[ 0 ] === 'collapseAll' ) {
+			return this.each( function() {
+				collapseAll( getRootNode( $( this ) ) );
+			} );
+		} else if ( arguments[ 0 ] === 'expand' ) {
+			var depth = arguments[ 1 ];
+			return this.each( function() {
+				collapseAll( getRootNode( $( this ) ) );
+				expand( getRootNode( $( this ) ), depth );
+			} );
+		}
+	};
+} )( jQuery );
diff --git a/resources/jupyterhub-mod/template-admin.html b/resources/jupyterhub-mod/template-admin.html
index eb94b11..258ebc3 100644
--- a/resources/jupyterhub-mod/template-admin.html
+++ b/resources/jupyterhub-mod/template-admin.html
@@ -88,6 +88,13 @@
         
         {%- endif %}
       
+      
+        {%- if admin_access %}
+        
+        info
+        
+        {%- endif %}
+      
       
       {%- if spawner.name == '' -%}
         edit user
@@ -146,6 +153,8 @@
   
 {% endcall %}
 
+{# END CUSTOM STUFF #}
+
 {% macro user_modal(name, multi=False) %}
 {% call modal(name, btn_class='btn-primary save-button') %}
 

@@ -185,6 +194,7 @@
     if (serverName && serverName !== '') {
         serverName = "/" + serverName; // do it like this, otherwise there is one slash too much in the url if the serverName is empty
     }
+    // the name ssh-setup-dialog is created by the `call modal` command above and the provided name there
     let dialog = $("#ssh-setup-dialog");
     dialog.find(".server-name").text(user + serverName);
     let setupCommandTextField = dialog.find(".ssh-setup-command");
@@ -204,6 +214,8 @@
         .then(res =>  setupCommandTextField.text(res));
     dialog.modal();
   });
-
 
+
+{% include 'info-dialog-snippet.html' %}
+
 {% endblock %}
diff --git a/resources/jupyterhub-mod/template-home.html b/resources/jupyterhub-mod/template-home.html
index 82d28f8..64ad196 100644
--- a/resources/jupyterhub-mod/template-home.html
+++ b/resources/jupyterhub-mod/template-home.html
@@ -112,6 +112,9 @@ 

             start
           
           delete
+          
+            info
+          
         
       
       {% endfor %}
@@ -160,4 +163,7 @@ 

         }).catch(() => {});
 });
 
+
+{% include 'info-dialog-snippet.html' %}
+
 {% endblock %}
diff --git a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
index 8448eac..1475ed3 100644
--- a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
+++ b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
@@ -67,6 +67,9 @@ def get_env(self):
     @gen.coroutine
     def start(self):
         """Set custom configuration during start before calling the super.start method of Dockerspawner"""
+
+        self.saved_user_options = self.user_options
+
         if self.user_options.get('image'):
             self.image = self.user_options.get('image')
 
@@ -153,6 +156,9 @@ def get_container_metadata(self) -> str:
 
         return utils.get_container_metadata(self)
 
+    def get_workspace_config(self) -> str:
+        return utils.get_workspace_config(self)
+
     def get_lifetime_timestamp(self, labels: dict) -> float:
         return float(labels.get(utils.LABEL_EXPIRATION_TIMESTAMP, '0'))
 
@@ -171,3 +177,15 @@ def delete_if_exists(self, kind, safe_name, future):
             if e.status != 404:
                 raise
             self.log.warn("Could not delete %s/%s: does not exist", kind, safe_name)
+
+    # get_state and load_state are functions used by Jupyterhub to save and load variables that shall be persisted even if the hub is removed and re-created
+    # Override
+    def get_state(self):
+        state = super(MLHubKubernetesSpawner, self).get_state()
+        state = utils.get_state(self, state)
+        return state
+    
+    # Override
+    def load_state(self, state):
+        super(MLHubKubernetesSpawner, self).load_state(state)
+        utils.load_state(self, state)
diff --git a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
index 3f7ffd6..9c626e0 100644
--- a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
+++ b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
@@ -149,13 +149,15 @@ def start(self) -> (str, int):
             (str, int): container's ip address or '127.0.0.1', container's port
         """
 
+        self.saved_user_options = self.user_options
+
         if self.user_options.get('image'):
             self.image = self.user_options.get('image')
 
         extra_host_config = {}
         if self.user_options.get('cpu_limit'):
             # nano_cpus cannot be bigger than the number of CPUs of the machine (this method would currently not work in a cluster, as machines could be different than the machine where the runtime-manager and this code run.
-            max_available_cpus = self.resource_information.cpu_count
+            max_available_cpus = self.resource_information["cpu_count"]
             limited_cpus = min(
                 int(self.user_options.get('cpu_limit')), max_available_cpus)
 
@@ -295,6 +297,13 @@ def get_container_metadata(self) -> str:
             return ""
 
         return utils.get_container_metadata(self)
+    
+    def get_workspace_config(self) -> str:
+        return utils.get_workspace_config(self)
+        # if not hasattr(self, "saved_user_options"):
+        #     return "{}"
+        
+        # return json.dumps(self.saved_user_options)
 
     def get_lifetime_timestamp(self, labels: dict) -> float:
         return float(labels.get(utils.LABEL_EXPIRATION_TIMESTAMP, '0'))
@@ -319,6 +328,18 @@ def template_namespace(self):
         
         return template
 
+    # get_state and load_state are functions used by Jupyterhub to save and load variables that shall be persisted even if the hub is removed and re-created
+    # Override
+    def get_state(self):
+        state = super(MLHubDockerSpawner, self).get_state()
+        state = utils.get_state(self, state)
+        return state
+    
+    # Override
+    def load_state(self, state):
+        super(MLHubDockerSpawner, self).load_state(state)
+        utils.load_state(self, state)
+
     def get_gpu_info(self) -> list:
         count_gpu = 0
         try:
diff --git a/resources/mlhubspawner/mlhubspawner/utils.py b/resources/mlhubspawner/mlhubspawner/utils.py
index 75c2775..48869b7 100644
--- a/resources/mlhubspawner/mlhubspawner/utils.py
+++ b/resources/mlhubspawner/mlhubspawner/utils.py
@@ -8,6 +8,8 @@
 import docker
 from docker.utils import kwargs_from_env
 
+import json
+
 LABEL_NVIDIA_VISIBLE_DEVICES = 'nvidia_visible_devices'
 LABEL_EXPIRATION_TIMESTAMP = 'expiration_timestamp_seconds'
 
@@ -43,3 +45,19 @@ def init_docker_client(client_kwargs: dict, tls_config: dict) -> docker.DockerCl
     if client_kwargs:
         kwargs.update(client_kwargs)
     return docker.DockerClient(**kwargs)
+
+def get_state(spawner, state) -> dict:
+    if hasattr(spawner, "saved_user_options"):
+        state["saved_user_options"] = spawner.saved_user_options
+    
+    return state
+
+def load_state(spawner, state):    
+    if "saved_user_options" in state:
+        spawner.saved_user_options = state.get("saved_user_options")
+
+def get_workspace_config(spawner) -> str:
+    if not hasattr(spawner, "saved_user_options"):
+        return "{}"
+    
+    return json.dumps(spawner.saved_user_options)

From f16f9cb6646121710ebae66e2e947d366c4a7afe Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Mon, 4 Nov 2019 17:27:41 +0100
Subject: [PATCH 06/21] Fix issue with image comparison in
 `is_update_available` function, where the names could differ based on whether
 the fully-qualified name is used, which can happen if the image is pulled
 instead of built locally

---
 resources/mlhubspawner/mlhubspawner/mlhubspawner.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
index 9c626e0..1f066bd 100644
--- a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
+++ b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
@@ -300,17 +300,17 @@ def get_container_metadata(self) -> str:
     
     def get_workspace_config(self) -> str:
         return utils.get_workspace_config(self)
-        # if not hasattr(self, "saved_user_options"):
-        #     return "{}"
-        
-        # return json.dumps(self.saved_user_options)
 
     def get_lifetime_timestamp(self, labels: dict) -> float:
         return float(labels.get(utils.LABEL_EXPIRATION_TIMESTAMP, '0'))
 
     def is_update_available(self):
         try:
-            return self.image != self.highlevel_docker_client.containers.get(self.container_id).image.tags[0]
+            # compare the last parts of the images, so that also "mltooling/ml-workspace:0.8.7 = ml-workspace:0.8.7" would match
+            config_image = self.image.split("/")[-1]
+            workspace_image = self.highlevel_docker_client.containers.get(self.container_id).image.tags[0].split("/")[-1]
+
+            return config_image != workspace_image
         except (docker.errors.NotFound, docker.errors.NullResource):
             return False
 

From dbc690c49b518fd6a49174194f2b775c4991b0dc Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Mon, 4 Nov 2019 17:51:10 +0100
Subject: [PATCH 07/21] Fix issue where MLHub container cannot be restarted
 because rename conflict

---
 resources/jupyterhub_config.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/resources/jupyterhub_config.py b/resources/jupyterhub_config.py
index d20bb86..43cd11f 100644
--- a/resources/jupyterhub_config.py
+++ b/resources/jupyterhub_config.py
@@ -129,7 +129,10 @@ def combine_config_dicts(*configs) -> dict:
 
     docker_client = utils.init_docker_client(client_kwargs, tls_config)
     try:
-        docker_client.containers.list(filters={"id": socket.gethostname()})[0].rename(ENV_HUB_NAME)
+        container = docker_client.containers.list(filters={"id": socket.gethostname()})[0]
+
+        if container.name.lower() != ENV_HUB_NAME.lower():
+            container.rename(ENV_HUB_NAME.lower())
     except docker.errors.APIError as e:
         print("Could not correctly start MLHub container. " + str(e))
         os.kill(os.getpid(), signal.SIGTERM)

From 0757f826a80fe984e02c47b426a8a1d310597382 Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Thu, 7 Nov 2019 14:59:01 +0100
Subject: [PATCH 08/21] Add a service to cleanup the resources belonging to
 MLHub when a user is deleted. Refactor some components such as used labels
 for shared code.

---
 Dockerfile                                    |   1 +
 resources/jupyterhub-mod/cleanup-service.py   | 139 ++++++++++++++++++
 resources/jupyterhub_config.py                |  24 ++-
 .../mlhubspawner/mlhubkubernetesspawner.py    |   4 +-
 .../mlhubspawner/mlhubspawner/mlhubspawner.py |  10 +-
 resources/mlhubspawner/mlhubspawner/utils.py  |  15 ++
 6 files changed, 184 insertions(+), 9 deletions(-)
 create mode 100644 resources/jupyterhub-mod/cleanup-service.py

diff --git a/Dockerfile b/Dockerfile
index c08b7a9..44667d6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -135,6 +135,7 @@ COPY resources/jupyterhub-mod/template-home.html /usr/local/share/jupyterhub/tem
 COPY resources/jupyterhub-mod/template-admin.html /usr/local/share/jupyterhub/templates/admin.html
 COPY resources/jupyterhub-mod/info-dialog-snippet.html /usr/local/share/jupyterhub/templates/info-dialog-snippet.html
 COPY resources/jupyterhub-mod/jsonpresenter /usr/local/share/jupyterhub/static/components/jsonpresenter/
+COPY resources/jupyterhub-mod/cleanup-service.py /resources/cleanup-service.py
 
 RUN \
    touch $_RESOURCES_PATH/jupyterhub_user_config.py && \
diff --git a/resources/jupyterhub-mod/cleanup-service.py b/resources/jupyterhub-mod/cleanup-service.py
new file mode 100644
index 0000000..1ca76a1
--- /dev/null
+++ b/resources/jupyterhub-mod/cleanup-service.py
@@ -0,0 +1,139 @@
+"""
+Web service that is supposed to be started via JupyterHub.
+By this, the service has access to some information passed
+by JupyterHub. For more information check out https://jupyterhub.readthedocs.io/en/stable/reference/services.html
+"""
+
+import os
+import urllib3
+import json
+import time
+
+from tornado import web, ioloop
+from jupyterhub.services.auth import HubAuthenticated
+
+import docker.errors
+
+from mlhubspawner import utils
+
+# Environment variables passed by JupyterHub to the service
+prefix = os.environ.get('JUPYTERHUB_SERVICE_PREFIX', '/')
+service_url = os.getenv('JUPYTERHUB_SERVICE_URL')
+jupyterhub_api_url = os.getenv('JUPYTERHUB_API_URL')
+jupyterhub_api_token = os.getenv('JUPYTERHUB_API_TOKEN')
+
+auth_header = {"Authorization": "token " + jupyterhub_api_token}
+
+http = urllib3.PoolManager()
+
+docker_client_kwargs = json.loads(os.getenv("DOCKER_CLIENT_KWARGS"))
+docker_tls_kwargs = json.loads(os.getenv("DOCKER_TLS_CONFIG"))
+docker_client = utils.init_docker_client(docker_client_kwargs, docker_tls_kwargs)
+
+origin_key, hub_name = utils.get_origin_label()
+origin_label_filter = {"label": "{}={}".format(origin_key, hub_name)}
+
+
+def remove_deleted_user_resources(existing_user_names: []):
+    """Remove resources for which no user exists anymore by checking whether the label of user name occurs in the existing
+    users list.
+
+        Args:
+            existing_user_names: list of user names that exist in the JupyterHub database
+    """
+
+    def try_to_remove(remove_callback, resource) -> bool:
+        """Call the remove callback until the call succeeds or until the number of tries is exceeded.
+
+            Returns:
+                bool: True if it could be removed, False if it was not removable within the number of tries
+        """
+
+        for i in range(3):
+            try:
+                remove_callback()
+                return True
+            except docker.errors.APIError:
+                time.sleep(3)
+        
+        print("Could not remove " + resource.name)
+        return False
+
+
+    def find_and_remove(docker_client_obj, get_labels, action_callback) -> None:
+        """List all resources belonging to `docker_client_obj` which were created by MLHub.
+        Then check the list of resources for resources that belong to a user who does not exist anymore 
+        and call the remove function on them.
+
+            Args:
+                docker_client_obj: A Python docker client object, such as docker_client.containers, docker_client.networks,... It must implement a .list() function (check https://docker-py.readthedocs.io/en/stable/containers.html)
+                get_labels (func): function to call on the docker resource to get the labels
+                remove (func): function to call on the docker resource to remove it
+        """
+
+        resources = docker_client_obj.list(filters=origin_label_filter)
+        for resource in resources:
+            user_label = get_labels(resource)[utils.LABEL_MLHUB_USER]
+            if user_label not in existing_user_names:
+                action_callback(resource)
+                # successful = try_to_remove(remove, resource)
+                    
+    def container_action(container):
+        try_to_remove(
+            lambda: container.remove(v=True, force=True), 
+            container
+        )
+
+    find_and_remove(
+        docker_client.containers, 
+        lambda res: res.labels, 
+        container_action
+    )
+
+    def network_action(network):
+        try:
+            network.disconnect(hub_name)
+        except docker.errors.APIError:
+            pass
+        
+        try_to_remove(network.remove, network)
+
+    find_and_remove(
+        docker_client.networks, 
+        lambda res: res.attrs["Labels"], 
+        network_action
+    )
+
+    find_and_remove(
+        docker_client.volumes,
+        lambda res: res.attrs["Labels"],
+        lambda res: try_to_remove(res.remove, res)
+    )
+
+class CleanupUserResources(HubAuthenticated, web.RequestHandler):
+
+    @web.authenticated
+    def get(self):
+        # TODO: check for admin rights
+        print(self.get_current_user())
+        
+        r = http.request('GET', jupyterhub_api_url + "/users", 
+            headers = {**auth_header}
+        )
+
+        data = json.loads(r.data.decode("utf-8"))
+        existing_user_names = []
+        for user in data:
+            existing_user_names.append(user["name"])
+
+        remove_deleted_user_resources(existing_user_names)
+
+        #self.write()
+
+app = web.Application([
+    (r"{}users".format(prefix), CleanupUserResources)
+])
+
+service_port = int(service_url.split(":")[-1])
+app.listen(service_port)
+ioloop.IOLoop.current().start()
diff --git a/resources/jupyterhub_config.py b/resources/jupyterhub_config.py
index 43cd11f..db7d877 100644
--- a/resources/jupyterhub_config.py
+++ b/resources/jupyterhub_config.py
@@ -5,6 +5,7 @@
 import os
 import signal
 import socket
+
 import docker.errors
 
 from mlhubspawner import utils
@@ -42,7 +43,8 @@ def combine_config_dicts(*configs) -> dict:
 
 ### END HELPER FUNCTIONS###
 
-ENV_HUB_NAME = os.environ['HUB_NAME']
+ENV_HUB_NAME_KEY = 'HUB_NAME'
+ENV_HUB_NAME = os.environ[ENV_HUB_NAME_KEY]
 
 # User containers will access hub by container name on the Docker network
 c.JupyterHub.hub_ip = '0.0.0.0' #'research-hub'
@@ -67,7 +69,7 @@ def combine_config_dicts(*configs) -> dict:
 c.Spawner.environment = default_env
 
 # Workaround to prevent api problems
-c.Spawner.will_resume = True
+#c.Spawner.will_resume = True
 
 # --- Spawner-specific ----
 c.JupyterHub.spawner_class = 'mlhubspawner.MLHubDockerSpawner' # override in your config if you want to have a different spawner. If it is the or inherits from DockerSpawner, the c.DockerSpawner config can have an effect.
@@ -137,7 +139,23 @@ def combine_config_dicts(*configs) -> dict:
         print("Could not correctly start MLHub container. " + str(e))
         os.kill(os.getpid(), signal.SIGTERM)
 
-    c.MLHubDockerSpawner.hub_name = ENV_HUB_NAME
+    #c.MLHubDockerSpawner.hub_name = ENV_HUB_NAME
+
+    import sys
+    import json
+    c.JupyterHub.services = [
+        {
+            'name': 'cleanup-service',
+            'admin': True,
+            'url': 'http://127.0.0.1:9000',
+            'environment': {
+                ENV_HUB_NAME_KEY: ENV_HUB_NAME,
+                "DOCKER_CLIENT_KWARGS": json.dumps(client_kwargs), 
+                "DOCKER_TLS_CONFIG": json.dumps(tls_config)
+            },
+            'command': [sys.executable, '/resources/cleanup-service.py']
+        }
+    ]
 
 # Add nativeauthenticator-specific templates
 if c.JupyterHub.authenticator_class == NATIVE_AUTHENTICATOR_CLASS:
diff --git a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
index 1475ed3..5261c91 100644
--- a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
+++ b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
@@ -30,8 +30,8 @@ class MLHubKubernetesSpawner(KubeSpawner):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        self.hub_name = os.getenv("HUB_NAME", "mlhub")
-        self.default_label = {"origin": self.hub_name}   
+        origin_key, self.hub_name = utils.get_origin_label()
+        self.default_label = {origin_key: self.hub_name, utils.LABEL_MLHUB_USER: self.user.name}  
     
     @default('options_form')
     def _options_form(self):
diff --git a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
index 1f066bd..eb82f56 100644
--- a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
+++ b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
@@ -47,7 +47,7 @@ def has_complete_network_information(network):
 class MLHubDockerSpawner(DockerSpawner):
     """Provides the possibility to spawn docker containers with specific options, such as resource limits (CPU and Memory), Environment Variables, ..."""
 
-    hub_name = Unicode(config=True, help="Name of the hub container.")
+    #hub_name = Unicode(config=True, help="Name of the hub container.")
 
     workspace_images = List(
         trait = Unicode(),
@@ -58,11 +58,12 @@ class MLHubDockerSpawner(DockerSpawner):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        origin_key, self.hub_name = utils.get_origin_label()
+        self.default_labels = {origin_key: self.hub_name, utils.LABEL_MLHUB_USER: self.user.name}
 
         # Get the MLHub container name to be used as the DNS name for the spawned workspaces, so they can connect to the Hub even if the container is
         # removed and restarted
         client = self.highlevel_docker_client
-        self.default_label = {"origin": self.hub_name}
 
         # Connect MLHub to the existing workspace networks (in case of removing / recreation). By this, the hub can connect to the existing
         # workspaces and does not have to restart them.
@@ -171,11 +172,12 @@ def start(self) -> (str, int):
         if self.user_options.get('is_mount_volume') == 'on':
             # {username} and {servername} will be automatically replaced by DockerSpawner with the right values as in template_namespace
             #volumeName = self.name_template.format(prefix=self.prefix)
+            self.highlevel_docker_client.volumes.create(name=self.object_name, labels=self.default_labels)
             self.volumes = {self.object_name: "/workspace"}
 
         extra_create_kwargs = {}
         # set default label 'origin' to know for sure which containers where started via the hub
-        extra_create_kwargs['labels'] = self.default_label
+        extra_create_kwargs['labels'] = self.default_labels
         if self.user_options.get('days_to_live'):
             days_to_live_in_seconds = int(self.user_options.get('days_to_live')) * 24 * 60 * 60 # days * hours_per_day * minutes_per_hour * seconds_per_minute
             expiration_timestamp = time.time() + days_to_live_in_seconds
@@ -278,7 +280,7 @@ def create_network(self, name):
         ipam_pool = docker.types.IPAMPool(subnet=next_cidr.exploded,
                                           gateway=(next_cidr.network_address + 1).exploded)
         ipam_config = docker.types.IPAMConfig(pool_configs=[ipam_pool])
-        return client.networks.create(name, ipam=ipam_config, labels=self.default_label)
+        return client.networks.create(name, ipam=ipam_config, labels=self.default_labels)
     
     def connect_hub_to_network(self, network):
         try:
diff --git a/resources/mlhubspawner/mlhubspawner/utils.py b/resources/mlhubspawner/mlhubspawner/utils.py
index 48869b7..15eadaf 100644
--- a/resources/mlhubspawner/mlhubspawner/utils.py
+++ b/resources/mlhubspawner/mlhubspawner/utils.py
@@ -2,6 +2,8 @@
 Shared util functions
 """
 
+import os
+
 import math
 import time
 
@@ -13,6 +15,18 @@
 LABEL_NVIDIA_VISIBLE_DEVICES = 'nvidia_visible_devices'
 LABEL_EXPIRATION_TIMESTAMP = 'expiration_timestamp_seconds'
 
+LABEL_MLHUB_USER = "mlhub.user"
+
+ENV_HUB_NAME = os.getenv("HUB_NAME", "mlhub")
+
+def get_origin_label() -> tuple:
+    """
+    Returns:
+        tuple (str, str): (key, value) for origin label
+    """
+
+    return "mlhub.origin", ENV_HUB_NAME
+
 def get_container_metadata(spawner):
     meta_information = []
     container_labels = spawner.get_labels()
@@ -44,6 +58,7 @@ def init_docker_client(client_kwargs: dict, tls_config: dict) -> docker.DockerCl
     kwargs.update(kwargs_from_env())
     if client_kwargs:
         kwargs.update(client_kwargs)
+        
     return docker.DockerClient(**kwargs)
 
 def get_state(spawner, state) -> dict:

From 4f9e2d2b40b1f8b0ea08584f6c76acb479bab7fa Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Thu, 7 Nov 2019 16:34:35 +0100
Subject: [PATCH 09/21] Add cleanup function for expired containers

---
 resources/jupyterhub-mod/cleanup-service.py   | 59 ++++++++++++++-----
 .../mlhubspawner/mlhubkubernetesspawner.py    |  3 -
 .../mlhubspawner/mlhubspawner/mlhubspawner.py |  3 -
 resources/mlhubspawner/mlhubspawner/utils.py  |  6 +-
 4 files changed, 49 insertions(+), 22 deletions(-)

diff --git a/resources/jupyterhub-mod/cleanup-service.py b/resources/jupyterhub-mod/cleanup-service.py
index 1ca76a1..b9b8369 100644
--- a/resources/jupyterhub-mod/cleanup-service.py
+++ b/resources/jupyterhub-mod/cleanup-service.py
@@ -8,6 +8,7 @@
 import urllib3
 import json
 import time
+import math
 
 from tornado import web, ioloop
 from jupyterhub.services.auth import HubAuthenticated
@@ -33,6 +34,8 @@
 origin_key, hub_name = utils.get_origin_label()
 origin_label_filter = {"label": "{}={}".format(origin_key, hub_name)}
 
+def get_hub_docker_resources(docker_client_obj):
+    return docker_client_obj.list(filters=origin_label_filter)
 
 def remove_deleted_user_resources(existing_user_names: []):
     """Remove resources for which no user exists anymore by checking whether the label of user name occurs in the existing
@@ -71,7 +74,7 @@ def find_and_remove(docker_client_obj, get_labels, action_callback) -> None:
                 remove (func): function to call on the docker resource to remove it
         """
 
-        resources = docker_client_obj.list(filters=origin_label_filter)
+        resources = get_hub_docker_resources(docker_client_obj)
         for resource in resources:
             user_label = get_labels(resource)[utils.LABEL_MLHUB_USER]
             if user_label not in existing_user_names:
@@ -110,28 +113,54 @@ def network_action(network):
         lambda res: try_to_remove(res.remove, res)
     )
 
+def get_hub_usernames() -> []:
+    r = http.request('GET', jupyterhub_api_url + "/users", 
+        headers = {**auth_header}
+    )
+
+    data = json.loads(r.data.decode("utf-8"))
+    existing_user_names = []
+    for user in data:
+        existing_user_names.append(user["name"])
+
+    return existing_user_names
+
+def remove_expired_workspaces():
+    hub_containers = get_hub_docker_resources(docker_client.containers)
+    for container in hub_containers:
+        lifetime_timestamp = utils.get_lifetime_timestamp(container.labels)
+        if lifetime_timestamp != 0:
+            difference = math.ceil(lifetime_timestamp - time.time())
+            # container lifetime is exceeded (remaining lifetime is negative)
+            if difference < 0:
+                container.remove(v=True, force=True)
+
 class CleanupUserResources(HubAuthenticated, web.RequestHandler):
 
     @web.authenticated
     def get(self):
-        # TODO: check for admin rights
-        print(self.get_current_user())
-        
-        r = http.request('GET', jupyterhub_api_url + "/users", 
-            headers = {**auth_header}
-        )
-
-        data = json.loads(r.data.decode("utf-8"))
-        existing_user_names = []
-        for user in data:
-            existing_user_names.append(user["name"])
+        current_user = self.get_current_user()
+        if current_user.admin is False:
+            self.set_status(401)
+            self.finish()
+            return
 
-        remove_deleted_user_resources(existing_user_names)
+        remove_deleted_user_resources(get_hub_usernames())
 
-        #self.write()
+class CleanupExpiredContainers(HubAuthenticated, web.RequestHandler):
+    @web.authenticated
+    def get(self):
+        current_user = self.get_current_user()
+        if current_user.admin is False:
+            self.set_status(401)
+            self.finish()
+            return
+        
+        remove_expired_workspaces()
 
 app = web.Application([
-    (r"{}users".format(prefix), CleanupUserResources)
+    (r"{}users".format(prefix), CleanupUserResources),
+    (r"{}expired".format(prefix), CleanupExpiredContainers)
 ])
 
 service_port = int(service_url.split(":")[-1])
diff --git a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
index 5261c91..d1a8c32 100644
--- a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
+++ b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
@@ -159,9 +159,6 @@ def get_container_metadata(self) -> str:
     def get_workspace_config(self) -> str:
         return utils.get_workspace_config(self)
 
-    def get_lifetime_timestamp(self, labels: dict) -> float:
-        return float(labels.get(utils.LABEL_EXPIRATION_TIMESTAMP, '0'))
-
     def get_labels(self) -> dict:
         try:
             return self.pod_reflector.pods.get(self.pod_name, None).metadata.labels
diff --git a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
index eb82f56..f250531 100644
--- a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
+++ b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
@@ -303,9 +303,6 @@ def get_container_metadata(self) -> str:
     def get_workspace_config(self) -> str:
         return utils.get_workspace_config(self)
 
-    def get_lifetime_timestamp(self, labels: dict) -> float:
-        return float(labels.get(utils.LABEL_EXPIRATION_TIMESTAMP, '0'))
-
     def is_update_available(self):
         try:
             # compare the last parts of the images, so that also "mltooling/ml-workspace:0.8.7 = ml-workspace:0.8.7" would match
diff --git a/resources/mlhubspawner/mlhubspawner/utils.py b/resources/mlhubspawner/mlhubspawner/utils.py
index 15eadaf..07c2558 100644
--- a/resources/mlhubspawner/mlhubspawner/utils.py
+++ b/resources/mlhubspawner/mlhubspawner/utils.py
@@ -27,10 +27,14 @@ def get_origin_label() -> tuple:
 
     return "mlhub.origin", ENV_HUB_NAME
 
+
+def get_lifetime_timestamp(labels: dict) -> float:
+    return float(labels.get(LABEL_EXPIRATION_TIMESTAMP, '0'))
+
 def get_container_metadata(spawner):
     meta_information = []
     container_labels = spawner.get_labels()
-    lifetime_timestamp = spawner.get_lifetime_timestamp(container_labels)
+    lifetime_timestamp = get_lifetime_timestamp(container_labels)
     if lifetime_timestamp != 0:
         difference_in_days = math.ceil((lifetime_timestamp - time.time())/60/60/24)
         meta_information.append("Expires: {}d".format(difference_in_days))

From 3a8ff0c74ca9debfdb3c1ca012f4694a062ed1ae Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Thu, 7 Nov 2019 16:44:45 +0100
Subject: [PATCH 10/21] Use the JupyterHub API to "delete" a named server when
 expired

---
 resources/jupyterhub-mod/cleanup-service.py          | 12 ++++++++++--
 .../mlhubspawner/mlhubkubernetesspawner.py           |  2 +-
 resources/mlhubspawner/mlhubspawner/mlhubspawner.py  |  2 +-
 resources/mlhubspawner/mlhubspawner/utils.py         |  1 +
 4 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/resources/jupyterhub-mod/cleanup-service.py b/resources/jupyterhub-mod/cleanup-service.py
index b9b8369..4022a9f 100644
--- a/resources/jupyterhub-mod/cleanup-service.py
+++ b/resources/jupyterhub-mod/cleanup-service.py
@@ -133,7 +133,15 @@ def remove_expired_workspaces():
             difference = math.ceil(lifetime_timestamp - time.time())
             # container lifetime is exceeded (remaining lifetime is negative)
             if difference < 0:
-                container.remove(v=True, force=True)
+                user_name = container.labels[utils.LABEL_MLHUB_USER]
+                server_name = container.labels[utils.LABEL_MLHUB_SERVER_NAME]
+                url = jupyterhub_api_url + "/users/{user_name}/servers/{servers_name}".format(user_name=user_name, server_name=server_name)
+                r = http.request('DELETE', url, 
+                    headers = {**auth_header}
+                )
+
+                # TODO: also remove the underlying container?
+                # container.remove(v=True, force=True)
 
 class CleanupUserResources(HubAuthenticated, web.RequestHandler):
 
@@ -155,7 +163,7 @@ def get(self):
             self.set_status(401)
             self.finish()
             return
-        
+
         remove_expired_workspaces()
 
 app = web.Application([
diff --git a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
index d1a8c32..5637632 100644
--- a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
+++ b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
@@ -31,7 +31,7 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
         origin_key, self.hub_name = utils.get_origin_label()
-        self.default_label = {origin_key: self.hub_name, utils.LABEL_MLHUB_USER: self.user.name}  
+        self.default_label = {origin_key: self.hub_name, utils.LABEL_MLHUB_USER: self.user.name, utils.LABEL_MLHUB_SERVER_NAME: self.name}  
     
     @default('options_form')
     def _options_form(self):
diff --git a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
index f250531..17ba6c3 100644
--- a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
+++ b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
@@ -59,7 +59,7 @@ class MLHubDockerSpawner(DockerSpawner):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         origin_key, self.hub_name = utils.get_origin_label()
-        self.default_labels = {origin_key: self.hub_name, utils.LABEL_MLHUB_USER: self.user.name}
+        self.default_labels = {origin_key: self.hub_name, utils.LABEL_MLHUB_USER: self.user.name, utils.LABEL_MLHUB_SERVER_NAME: self.name}
 
         # Get the MLHub container name to be used as the DNS name for the spawned workspaces, so they can connect to the Hub even if the container is
         # removed and restarted
diff --git a/resources/mlhubspawner/mlhubspawner/utils.py b/resources/mlhubspawner/mlhubspawner/utils.py
index 07c2558..4b8aea8 100644
--- a/resources/mlhubspawner/mlhubspawner/utils.py
+++ b/resources/mlhubspawner/mlhubspawner/utils.py
@@ -16,6 +16,7 @@
 LABEL_EXPIRATION_TIMESTAMP = 'expiration_timestamp_seconds'
 
 LABEL_MLHUB_USER = "mlhub.user"
+LABEL_MLHUB_SERVER_NAME = "mlhub.server_name"
 
 ENV_HUB_NAME = os.getenv("HUB_NAME", "mlhub")
 

From abb0b7fca88275feae11f67d735fbad3f371502c Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Thu, 7 Nov 2019 17:17:32 +0100
Subject: [PATCH 11/21] Start with differentiating of docker and kubernetes
 mode in cleanup script

---
 resources/jupyterhub-mod/cleanup-service.py  | 22 ++++++++++++++++++--
 resources/jupyterhub_config.py               |  2 +-
 resources/mlhubspawner/mlhubspawner/utils.py |  4 ++++
 3 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/resources/jupyterhub-mod/cleanup-service.py b/resources/jupyterhub-mod/cleanup-service.py
index 4022a9f..81a66ec 100644
--- a/resources/jupyterhub-mod/cleanup-service.py
+++ b/resources/jupyterhub-mod/cleanup-service.py
@@ -25,6 +25,8 @@
 
 auth_header = {"Authorization": "token " + jupyterhub_api_token}
 
+execution_mode = os.environ[utils.ENV_NAME_EXECUTION_MODE]
+
 http = urllib3.PoolManager()
 
 docker_client_kwargs = json.loads(os.getenv("DOCKER_CLIENT_KWARGS"))
@@ -37,6 +39,13 @@
 def get_hub_docker_resources(docker_client_obj):
     return docker_client_obj.list(filters=origin_label_filter)
 
+def get_resource_labels(resource):
+    if execution_mode == utils.EXECUTION_MODE_DOCKER:
+        return resource.labels
+    elif execution_mode == utils.EXECUTION_MODE_KUBERNETES:
+        # TODO: FINISH
+        return {}
+
 def remove_deleted_user_resources(existing_user_names: []):
     """Remove resources for which no user exists anymore by checking whether the label of user name occurs in the existing
     users list.
@@ -126,9 +135,14 @@ def get_hub_usernames() -> []:
     return existing_user_names
 
 def remove_expired_workspaces():
-    hub_containers = get_hub_docker_resources(docker_client.containers)
+    if execution_mode == utils.EXECUTION_MODE_DOCKER:
+        hub_containers = get_hub_docker_resources(docker_client.containers)
+    elif execution_mode == utils.EXECUTION_MODE_KUBERNETES:
+        # TODO: finish
+        hub_containers = []
+    
     for container in hub_containers:
-        lifetime_timestamp = utils.get_lifetime_timestamp(container.labels)
+        lifetime_timestamp = utils.get_lifetime_timestamp(get_resource_labels(container))
         if lifetime_timestamp != 0:
             difference = math.ceil(lifetime_timestamp - time.time())
             # container lifetime is exceeded (remaining lifetime is negative)
@@ -153,6 +167,10 @@ def get(self):
             self.finish()
             return
 
+        if execution_mode == utils.EXECUTION_MODE_KUBERNETES:
+            self.finish("This method cannot be used in hub execution mode " + execution_mode)
+            return
+        
         remove_deleted_user_resources(get_hub_usernames())
 
 class CleanupExpiredContainers(HubAuthenticated, web.RequestHandler):
diff --git a/resources/jupyterhub_config.py b/resources/jupyterhub_config.py
index db7d877..f71e5dd 100644
--- a/resources/jupyterhub_config.py
+++ b/resources/jupyterhub_config.py
@@ -112,7 +112,7 @@ def combine_config_dicts(*configs) -> dict:
 
 # In Kubernetes mode, load the Kubernetes Jupyterhub config that can be configured via a config.yaml.
 # Those values will override the values set above, as it is loaded afterwards.
-if os.environ['EXECUTION_MODE'] == "k8s":
+if os.environ[utils.ENV_NAME_EXECUTION_MODE] == utils.EXECUTION_MODE_KUBERNETES:
     load_subconfig("{}/kubernetes/jupyterhub_chart_config.py".format(os.getenv("_RESOURCES_PATH")))
 
     c.JupyterHub.spawner_class = 'mlhubspawner.MLHubKubernetesSpawner'
diff --git a/resources/mlhubspawner/mlhubspawner/utils.py b/resources/mlhubspawner/mlhubspawner/utils.py
index 4b8aea8..6ecfcb0 100644
--- a/resources/mlhubspawner/mlhubspawner/utils.py
+++ b/resources/mlhubspawner/mlhubspawner/utils.py
@@ -18,6 +18,10 @@
 LABEL_MLHUB_USER = "mlhub.user"
 LABEL_MLHUB_SERVER_NAME = "mlhub.server_name"
 
+ENV_NAME_EXECUTION_MODE = "EXECUTION_MODE"
+EXECUTION_MODE_DOCKER = "docker"
+EXECUTION_MODE_KUBERNETES = "k8s"
+
 ENV_HUB_NAME = os.getenv("HUB_NAME", "mlhub")
 
 def get_origin_label() -> tuple:

From 240daef790f3b31f611089e961c59db5d0b3bc0e Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Fri, 8 Nov 2019 12:56:50 +0100
Subject: [PATCH 12/21] Add support for cleaning expired containers in
 Kubernetes mode

---
 resources/jupyterhub-mod/cleanup-service.py | 38 +++++++++++++--------
 1 file changed, 24 insertions(+), 14 deletions(-)

diff --git a/resources/jupyterhub-mod/cleanup-service.py b/resources/jupyterhub-mod/cleanup-service.py
index 81a66ec..3a7778e 100644
--- a/resources/jupyterhub-mod/cleanup-service.py
+++ b/resources/jupyterhub-mod/cleanup-service.py
@@ -14,6 +14,7 @@
 from jupyterhub.services.auth import HubAuthenticated
 
 import docker.errors
+from kubernetes import client, config, stream
 
 from mlhubspawner import utils
 
@@ -29,22 +30,30 @@
 
 http = urllib3.PoolManager()
 
-docker_client_kwargs = json.loads(os.getenv("DOCKER_CLIENT_KWARGS"))
-docker_tls_kwargs = json.loads(os.getenv("DOCKER_TLS_CONFIG"))
-docker_client = utils.init_docker_client(docker_client_kwargs, docker_tls_kwargs)
+if execution_mode == utils.EXECUTION_MODE_DOCKER:
+    docker_client_kwargs = json.loads(os.getenv("DOCKER_CLIENT_KWARGS"))
+    docker_tls_kwargs = json.loads(os.getenv("DOCKER_TLS_CONFIG"))
+    docker_client = utils.init_docker_client(docker_client_kwargs, docker_tls_kwargs)
+elif execution_mode == utils.EXECUTION_MODE_KUBERNETES:
+    # incluster config is the config given by a service account and it's role permissions
+    config.load_incluster_config()
+    kubernetes_client = client.CoreV1Api()
 
 origin_key, hub_name = utils.get_origin_label()
-origin_label_filter = {"label": "{}={}".format(origin_key, hub_name)}
+origin_label = "{}={}".format(origin_key, hub_name)
+origin_label_filter = {"label": origin_label}
 
 def get_hub_docker_resources(docker_client_obj):
     return docker_client_obj.list(filters=origin_label_filter)
 
+def get_hub_kubernetes_resources(namespaced_list_command, **kwargs):
+    return namespaced_list_command(hub_name, **kwargs).items
+
 def get_resource_labels(resource):
     if execution_mode == utils.EXECUTION_MODE_DOCKER:
         return resource.labels
     elif execution_mode == utils.EXECUTION_MODE_KUBERNETES:
-        # TODO: FINISH
-        return {}
+        return resource.metadata.labels
 
 def remove_deleted_user_resources(existing_user_names: []):
     """Remove resources for which no user exists anymore by checking whether the label of user name occurs in the existing
@@ -138,17 +147,17 @@ def remove_expired_workspaces():
     if execution_mode == utils.EXECUTION_MODE_DOCKER:
         hub_containers = get_hub_docker_resources(docker_client.containers)
     elif execution_mode == utils.EXECUTION_MODE_KUBERNETES:
-        # TODO: finish
-        hub_containers = []
-    
+        hub_containers = get_hub_kubernetes_resources(kubernetes_client.list_namespaced_pod, field_selector="status.phase=Running", label_selector=origin_label)
+
     for container in hub_containers:
-        lifetime_timestamp = utils.get_lifetime_timestamp(get_resource_labels(container))
+        container_labels = get_resource_labels(container)
+        lifetime_timestamp = utils.get_lifetime_timestamp(container_labels)
         if lifetime_timestamp != 0:
             difference = math.ceil(lifetime_timestamp - time.time())
             # container lifetime is exceeded (remaining lifetime is negative)
             if difference < 0:
-                user_name = container.labels[utils.LABEL_MLHUB_USER]
-                server_name = container.labels[utils.LABEL_MLHUB_SERVER_NAME]
+                user_name = container_labels[utils.LABEL_MLHUB_USER]
+                server_name = container_labels[utils.LABEL_MLHUB_SERVER_NAME]
                 url = jupyterhub_api_url + "/users/{user_name}/servers/{servers_name}".format(user_name=user_name, server_name=server_name)
                 r = http.request('DELETE', url, 
                     headers = {**auth_header}
@@ -168,12 +177,13 @@ def get(self):
             return
 
         if execution_mode == utils.EXECUTION_MODE_KUBERNETES:
-            self.finish("This method cannot be used in hub execution mode " + execution_mode)
+            self.finish("This method cannot be used in following hub execution mode " + execution_mode)
             return
-        
+
         remove_deleted_user_resources(get_hub_usernames())
 
 class CleanupExpiredContainers(HubAuthenticated, web.RequestHandler):
+
     @web.authenticated
     def get(self):
         current_user = self.get_current_user()

From 2ac00e13e870bfcddbf8ed26b9dc47caaee98d10 Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Fri, 8 Nov 2019 16:34:52 +0100
Subject: [PATCH 13/21] Add internal caller that automatically periodically
 calls the clean functions

---
 Dockerfile                                   |  3 +-
 README.md                                    | 10 +++++
 resources/jupyterhub-mod/cleanup-service.py  | 46 ++++++++++++++------
 resources/jupyterhub_config.py               | 10 +++--
 resources/mlhubspawner/mlhubspawner/utils.py |  2 +-
 5 files changed, 52 insertions(+), 19 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 44667d6..c127b31 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -157,7 +157,8 @@ ENV \
    START_JHUB=true \
    START_CHP=false \
    EXECUTION_MODE="local" \
-   HUB_NAME="mlhub"
+   HUB_NAME="mlhub" \ 
+   CLEANUP_INTERVAL_SECONDS=3600
 
 ### END CONFIGURATION ###
 
diff --git a/README.md b/README.md
index 8e0e91a..bb72800 100644
--- a/README.md
+++ b/README.md
@@ -84,6 +84,16 @@ Here are the additional environment variables for the hub:
         
         mlhub
     
+    
+        EXECUTION_MODE
+        Defines in which execution mode the hub is running in. Value is one of [docker | k8s]
+        local
+    
+    
+        CLEANUP_INTERVAL_SECONDS
+        Interval in which expired and not-used resources are deleted. Set to -1 to disable the automatic cleanup.
+        3600
+    
     
         SSL_ENABLED
         Enable SSL. If you don't provide an ssl certificate as described in Section "Enable SSL/HTTPS", certificates will be generated automatically. As this auto-generated certificate is not signed, you have to trust it in the browser. Without ssl enabled, ssh access won't work as the container uses a single port and has to tell https and ssh traffic apart.
diff --git a/resources/jupyterhub-mod/cleanup-service.py b/resources/jupyterhub-mod/cleanup-service.py
index 3a7778e..bb55a2c 100644
--- a/resources/jupyterhub-mod/cleanup-service.py
+++ b/resources/jupyterhub-mod/cleanup-service.py
@@ -9,6 +9,7 @@
 import json
 import time
 import math
+from threading import Thread
 
 from tornado import web, ioloop
 from jupyterhub.services.auth import HubAuthenticated
@@ -30,7 +31,7 @@
 
 http = urllib3.PoolManager()
 
-if execution_mode == utils.EXECUTION_MODE_DOCKER:
+if execution_mode == utils.EXECUTION_MODE_LOCAL:
     docker_client_kwargs = json.loads(os.getenv("DOCKER_CLIENT_KWARGS"))
     docker_tls_kwargs = json.loads(os.getenv("DOCKER_TLS_CONFIG"))
     docker_client = utils.init_docker_client(docker_client_kwargs, docker_tls_kwargs)
@@ -50,7 +51,7 @@ def get_hub_kubernetes_resources(namespaced_list_command, **kwargs):
     return namespaced_list_command(hub_name, **kwargs).items
 
 def get_resource_labels(resource):
-    if execution_mode == utils.EXECUTION_MODE_DOCKER:
+    if execution_mode == utils.EXECUTION_MODE_LOCAL:
         return resource.labels
     elif execution_mode == utils.EXECUTION_MODE_KUBERNETES:
         return resource.metadata.labels
@@ -61,8 +62,14 @@ def remove_deleted_user_resources(existing_user_names: []):
 
         Args:
             existing_user_names: list of user names that exist in the JupyterHub database
+
+        Raises:
+            UserWarning: in Kubernetes mode, the function does not work
     """
 
+    if execution_mode == utils.EXECUTION_MODE_KUBERNETES:
+        raise UserWarning("This method cannot be used in following hub execution mode " + execution_mode)
+
     def try_to_remove(remove_callback, resource) -> bool:
         """Call the remove callback until the call succeeds or until the number of tries is exceeded.
 
@@ -89,7 +96,7 @@ def find_and_remove(docker_client_obj, get_labels, action_callback) -> None:
             Args:
                 docker_client_obj: A Python docker client object, such as docker_client.containers, docker_client.networks,... It must implement a .list() function (check https://docker-py.readthedocs.io/en/stable/containers.html)
                 get_labels (func): function to call on the docker resource to get the labels
-                remove (func): function to call on the docker resource to remove it
+                remove (func): function to call on the docker resource to remove it                
         """
 
         resources = get_hub_docker_resources(docker_client_obj)
@@ -144,7 +151,7 @@ def get_hub_usernames() -> []:
     return existing_user_names
 
 def remove_expired_workspaces():
-    if execution_mode == utils.EXECUTION_MODE_DOCKER:
+    if execution_mode == utils.EXECUTION_MODE_LOCAL:
         hub_containers = get_hub_docker_resources(docker_client.containers)
     elif execution_mode == utils.EXECUTION_MODE_KUBERNETES:
         hub_containers = get_hub_kubernetes_resources(kubernetes_client.list_namespaced_pod, field_selector="status.phase=Running", label_selector=origin_label)
@@ -155,16 +162,17 @@ def remove_expired_workspaces():
         if lifetime_timestamp != 0:
             difference = math.ceil(lifetime_timestamp - time.time())
             # container lifetime is exceeded (remaining lifetime is negative)
-            if difference < 0:
+            if difference < 48 * 24 * 60 * 60: #0:
                 user_name = container_labels[utils.LABEL_MLHUB_USER]
                 server_name = container_labels[utils.LABEL_MLHUB_SERVER_NAME]
-                url = jupyterhub_api_url + "/users/{user_name}/servers/{servers_name}".format(user_name=user_name, server_name=server_name)
+                url = jupyterhub_api_url + "/users/{user_name}/servers/{server_name}".format(user_name=user_name, server_name=server_name)
                 r = http.request('DELETE', url, 
                     headers = {**auth_header}
                 )
 
-                # TODO: also remove the underlying container?
-                # container.remove(v=True, force=True)
+                if r.status == 202 or r.status == 204:
+                    print("Delete expired container " + container.name)
+                    container.remove(v=True, force=True)
 
 class CleanupUserResources(HubAuthenticated, web.RequestHandler):
 
@@ -176,11 +184,10 @@ def get(self):
             self.finish()
             return
 
-        if execution_mode == utils.EXECUTION_MODE_KUBERNETES:
-            self.finish("This method cannot be used in following hub execution mode " + execution_mode)
-            return
-
-        remove_deleted_user_resources(get_hub_usernames())
+        try:
+            remove_deleted_user_resources(get_hub_usernames())
+        except UserWarning as e:
+            self.finish(str(e))
 
 class CleanupExpiredContainers(HubAuthenticated, web.RequestHandler):
 
@@ -201,4 +208,17 @@ def get(self):
 
 service_port = int(service_url.split(":")[-1])
 app.listen(service_port)
+
+def internal_service_caller():
+    clean_interval_seconds = os.getenv("CLEANUP_INTERVAL_SECONDS")
+    while True and clean_interval_seconds != -1:
+        time.sleep(clean_interval_seconds)
+        try:
+            remove_deleted_user_resources(get_hub_usernames())
+        except UserWarning:
+            pass
+        remove_expired_workspaces()
+
+Thread(target=internal_service_caller).start()
+
 ioloop.IOLoop.current().start()
diff --git a/resources/jupyterhub_config.py b/resources/jupyterhub_config.py
index f71e5dd..a573c51 100644
--- a/resources/jupyterhub_config.py
+++ b/resources/jupyterhub_config.py
@@ -43,8 +43,9 @@ def combine_config_dicts(*configs) -> dict:
 
 ### END HELPER FUNCTIONS###
 
-ENV_HUB_NAME_KEY = 'HUB_NAME'
-ENV_HUB_NAME = os.environ[ENV_HUB_NAME_KEY]
+ENV_NAME_HUB_NAME = 'HUB_NAME'
+ENV_HUB_NAME = os.environ[ENV_NAME_HUB_NAME]
+ENV_EXECUTION_MODE = os.environ[utils.ENV_NAME_EXECUTION_MODE]
 
 # User containers will access hub by container name on the Docker network
 c.JupyterHub.hub_ip = '0.0.0.0' #'research-hub'
@@ -112,7 +113,7 @@ def combine_config_dicts(*configs) -> dict:
 
 # In Kubernetes mode, load the Kubernetes Jupyterhub config that can be configured via a config.yaml.
 # Those values will override the values set above, as it is loaded afterwards.
-if os.environ[utils.ENV_NAME_EXECUTION_MODE] == utils.EXECUTION_MODE_KUBERNETES:
+if ENV_EXECUTION_MODE == utils.EXECUTION_MODE_KUBERNETES:
     load_subconfig("{}/kubernetes/jupyterhub_chart_config.py".format(os.getenv("_RESOURCES_PATH")))
 
     c.JupyterHub.spawner_class = 'mlhubspawner.MLHubKubernetesSpawner'
@@ -149,7 +150,8 @@ def combine_config_dicts(*configs) -> dict:
             'admin': True,
             'url': 'http://127.0.0.1:9000',
             'environment': {
-                ENV_HUB_NAME_KEY: ENV_HUB_NAME,
+                ENV_NAME_HUB_NAME: ENV_HUB_NAME,
+                utils.ENV_NAME_EXECUTION_MODE: ENV_EXECUTION_MODE,
                 "DOCKER_CLIENT_KWARGS": json.dumps(client_kwargs), 
                 "DOCKER_TLS_CONFIG": json.dumps(tls_config)
             },
diff --git a/resources/mlhubspawner/mlhubspawner/utils.py b/resources/mlhubspawner/mlhubspawner/utils.py
index 6ecfcb0..1185218 100644
--- a/resources/mlhubspawner/mlhubspawner/utils.py
+++ b/resources/mlhubspawner/mlhubspawner/utils.py
@@ -19,7 +19,7 @@
 LABEL_MLHUB_SERVER_NAME = "mlhub.server_name"
 
 ENV_NAME_EXECUTION_MODE = "EXECUTION_MODE"
-EXECUTION_MODE_DOCKER = "docker"
+EXECUTION_MODE_LOCAL = "local"
 EXECUTION_MODE_KUBERNETES = "k8s"
 
 ENV_HUB_NAME = os.getenv("HUB_NAME", "mlhub")

From ff79e9bdff1efbb335d8a91b7169d00705c25610 Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Mon, 11 Nov 2019 17:26:48 +0100
Subject: [PATCH 14/21] The cleanup service now works in Docker local and
 Kubernetes mode. Add some more labels for created resources.

---
 resources/jupyterhub-mod/cleanup-service.py   | 88 ++++++++++++++-----
 resources/jupyterhub_config.py                | 55 ++++++++----
 .../mlhubspawner/mlhubkubernetesspawner.py    | 14 +--
 .../mlhubspawner/mlhubspawner/mlhubspawner.py |  5 +-
 resources/mlhubspawner/mlhubspawner/utils.py  | 11 +--
 5 files changed, 114 insertions(+), 59 deletions(-)

diff --git a/resources/jupyterhub-mod/cleanup-service.py b/resources/jupyterhub-mod/cleanup-service.py
index bb55a2c..a0435e8 100644
--- a/resources/jupyterhub-mod/cleanup-service.py
+++ b/resources/jupyterhub-mod/cleanup-service.py
@@ -2,6 +2,8 @@
 Web service that is supposed to be started via JupyterHub.
 By this, the service has access to some information passed
 by JupyterHub. For more information check out https://jupyterhub.readthedocs.io/en/stable/reference/services.html
+
+Note: Logs probably don't appear in stdout, as the service is started as a subprocess by JupyterHub
 """
 
 import os
@@ -10,6 +12,7 @@
 import time
 import math
 from threading import Thread
+import logging
 
 from tornado import web, ioloop
 from jupyterhub.services.auth import HubAuthenticated
@@ -40,21 +43,63 @@
     config.load_incluster_config()
     kubernetes_client = client.CoreV1Api()
 
-origin_key, hub_name = utils.get_origin_label()
-origin_label = "{}={}".format(origin_key, hub_name)
+hub_name = utils.ENV_HUB_NAME
+origin_label = "{}={}".format(utils.LABEL_MLHUB_ORIGIN, hub_name)
 origin_label_filter = {"label": origin_label}
 
+class UnifiedContainer():
+
+    def __init__(self, resource):
+        self.remove = lambda: logging.info("Remove property is not defined")
+        self.resource = resource
+
+    def with_id(self, id):
+        self.id = id
+        return self
+    
+    def with_name(self, name):
+        self.name = name
+        return self
+    
+    def with_labels(self, labels):
+        self.labels = labels
+        return self
+    
+    def with_remove(self, func):
+        self.remove = lambda: func(self.resource)
+        return self
+
+def extract_container(resource):
+    if execution_mode == utils.EXECUTION_MODE_LOCAL:
+        unified_container = UnifiedContainer(resource) \
+            .with_id(resource.id) \
+            .with_name(resource.name) \
+            .with_labels(resource.labels) \
+            .with_remove(lambda container: container.remove(v=True, force=True))
+    elif execution_mode == utils.EXECUTION_MODE_KUBERNETES:
+        unified_container = UnifiedContainer(resource) \
+            .with_id(resource.metadata.uid) \
+            .with_name(resource.metadata.name) \
+            .with_labels(resource.metadata.labels)
+
+    if unified_container == None:
+        raise UserWarning("The execution mode environment variable is not set correctly")
+
+    return unified_container
+
 def get_hub_docker_resources(docker_client_obj):
     return docker_client_obj.list(filters=origin_label_filter)
 
 def get_hub_kubernetes_resources(namespaced_list_command, **kwargs):
     return namespaced_list_command(hub_name, **kwargs).items
 
-def get_resource_labels(resource):
+def get_hub_containers():
     if execution_mode == utils.EXECUTION_MODE_LOCAL:
-        return resource.labels
+        hub_containers = get_hub_docker_resources(docker_client.containers)
     elif execution_mode == utils.EXECUTION_MODE_KUBERNETES:
-        return resource.metadata.labels
+        hub_containers = get_hub_kubernetes_resources(kubernetes_client.list_namespaced_pod, field_selector="status.phase=Running", label_selector=origin_label)
+    
+    return hub_containers
 
 def remove_deleted_user_resources(existing_user_names: []):
     """Remove resources for which no user exists anymore by checking whether the label of user name occurs in the existing
@@ -84,7 +129,7 @@ def try_to_remove(remove_callback, resource) -> bool:
             except docker.errors.APIError:
                 time.sleep(3)
         
-        print("Could not remove " + resource.name)
+        logging.info("Could not remove " + resource.name)
         return False
 
 
@@ -150,36 +195,33 @@ def get_hub_usernames() -> []:
 
     return existing_user_names
 
-def remove_expired_workspaces():
-    if execution_mode == utils.EXECUTION_MODE_LOCAL:
-        hub_containers = get_hub_docker_resources(docker_client.containers)
-    elif execution_mode == utils.EXECUTION_MODE_KUBERNETES:
-        hub_containers = get_hub_kubernetes_resources(kubernetes_client.list_namespaced_pod, field_selector="status.phase=Running", label_selector=origin_label)
-
+def remove_expired_workspaces(): 
+    hub_containers = get_hub_containers()
     for container in hub_containers:
-        container_labels = get_resource_labels(container)
-        lifetime_timestamp = utils.get_lifetime_timestamp(container_labels)
+        unified_container = extract_container(container)
+        lifetime_timestamp = utils.get_lifetime_timestamp(unified_container.labels)
         if lifetime_timestamp != 0:
             difference = math.ceil(lifetime_timestamp - time.time())
             # container lifetime is exceeded (remaining lifetime is negative)
-            if difference < 48 * 24 * 60 * 60: #0:
-                user_name = container_labels[utils.LABEL_MLHUB_USER]
-                server_name = container_labels[utils.LABEL_MLHUB_SERVER_NAME]
+            if difference < 0:
+                user_name = unified_container.labels[utils.LABEL_MLHUB_USER]
+                server_name = unified_container.labels[utils.LABEL_MLHUB_SERVER_NAME]
                 url = jupyterhub_api_url + "/users/{user_name}/servers/{server_name}".format(user_name=user_name, server_name=server_name)
-                r = http.request('DELETE', url, 
+                r = http.request('DELETE', url,
+                    body = json.dumps({"remove": True}).encode('utf-8'),
                     headers = {**auth_header}
                 )
 
                 if r.status == 202 or r.status == 204:
-                    print("Delete expired container " + container.name)
-                    container.remove(v=True, force=True)
+                    logging.info("Delete expired container " + unified_container.name)
+                    unified_container.remove()
 
 class CleanupUserResources(HubAuthenticated, web.RequestHandler):
 
     @web.authenticated
     def get(self):
         current_user = self.get_current_user()
-        if current_user.admin is False:
+        if current_user["admin"] is False:
             self.set_status(401)
             self.finish()
             return
@@ -194,7 +236,7 @@ class CleanupExpiredContainers(HubAuthenticated, web.RequestHandler):
     @web.authenticated
     def get(self):
         current_user = self.get_current_user()
-        if current_user.admin is False:
+        if current_user["admin"] is False:
             self.set_status(401)
             self.finish()
             return
@@ -210,7 +252,7 @@ def get(self):
 app.listen(service_port)
 
 def internal_service_caller():
-    clean_interval_seconds = os.getenv("CLEANUP_INTERVAL_SECONDS")
+    clean_interval_seconds = int(os.getenv(utils.ENV_NAME_CLEANUP_INTERVAL_SECONDS))
     while True and clean_interval_seconds != -1:
         time.sleep(clean_interval_seconds)
         try:
diff --git a/resources/jupyterhub_config.py b/resources/jupyterhub_config.py
index a573c51..4dc388b 100644
--- a/resources/jupyterhub_config.py
+++ b/resources/jupyterhub_config.py
@@ -5,9 +5,12 @@
 import os
 import signal
 import socket
+import sys
 
 import docker.errors
 
+import json
+
 from mlhubspawner import utils
 from subprocess import call
 
@@ -111,6 +114,14 @@ def combine_config_dicts(*configs) -> dict:
 # See https://traitlets.readthedocs.io/en/stable/config.html#configuration-files-inheritance
 load_subconfig("{}/jupyterhub_user_config.py".format(os.getenv("_RESOURCES_PATH")))
 
+
+
+service_environment = {
+    ENV_NAME_HUB_NAME: ENV_HUB_NAME,
+    utils.ENV_NAME_EXECUTION_MODE: ENV_EXECUTION_MODE,
+    utils.ENV_NAME_CLEANUP_INTERVAL_SECONDS: os.getenv(utils.ENV_NAME_CLEANUP_INTERVAL_SECONDS),
+}
+
 # In Kubernetes mode, load the Kubernetes Jupyterhub config that can be configured via a config.yaml.
 # Those values will override the values set above, as it is loaded afterwards.
 if ENV_EXECUTION_MODE == utils.EXECUTION_MODE_KUBERNETES:
@@ -126,7 +137,19 @@ def combine_config_dicts(*configs) -> dict:
     # if not isinstance(c.KubeSpawner.environment, dict):
     #     c.KubeSpawner.environment = {}
     c.KubeSpawner.environment.update(default_env)
-else:
+
+    # For cleanup-service
+    ## Env variables that are used by the Python Kubernetes library to load the incluster config
+    SERVICE_HOST_ENV_NAME = "KUBERNETES_SERVICE_HOST"
+    SERVICE_PORT_ENV_NAME = "KUBERNETES_SERVICE_PORT"
+    service_environment.update({
+        SERVICE_HOST_ENV_NAME: os.getenv(SERVICE_HOST_ENV_NAME), 
+        SERVICE_PORT_ENV_NAME: os.getenv(SERVICE_PORT_ENV_NAME)
+    })
+    service_host = "hub"
+    
+
+elif ENV_EXECUTION_MODE == utils.EXECUTION_MODE_LOCAL:
     client_kwargs = {**get_or_init(c.DockerSpawner.client_kwargs, dict), **get_or_init(c.MLHubDockerSpawner.client_kwargs, dict)}
     tls_config = {**get_or_init(c.DockerSpawner.tls_config, dict), **get_or_init(c.MLHubDockerSpawner.tls_config, dict)}
 
@@ -140,25 +163,11 @@ def combine_config_dicts(*configs) -> dict:
         print("Could not correctly start MLHub container. " + str(e))
         os.kill(os.getpid(), signal.SIGTERM)
 
+    # For cleanup-service
+    service_environment.update({"DOCKER_CLIENT_KWARGS": json.dumps(client_kwargs), "DOCKER_TLS_CONFIG": json.dumps(tls_config)})
+    service_host = "127.0.0.1"
     #c.MLHubDockerSpawner.hub_name = ENV_HUB_NAME
 
-    import sys
-    import json
-    c.JupyterHub.services = [
-        {
-            'name': 'cleanup-service',
-            'admin': True,
-            'url': 'http://127.0.0.1:9000',
-            'environment': {
-                ENV_NAME_HUB_NAME: ENV_HUB_NAME,
-                utils.ENV_NAME_EXECUTION_MODE: ENV_EXECUTION_MODE,
-                "DOCKER_CLIENT_KWARGS": json.dumps(client_kwargs), 
-                "DOCKER_TLS_CONFIG": json.dumps(tls_config)
-            },
-            'command': [sys.executable, '/resources/cleanup-service.py']
-        }
-    ]
-
 # Add nativeauthenticator-specific templates
 if c.JupyterHub.authenticator_class == NATIVE_AUTHENTICATOR_CLASS:
     import nativeauthenticator
@@ -167,3 +176,13 @@ def combine_config_dicts(*configs) -> dict:
     # if not isinstance(c.JupyterHub.template_paths, list):
     #     c.JupyterHub.template_paths = []
     c.JupyterHub.template_paths.append("{}/templates/".format(os.path.dirname(nativeauthenticator.__file__)))
+
+c.JupyterHub.services = [
+    {
+        'name': 'cleanup-service',
+        'admin': True,
+        'url': 'http://{}:9000'.format(service_host),
+        'environment': service_environment,
+        'command': [sys.executable, '/resources/cleanup-service.py']
+    }
+]
diff --git a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
index 5637632..809e7f5 100644
--- a/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
+++ b/resources/mlhubspawner/mlhubspawner/mlhubkubernetesspawner.py
@@ -17,6 +17,8 @@
 import re
 
 from mlhubspawner import spawner_options, utils
+
+LABEL_POD_NAME = "pod_name"
 class MLHubKubernetesSpawner(KubeSpawner):
     """Provides the possibility to spawn docker containers with specific options, such as resource limits (CPU and Memory), Environment Variables, ..."""
 
@@ -30,8 +32,9 @@ class MLHubKubernetesSpawner(KubeSpawner):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        origin_key, self.hub_name = utils.get_origin_label()
-        self.default_label = {origin_key: self.hub_name, utils.LABEL_MLHUB_USER: self.user.name, utils.LABEL_MLHUB_SERVER_NAME: self.name}  
+        self.hub_name = utils.ENV_HUB_NAME
+        self.default_label = {utils.LABEL_MLHUB_ORIGIN: self.hub_name, utils.LABEL_MLHUB_USER: self.user.name, utils.LABEL_MLHUB_SERVER_NAME: self.name, LABEL_POD_NAME: self.pod_name}
+        self.extra_labels.update(self.default_label)
     
     @default('options_form')
     def _options_form(self):
@@ -90,8 +93,7 @@ def start(self):
         #    self.volumes = {'jhub-user-{username}{servername}': "/workspace"}
 
         # set default label 'origin' to know for sure which containers where started via the hub
-        self.extra_labels['origin'] = self.hub_name
-        self.extra_labels['pod_name'] = self.pod_name
+        #self.extra_labels['pod_name'] = self.pod_name
         if self.user_options.get('days_to_live'):
             days_to_live_in_seconds = int(self.user_options.get('days_to_live')) * 24 * 60 * 60 # days * hours_per_day * minutes_per_hour * seconds_per_minute
             expiration_timestamp = time.time() + days_to_live_in_seconds
@@ -112,8 +114,8 @@ def start(self):
                 type='ClusterIP',
                 ports=[V1ServicePort(port=self.port, target_port=self.port)],
                 selector={
-                    'origin': self.extra_labels['origin'], 
-                    'pod_name': self.extra_labels['pod_name']
+                    utils.LABEL_MLHUB_ORIGIN: self.extra_labels[utils.LABEL_MLHUB_ORIGIN], 
+                    LABEL_POD_NAME: self.extra_labels[LABEL_POD_NAME]
                 }
             ),
             metadata = V1ObjectMeta(
diff --git a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
index 17ba6c3..384d46d 100644
--- a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
+++ b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
@@ -58,9 +58,8 @@ class MLHubDockerSpawner(DockerSpawner):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        origin_key, self.hub_name = utils.get_origin_label()
-        self.default_labels = {origin_key: self.hub_name, utils.LABEL_MLHUB_USER: self.user.name, utils.LABEL_MLHUB_SERVER_NAME: self.name}
-
+        self.hub_name = utils.ENV_HUB_NAME
+        self.default_labels = {utils.LABEL_MLHUB_ORIGIN: self.hub_name, utils.LABEL_MLHUB_USER: self.user.name, utils.LABEL_MLHUB_SERVER_NAME: self.name}
         # Get the MLHub container name to be used as the DNS name for the spawned workspaces, so they can connect to the Hub even if the container is
         # removed and restarted
         client = self.highlevel_docker_client
diff --git a/resources/mlhubspawner/mlhubspawner/utils.py b/resources/mlhubspawner/mlhubspawner/utils.py
index 1185218..d362daf 100644
--- a/resources/mlhubspawner/mlhubspawner/utils.py
+++ b/resources/mlhubspawner/mlhubspawner/utils.py
@@ -21,18 +21,11 @@
 ENV_NAME_EXECUTION_MODE = "EXECUTION_MODE"
 EXECUTION_MODE_LOCAL = "local"
 EXECUTION_MODE_KUBERNETES = "k8s"
+ENV_NAME_CLEANUP_INTERVAL_SECONDS = "CLEANUP_INTERVAL_SECONDS"
 
+LABEL_MLHUB_ORIGIN = "mlhub.origin"
 ENV_HUB_NAME = os.getenv("HUB_NAME", "mlhub")
 
-def get_origin_label() -> tuple:
-    """
-    Returns:
-        tuple (str, str): (key, value) for origin label
-    """
-
-    return "mlhub.origin", ENV_HUB_NAME
-
-
 def get_lifetime_timestamp(labels: dict) -> float:
     return float(labels.get(LABEL_EXPIRATION_TIMESTAMP, '0'))
 

From ea30fc39bb8df790f7f387e16aeb5ebca7f98922 Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Mon, 11 Nov 2019 18:33:52 +0100
Subject: [PATCH 15/21] Show container expiration date and gpu indices in the
 "Info" dialog instead of directly next to the name

---
 resources/jupyterhub-mod/template-admin.html  |  1 -
 .../mlhubspawner/mlhubspawner/mlhubspawner.py |  6 ----
 resources/mlhubspawner/mlhubspawner/utils.py  | 31 +++++++------------
 3 files changed, 11 insertions(+), 27 deletions(-)

diff --git a/resources/jupyterhub-mod/template-admin.html b/resources/jupyterhub-mod/template-admin.html
index 258ebc3..3a07bbc 100644
--- a/resources/jupyterhub-mod/template-admin.html
+++ b/resources/jupyterhub-mod/template-admin.html
@@ -49,7 +49,6 @@
         {%- if spawner.name -%}
         /{{ spawner.name }}
         {%- endif -%}
-          {{ spawner.get_container_metadata() if spawner.get_container_metadata is defined }} {# comment {{ current_user.spawner_class().get_days_to_live() }} #}
       
 
       
diff --git a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
index 384d46d..7a2f0c7 100644
--- a/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
+++ b/resources/mlhubspawner/mlhubspawner/mlhubspawner.py
@@ -293,12 +293,6 @@ def connect_hub_to_network(self, network):
                     "Could not connect mlhub to the network and, thus, cannot create the container.")
                 return
     
-    def get_container_metadata(self) -> str:
-        if self.container_id is None or self.container_id == '':
-            return ""
-
-        return utils.get_container_metadata(self)
-    
     def get_workspace_config(self) -> str:
         return utils.get_workspace_config(self)
 
diff --git a/resources/mlhubspawner/mlhubspawner/utils.py b/resources/mlhubspawner/mlhubspawner/utils.py
index d362daf..e9506b2 100644
--- a/resources/mlhubspawner/mlhubspawner/utils.py
+++ b/resources/mlhubspawner/mlhubspawner/utils.py
@@ -29,23 +29,6 @@
 def get_lifetime_timestamp(labels: dict) -> float:
     return float(labels.get(LABEL_EXPIRATION_TIMESTAMP, '0'))
 
-def get_container_metadata(spawner):
-    meta_information = []
-    container_labels = spawner.get_labels()
-    lifetime_timestamp = get_lifetime_timestamp(container_labels)
-    if lifetime_timestamp != 0:
-        difference_in_days = math.ceil((lifetime_timestamp - time.time())/60/60/24)
-        meta_information.append("Expires: {}d".format(difference_in_days))
-    
-    nvidia_visible_devices = container_labels.get(LABEL_NVIDIA_VISIBLE_DEVICES, "")
-    if nvidia_visible_devices != "":
-        meta_information.append("GPUs: {}".format(nvidia_visible_devices))
-    
-    if len(meta_information) == 0:
-        return ""
-    
-    return "({})".format(", ".join(meta_information))
-
 def init_docker_client(client_kwargs: dict, tls_config: dict) -> docker.DockerClient:
     """Create a docker client. 
     The configuration is done the same way DockerSpawner initializes the low-level API client.
@@ -74,7 +57,15 @@ def load_state(spawner, state):
         spawner.saved_user_options = state.get("saved_user_options")
 
 def get_workspace_config(spawner) -> str:
-    if not hasattr(spawner, "saved_user_options"):
-        return "{}"
+    workspace_config = {}
+    if hasattr(spawner, "saved_user_options"):
+        workspace_config = {**spawner.saved_user_options}
+
+    # Add remaining lifetime information
+    lifetime_timestamp = get_lifetime_timestamp(spawner.get_labels())
+    if lifetime_timestamp != 0:
+        difference_in_seconds = math.ceil(lifetime_timestamp - time.time())
+        difference_in_days = math.ceil(difference_in_seconds/60/60/24)
+        workspace_config.update({"remaining_lifetime_seconds": difference_in_seconds, "remaining_lifetime_days": difference_in_days})
     
-    return json.dumps(spawner.saved_user_options)
+    return json.dumps(workspace_config)

From f607d9500d231fec6aaebf17dd432e49ffc922a1 Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Mon, 11 Nov 2019 19:06:51 +0100
Subject: [PATCH 16/21] Make the username normalization more error resistant

---
 resources/jupyterhub_config.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/resources/jupyterhub_config.py b/resources/jupyterhub_config.py
index 4dc388b..12d23d5 100644
--- a/resources/jupyterhub_config.py
+++ b/resources/jupyterhub_config.py
@@ -24,8 +24,19 @@
 original_normalize_username = Authenticator.normalize_username
 def custom_normalize_username(self, username):
     username = original_normalize_username(self, username)
-    for forbidden_username_char in [" ", ",", ";", "."]:
-        username = username.replace(forbidden_username_char, "")
+    more_than_one_forbidden_char = False
+    for forbidden_username_char in [" ", ",", ";", ".", "-"]:
+        # Replace special characters with a non-special character. Cannot just be empty, like "", because then it could happen that two distinct user names are transformed into the same username.
+        # Example: "foo, bar" and "fo, obar" would both become "foobar".
+        replace_char = "0"
+        # If there is more than one special character, just replace one of them. Otherwise, "foo, bar" would become "foo00bar" instead of "foo0bar"
+        if more_than_one_forbidden_char == True:
+            replace_char = ""
+        temp_username = username
+        username = username.replace(forbidden_username_char, replace_char)
+        if username != temp_username:
+            more_than_one_forbidden_char = True
+
     return username
 Authenticator.normalize_username = custom_normalize_username
 

From 7cbbd994d4d08fcdfa272e3b388c2bf6daf1f853 Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Mon, 11 Nov 2019 19:40:46 +0100
Subject: [PATCH 17/21] Show "SSH Access" button also in home.html

---
 Dockerfile                                    |  1 +
 .../jupyterhub-mod/ssh-dialog-snippet.html    | 38 +++++++++++++++++++
 resources/jupyterhub-mod/template-admin.html  | 38 ++-----------------
 resources/jupyterhub-mod/template-home.html   |  7 +++-
 4 files changed, 49 insertions(+), 35 deletions(-)
 create mode 100644 resources/jupyterhub-mod/ssh-dialog-snippet.html

diff --git a/Dockerfile b/Dockerfile
index c127b31..155d389 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -133,6 +133,7 @@ COPY resources/logo.png /usr/local/share/jupyterhub/static/images/jupyter.png
 COPY resources/jupyterhub_config.py $_RESOURCES_PATH/jupyterhub_config.py
 COPY resources/jupyterhub-mod/template-home.html /usr/local/share/jupyterhub/templates/home.html
 COPY resources/jupyterhub-mod/template-admin.html /usr/local/share/jupyterhub/templates/admin.html
+COPY resources/jupyterhub-mod/ssh-dialog-snippet.html /usr/local/share/jupyterhub/templates/ssh-dialog-snippet.html
 COPY resources/jupyterhub-mod/info-dialog-snippet.html /usr/local/share/jupyterhub/templates/info-dialog-snippet.html
 COPY resources/jupyterhub-mod/jsonpresenter /usr/local/share/jupyterhub/static/components/jsonpresenter/
 COPY resources/jupyterhub-mod/cleanup-service.py /resources/cleanup-service.py
diff --git a/resources/jupyterhub-mod/ssh-dialog-snippet.html b/resources/jupyterhub-mod/ssh-dialog-snippet.html
new file mode 100644
index 0000000..6b65b24
--- /dev/null
+++ b/resources/jupyterhub-mod/ssh-dialog-snippet.html
@@ -0,0 +1,38 @@
+
+
+
+
+{% call modal('SSH Setup', btn_label="Copy to Clipboard", btn_class='btn-primary copy-clipboard-button') %}
+  Execute this command to setup the SSH connection to server : 
+  
+{% endcall %}
diff --git a/resources/jupyterhub-mod/template-admin.html b/resources/jupyterhub-mod/template-admin.html
index 3a07bbc..61fc45d 100644
--- a/resources/jupyterhub-mod/template-admin.html
+++ b/resources/jupyterhub-mod/template-admin.html
@@ -147,10 +147,7 @@
 {% endcall %}
 
 {# CUSTOM STUFF #}
-{% call modal('SSH Setup', btn_label="Copy to Clipboard", btn_class='btn-primary copy-clipboard-button') %}
-  Execute this command to setup the SSH connection to server : 
-  
-{% endcall %}
+
 
 {# END CUSTOM STUFF #}
 
@@ -183,38 +180,11 @@
 {% block script %}
 {{ super() }}
 
 
+{% include 'ssh-dialog-snippet.html' %}
+
 {% include 'info-dialog-snippet.html' %}
 
 {% endblock %}
diff --git a/resources/jupyterhub-mod/template-home.html b/resources/jupyterhub-mod/template-home.html
index 64ad196..a4627aa 100644
--- a/resources/jupyterhub-mod/template-home.html
+++ b/resources/jupyterhub-mod/template-home.html
@@ -83,7 +83,7 @@ 

         
       
       {% for spawner in named_spawners %}
-      
+      
         {# name #}
         {{ spawner.name }}
         {# url #}
@@ -115,6 +115,9 @@ 

           
             info
           
+          
+            ssh access
+          
         
       
       {% endfor %}
@@ -164,6 +167,8 @@ 

 });
 
 
+{% include 'ssh-dialog-snippet.html' %}
+
 {% include 'info-dialog-snippet.html' %}
 
 {% endblock %}

From db40c5cca45ed0cf1252b63bfe89056332fa6837 Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Wed, 13 Nov 2019 15:12:00 +0100
Subject: [PATCH 18/21] Modify Readme

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index bb72800..c30c985 100644
--- a/README.md
+++ b/README.md
@@ -127,6 +127,8 @@ Here are the additional environment variables for the hub:
 
 #### Jupyterhub Config
 
+##### Docker-local
+
 Jupyterhub itself is configured via a `config.py` file. In case of MLHub, a default config file is stored under `/resources/jupyterhub_config.py`. If you want to override settings or set extra ones, you can put another config file under `/resources/jupyterhub_user_config.py`. Following settings should probably not be overriden:
 - `c.Spawner.environment` - we set default variables there. Instead of overriding it, you can add extra variables to the existing dict, e.g. via `c.Spawner.environment["myvar"] = "myvalue"`.
 - `c.DockerSpawner.prefix` and `c.DockerSpawner.name_template` - if you change those, check whether your SSH environment variables permit those names a target. Also, think about setting `c.Authenticator.username_pattern` to prevent a user having a username that is also a valid container name.

From e1f0dd8d97054cc73eba788548ba0ce8a50bfec3 Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Wed, 13 Nov 2019 15:28:42 +0100
Subject: [PATCH 19/21] Add documentation about cleanup service

---
 README.md | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c30c985..b825e37 100644
--- a/README.md
+++ b/README.md
@@ -91,7 +91,9 @@ Here are the additional environment variables for the hub:
     
     
         CLEANUP_INTERVAL_SECONDS
-        Interval in which expired and not-used resources are deleted. Set to -1 to disable the automatic cleanup.
+        
+            Interval in which expired and not-used resources are deleted. Set to -1 to disable the automatic cleanup. For more information, see Section Cleanup Service.
+        
         3600
     
     
@@ -214,6 +216,15 @@ The "Days to live" flag is purely informational currently and can be seen in the
 
 Picture of admin panel
 
+### Cleanup Service
+
+MLHub contains a cleanup service that is started as a [JupyterHub service](https://jupyterhub.readthedocs.io/en/stable/reference/services.html) inside the hub container. It can be accessed as a REST-API by an admin, but it is also triggered automatically every X timesteps when not disabled (see config for `CLEANUP_INTERVAL_SECONDS`). The service enhances the JupyterHub functionality with regards to the Docker and Kubernetes world. "Containers" is hereby used interchangeably for Docker containers and Kubernetes pods.
+The service has two endpoints which can be reached under the Hub service url `/services/cleanup-service/*`.
+
+- `/services/cleanup-service/users`: This endpoint is currently doing anything only in Docker-local mode. There, it will check for resources of deleted users, so users who are not in the JupyterHub database anymore, and delete them. This includes containers, networks, and volumes. This is done by looking for labeled Docker resources that point to containers started by hub and belonging to the specific users.
+
+- `/services/cleanup-service/expired`: When starting a named workspace, an expiration date can be assigned to it. This endpoint will delete all containers that are expired. The respective named server is deleted from the JupyterHub database and also the Docker/Kubernetes resource is deleted.
+
 ## Contribution
 
 - Pull requests are encouraged and always welcome. Read [`CONTRIBUTING.md`](https://github.com/ml-tooling/ml-hub/tree/master/CONTRIBUTING.md) and check out [help-wanted](https://github.com/ml-tooling/ml-hub/issues?utf8=%E2%9C%93&q=is%3Aopen+is%3Aissue+label%3A"help+wanted"+sort%3Areactions-%2B1-desc+) issues.

From 37d4a21d40717c3179c64b6cd793d7aec239afb7 Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Wed, 13 Nov 2019 15:30:46 +0100
Subject: [PATCH 20/21] Enhance Readme

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index b825e37..3b0e805 100644
--- a/README.md
+++ b/README.md
@@ -219,11 +219,11 @@ The "Days to live" flag is purely informational currently and can be seen in the
 ### Cleanup Service
 
 MLHub contains a cleanup service that is started as a [JupyterHub service](https://jupyterhub.readthedocs.io/en/stable/reference/services.html) inside the hub container. It can be accessed as a REST-API by an admin, but it is also triggered automatically every X timesteps when not disabled (see config for `CLEANUP_INTERVAL_SECONDS`). The service enhances the JupyterHub functionality with regards to the Docker and Kubernetes world. "Containers" is hereby used interchangeably for Docker containers and Kubernetes pods.
-The service has two endpoints which can be reached under the Hub service url `/services/cleanup-service/*`.
+The service has two endpoints which can be reached under the Hub service url `/services/cleanup-service/*` with admin permissions.
 
-- `/services/cleanup-service/users`: This endpoint is currently doing anything only in Docker-local mode. There, it will check for resources of deleted users, so users who are not in the JupyterHub database anymore, and delete them. This includes containers, networks, and volumes. This is done by looking for labeled Docker resources that point to containers started by hub and belonging to the specific users.
+- `GET /services/cleanup-service/users`: This endpoint is currently doing anything only in Docker-local mode. There, it will check for resources of deleted users, so users who are not in the JupyterHub database anymore, and delete them. This includes containers, networks, and volumes. This is done by looking for labeled Docker resources that point to containers started by hub and belonging to the specific users.
 
-- `/services/cleanup-service/expired`: When starting a named workspace, an expiration date can be assigned to it. This endpoint will delete all containers that are expired. The respective named server is deleted from the JupyterHub database and also the Docker/Kubernetes resource is deleted.
+- `GET /services/cleanup-service/expired`: When starting a named workspace, an expiration date can be assigned to it. This endpoint will delete all containers that are expired. The respective named server is deleted from the JupyterHub database and also the Docker/Kubernetes resource is deleted.
 
 ## Contribution
 

From 4129e1c94057b342fd68020a3996ed0db567caa1 Mon Sep 17 00:00:00 2001
From: Benjamin Raethlein 
Date: Wed, 13 Nov 2019 15:37:39 +0100
Subject: [PATCH 21/21] Add reason why we created the cleanup service

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3b0e805..158abe7 100644
--- a/README.md
+++ b/README.md
@@ -33,6 +33,7 @@ MLHub is based on [Jupyterhub](https://github.com/jupyterhub/jupyterhub) with co
 - 🖊️ Set configuration parameters such as CPU-limits for started workspaces. 
 - 🖥 Access additional tools within the started workspaces by having secured routes.
 - 🎛 Tunnel SSH connections to workspace containers.
+- 🐳 Focused on Docker and Kubernetes with enhanced functionality.
 
 ## Getting Started
 
@@ -218,7 +219,7 @@ The "Days to live" flag is purely informational currently and can be seen in the
 
 ### Cleanup Service
 
-MLHub contains a cleanup service that is started as a [JupyterHub service](https://jupyterhub.readthedocs.io/en/stable/reference/services.html) inside the hub container. It can be accessed as a REST-API by an admin, but it is also triggered automatically every X timesteps when not disabled (see config for `CLEANUP_INTERVAL_SECONDS`). The service enhances the JupyterHub functionality with regards to the Docker and Kubernetes world. "Containers" is hereby used interchangeably for Docker containers and Kubernetes pods.
+JupyterHub was originally not created with Docker or Kubernetes in mind, which can result in unfavorable scenarios such as that containers are stopped but not deleted on the host. Furthermore, our custom spawners might create some artifacts that should be cleaned up as well. MLHub contains a cleanup service that is started as a [JupyterHub service](https://jupyterhub.readthedocs.io/en/stable/reference/services.html) inside the hub container. It can be accessed as a REST-API by an admin, but it is also triggered automatically every X timesteps when not disabled (see config for `CLEANUP_INTERVAL_SECONDS`). The service enhances the JupyterHub functionality with regards to the Docker and Kubernetes world. "Containers" is hereby used interchangeably for Docker containers and Kubernetes pods.
 The service has two endpoints which can be reached under the Hub service url `/services/cleanup-service/*` with admin permissions.
 
 - `GET /services/cleanup-service/users`: This endpoint is currently doing anything only in Docker-local mode. There, it will check for resources of deleted users, so users who are not in the JupyterHub database anymore, and delete them. This includes containers, networks, and volumes. This is done by looking for labeled Docker resources that point to containers started by hub and belonging to the specific users.