From 3d4c20b683e298e6bcf321c3cfbf76665345afab Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Wed, 27 Mar 2019 08:56:21 -0700 Subject: [PATCH 01/19] Upgrade to JupyterHub 1.0.0 beta 1 --- jupyter-base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter-base/Dockerfile b/jupyter-base/Dockerfile index fa436b6..f9ab8a5 100644 --- a/jupyter-base/Dockerfile +++ b/jupyter-base/Dockerfile @@ -57,7 +57,7 @@ RUN \ npm install -g configurable-http-proxy && \ git clone https://github.com/jupyterhub/jupyterhub.git && \ cd jupyterhub && \ -# git checkout tags/0.9.4 && \ + git checkout tags/1.0.0b1 && \ /opt/anaconda3/bin/python setup.py js && \ /opt/anaconda3/bin/pip --no-cache-dir install . && \ cp examples/cull-idle/cull_idle_servers.py /opt/anaconda3/bin/. && \ From 1b43988e5a8ffd6ab76cf3bf457f589f3829ba67 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Wed, 27 Mar 2019 13:01:33 -0700 Subject: [PATCH 02/19] Allow user to specify host to test against --- jupyter-nersc/web/hub-scripts/test-user.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter-nersc/web/hub-scripts/test-user.sh b/jupyter-nersc/web/hub-scripts/test-user.sh index 5dbab43..1f3c0ca 100644 --- a/jupyter-nersc/web/hub-scripts/test-user.sh +++ b/jupyter-nersc/web/hub-scripts/test-user.sh @@ -2,8 +2,8 @@ # Test user's ability to ssh -hostname=app -username=$1 +hostname=$1 +username=$2 cert=/certs/$username.key echo $username $cert if [ ! -f $cert ]; then From da19acfff0b169b72a7d7bd5b37b54236cc3567e Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Wed, 27 Mar 2019 13:10:10 -0700 Subject: [PATCH 03/19] Align with Spin service naming conventions --- jupyter-nersc/{mods => app-monitoring}/Dockerfile | 0 jupyter-nersc/{mods => app-monitoring}/build.sh | 2 +- .../{mods => app-monitoring}/docker-entrypoint.sh | 0 jupyter-nersc/{mods => app-monitoring}/monitor.py | 0 jupyter-nersc/{app => app-notebooks}/Dockerfile | 0 jupyter-nersc/{app => app-notebooks}/NERSC-keys-api | 0 jupyter-nersc/{web => app-notebooks}/build.sh | 2 +- .../{app => app-notebooks}/docker-entrypoint.sh | 0 jupyter-nersc/{app => app-notebooks}/etc/ldap.conf | 0 jupyter-nersc/{app => app-notebooks}/etc/nslcd.conf | 0 .../{app => app-notebooks}/etc/nsswitch.conf | 0 .../{app => app-notebooks}/etc/pam.d/jupyterhub | 0 .../{app => app-notebooks}/etc/pam.d/password-auth | 0 .../{app => app-notebooks}/etc/user_ca.pub | 0 jupyter-nersc/{app => app-notebooks}/get_port.py | 0 jupyter-nersc/{app => app-notebooks}/packages2.txt | 0 jupyter-nersc/{app => app-notebooks}/packages3.txt | 0 .../{app => app-notebooks}/supervisord.conf | 0 jupyter-nersc/{web => web-hub}/Dockerfile | 0 jupyter-nersc/{web => web-hub}/announcement.py | 0 jupyter-nersc/{app => web-hub}/build.sh | 2 +- jupyter-nersc/{web => web-hub}/docker-entrypoint.sh | 0 .../{web => web-hub}/hub-scripts/flush-certs.sh | 0 .../{web => web-hub}/hub-scripts/test-user.sh | 0 jupyter-nersc/{web => web-hub}/jupyterhub_config.py | 0 jupyter-nersc/{web => web-hub}/templates/error.html | 0 jupyter-nersc/{web => web-hub}/templates/home.html | 0 jupyter-nersc/{web => web-hub}/templates/login.html | 0 jupyter-nersc/{web => web-hub}/templates/page.html | 0 jupyter-nersc/{off => web-offline}/Dockerfile | 0 jupyter-nersc/{off => web-offline}/README.md | 0 jupyter-nersc/{off => web-offline}/app.py | 0 jupyter-nersc/{off => web-offline}/build.sh | 2 +- jupyter-nersc/{off => web-offline}/static/logo.png | Bin .../rectanglelogo-greytext-orangebody-greymoons.png | Bin .../{off => web-offline}/templates/index.html | 0 36 files changed, 4 insertions(+), 4 deletions(-) rename jupyter-nersc/{mods => app-monitoring}/Dockerfile (100%) rename jupyter-nersc/{mods => app-monitoring}/build.sh (51%) rename jupyter-nersc/{mods => app-monitoring}/docker-entrypoint.sh (100%) rename jupyter-nersc/{mods => app-monitoring}/monitor.py (100%) rename jupyter-nersc/{app => app-notebooks}/Dockerfile (100%) rename jupyter-nersc/{app => app-notebooks}/NERSC-keys-api (100%) rename jupyter-nersc/{web => app-notebooks}/build.sh (65%) rename jupyter-nersc/{app => app-notebooks}/docker-entrypoint.sh (100%) rename jupyter-nersc/{app => app-notebooks}/etc/ldap.conf (100%) rename jupyter-nersc/{app => app-notebooks}/etc/nslcd.conf (100%) rename jupyter-nersc/{app => app-notebooks}/etc/nsswitch.conf (100%) rename jupyter-nersc/{app => app-notebooks}/etc/pam.d/jupyterhub (100%) rename jupyter-nersc/{app => app-notebooks}/etc/pam.d/password-auth (100%) rename jupyter-nersc/{app => app-notebooks}/etc/user_ca.pub (100%) rename jupyter-nersc/{app => app-notebooks}/get_port.py (100%) rename jupyter-nersc/{app => app-notebooks}/packages2.txt (100%) rename jupyter-nersc/{app => app-notebooks}/packages3.txt (100%) rename jupyter-nersc/{app => app-notebooks}/supervisord.conf (100%) rename jupyter-nersc/{web => web-hub}/Dockerfile (100%) rename jupyter-nersc/{web => web-hub}/announcement.py (100%) rename jupyter-nersc/{app => web-hub}/build.sh (66%) rename jupyter-nersc/{web => web-hub}/docker-entrypoint.sh (100%) rename jupyter-nersc/{web => web-hub}/hub-scripts/flush-certs.sh (100%) rename jupyter-nersc/{web => web-hub}/hub-scripts/test-user.sh (100%) rename jupyter-nersc/{web => web-hub}/jupyterhub_config.py (100%) rename jupyter-nersc/{web => web-hub}/templates/error.html (100%) rename jupyter-nersc/{web => web-hub}/templates/home.html (100%) rename jupyter-nersc/{web => web-hub}/templates/login.html (100%) rename jupyter-nersc/{web => web-hub}/templates/page.html (100%) rename jupyter-nersc/{off => web-offline}/Dockerfile (100%) rename jupyter-nersc/{off => web-offline}/README.md (100%) rename jupyter-nersc/{off => web-offline}/app.py (100%) rename jupyter-nersc/{off => web-offline}/build.sh (65%) rename jupyter-nersc/{off => web-offline}/static/logo.png (100%) rename jupyter-nersc/{off => web-offline}/static/rectanglelogo-greytext-orangebody-greymoons.png (100%) rename jupyter-nersc/{off => web-offline}/templates/index.html (100%) diff --git a/jupyter-nersc/mods/Dockerfile b/jupyter-nersc/app-monitoring/Dockerfile similarity index 100% rename from jupyter-nersc/mods/Dockerfile rename to jupyter-nersc/app-monitoring/Dockerfile diff --git a/jupyter-nersc/mods/build.sh b/jupyter-nersc/app-monitoring/build.sh similarity index 51% rename from jupyter-nersc/mods/build.sh rename to jupyter-nersc/app-monitoring/build.sh index e1fbce5..a3a4efc 100644 --- a/jupyter-nersc/mods/build.sh +++ b/jupyter-nersc/app-monitoring/build.sh @@ -4,4 +4,4 @@ branch=$(git symbolic-ref --short HEAD) docker build \ "$@" \ - --tag registry.spin.nersc.gov/das/jupyter-nersc-mods.$branch:latest . + --tag registry.spin.nersc.gov/das/app-monitoring.jupyter-nersc-$branch:latest . diff --git a/jupyter-nersc/mods/docker-entrypoint.sh b/jupyter-nersc/app-monitoring/docker-entrypoint.sh similarity index 100% rename from jupyter-nersc/mods/docker-entrypoint.sh rename to jupyter-nersc/app-monitoring/docker-entrypoint.sh diff --git a/jupyter-nersc/mods/monitor.py b/jupyter-nersc/app-monitoring/monitor.py similarity index 100% rename from jupyter-nersc/mods/monitor.py rename to jupyter-nersc/app-monitoring/monitor.py diff --git a/jupyter-nersc/app/Dockerfile b/jupyter-nersc/app-notebooks/Dockerfile similarity index 100% rename from jupyter-nersc/app/Dockerfile rename to jupyter-nersc/app-notebooks/Dockerfile diff --git a/jupyter-nersc/app/NERSC-keys-api b/jupyter-nersc/app-notebooks/NERSC-keys-api similarity index 100% rename from jupyter-nersc/app/NERSC-keys-api rename to jupyter-nersc/app-notebooks/NERSC-keys-api diff --git a/jupyter-nersc/web/build.sh b/jupyter-nersc/app-notebooks/build.sh similarity index 65% rename from jupyter-nersc/web/build.sh rename to jupyter-nersc/app-notebooks/build.sh index b136616..8fec9e1 100644 --- a/jupyter-nersc/web/build.sh +++ b/jupyter-nersc/app-notebooks/build.sh @@ -5,4 +5,4 @@ branch=$(git symbolic-ref --short HEAD) docker build \ --build-arg branch=$branch \ "$@" \ - --tag registry.spin.nersc.gov/das/jupyter-nersc-web.$branch:latest . + --tag registry.spin.nersc.gov/das/app-notebooks.jupyter-nersc-$branch:latest . diff --git a/jupyter-nersc/app/docker-entrypoint.sh b/jupyter-nersc/app-notebooks/docker-entrypoint.sh similarity index 100% rename from jupyter-nersc/app/docker-entrypoint.sh rename to jupyter-nersc/app-notebooks/docker-entrypoint.sh diff --git a/jupyter-nersc/app/etc/ldap.conf b/jupyter-nersc/app-notebooks/etc/ldap.conf similarity index 100% rename from jupyter-nersc/app/etc/ldap.conf rename to jupyter-nersc/app-notebooks/etc/ldap.conf diff --git a/jupyter-nersc/app/etc/nslcd.conf b/jupyter-nersc/app-notebooks/etc/nslcd.conf similarity index 100% rename from jupyter-nersc/app/etc/nslcd.conf rename to jupyter-nersc/app-notebooks/etc/nslcd.conf diff --git a/jupyter-nersc/app/etc/nsswitch.conf b/jupyter-nersc/app-notebooks/etc/nsswitch.conf similarity index 100% rename from jupyter-nersc/app/etc/nsswitch.conf rename to jupyter-nersc/app-notebooks/etc/nsswitch.conf diff --git a/jupyter-nersc/app/etc/pam.d/jupyterhub b/jupyter-nersc/app-notebooks/etc/pam.d/jupyterhub similarity index 100% rename from jupyter-nersc/app/etc/pam.d/jupyterhub rename to jupyter-nersc/app-notebooks/etc/pam.d/jupyterhub diff --git a/jupyter-nersc/app/etc/pam.d/password-auth b/jupyter-nersc/app-notebooks/etc/pam.d/password-auth similarity index 100% rename from jupyter-nersc/app/etc/pam.d/password-auth rename to jupyter-nersc/app-notebooks/etc/pam.d/password-auth diff --git a/jupyter-nersc/app/etc/user_ca.pub b/jupyter-nersc/app-notebooks/etc/user_ca.pub similarity index 100% rename from jupyter-nersc/app/etc/user_ca.pub rename to jupyter-nersc/app-notebooks/etc/user_ca.pub diff --git a/jupyter-nersc/app/get_port.py b/jupyter-nersc/app-notebooks/get_port.py similarity index 100% rename from jupyter-nersc/app/get_port.py rename to jupyter-nersc/app-notebooks/get_port.py diff --git a/jupyter-nersc/app/packages2.txt b/jupyter-nersc/app-notebooks/packages2.txt similarity index 100% rename from jupyter-nersc/app/packages2.txt rename to jupyter-nersc/app-notebooks/packages2.txt diff --git a/jupyter-nersc/app/packages3.txt b/jupyter-nersc/app-notebooks/packages3.txt similarity index 100% rename from jupyter-nersc/app/packages3.txt rename to jupyter-nersc/app-notebooks/packages3.txt diff --git a/jupyter-nersc/app/supervisord.conf b/jupyter-nersc/app-notebooks/supervisord.conf similarity index 100% rename from jupyter-nersc/app/supervisord.conf rename to jupyter-nersc/app-notebooks/supervisord.conf diff --git a/jupyter-nersc/web/Dockerfile b/jupyter-nersc/web-hub/Dockerfile similarity index 100% rename from jupyter-nersc/web/Dockerfile rename to jupyter-nersc/web-hub/Dockerfile diff --git a/jupyter-nersc/web/announcement.py b/jupyter-nersc/web-hub/announcement.py similarity index 100% rename from jupyter-nersc/web/announcement.py rename to jupyter-nersc/web-hub/announcement.py diff --git a/jupyter-nersc/app/build.sh b/jupyter-nersc/web-hub/build.sh similarity index 66% rename from jupyter-nersc/app/build.sh rename to jupyter-nersc/web-hub/build.sh index b408866..9b109cd 100644 --- a/jupyter-nersc/app/build.sh +++ b/jupyter-nersc/web-hub/build.sh @@ -5,4 +5,4 @@ branch=$(git symbolic-ref --short HEAD) docker build \ --build-arg branch=$branch \ "$@" \ - --tag registry.spin.nersc.gov/das/jupyter-nersc-app.$branch:latest . + --tag registry.spin.nersc.gov/das/web-hub.jupyter-nersc-$branch:latest . diff --git a/jupyter-nersc/web/docker-entrypoint.sh b/jupyter-nersc/web-hub/docker-entrypoint.sh similarity index 100% rename from jupyter-nersc/web/docker-entrypoint.sh rename to jupyter-nersc/web-hub/docker-entrypoint.sh diff --git a/jupyter-nersc/web/hub-scripts/flush-certs.sh b/jupyter-nersc/web-hub/hub-scripts/flush-certs.sh similarity index 100% rename from jupyter-nersc/web/hub-scripts/flush-certs.sh rename to jupyter-nersc/web-hub/hub-scripts/flush-certs.sh diff --git a/jupyter-nersc/web/hub-scripts/test-user.sh b/jupyter-nersc/web-hub/hub-scripts/test-user.sh similarity index 100% rename from jupyter-nersc/web/hub-scripts/test-user.sh rename to jupyter-nersc/web-hub/hub-scripts/test-user.sh diff --git a/jupyter-nersc/web/jupyterhub_config.py b/jupyter-nersc/web-hub/jupyterhub_config.py similarity index 100% rename from jupyter-nersc/web/jupyterhub_config.py rename to jupyter-nersc/web-hub/jupyterhub_config.py diff --git a/jupyter-nersc/web/templates/error.html b/jupyter-nersc/web-hub/templates/error.html similarity index 100% rename from jupyter-nersc/web/templates/error.html rename to jupyter-nersc/web-hub/templates/error.html diff --git a/jupyter-nersc/web/templates/home.html b/jupyter-nersc/web-hub/templates/home.html similarity index 100% rename from jupyter-nersc/web/templates/home.html rename to jupyter-nersc/web-hub/templates/home.html diff --git a/jupyter-nersc/web/templates/login.html b/jupyter-nersc/web-hub/templates/login.html similarity index 100% rename from jupyter-nersc/web/templates/login.html rename to jupyter-nersc/web-hub/templates/login.html diff --git a/jupyter-nersc/web/templates/page.html b/jupyter-nersc/web-hub/templates/page.html similarity index 100% rename from jupyter-nersc/web/templates/page.html rename to jupyter-nersc/web-hub/templates/page.html diff --git a/jupyter-nersc/off/Dockerfile b/jupyter-nersc/web-offline/Dockerfile similarity index 100% rename from jupyter-nersc/off/Dockerfile rename to jupyter-nersc/web-offline/Dockerfile diff --git a/jupyter-nersc/off/README.md b/jupyter-nersc/web-offline/README.md similarity index 100% rename from jupyter-nersc/off/README.md rename to jupyter-nersc/web-offline/README.md diff --git a/jupyter-nersc/off/app.py b/jupyter-nersc/web-offline/app.py similarity index 100% rename from jupyter-nersc/off/app.py rename to jupyter-nersc/web-offline/app.py diff --git a/jupyter-nersc/off/build.sh b/jupyter-nersc/web-offline/build.sh similarity index 65% rename from jupyter-nersc/off/build.sh rename to jupyter-nersc/web-offline/build.sh index f21b653..b427ff9 100644 --- a/jupyter-nersc/off/build.sh +++ b/jupyter-nersc/web-offline/build.sh @@ -5,4 +5,4 @@ branch=$(git symbolic-ref --short HEAD) docker build \ --build-arg branch=$branch \ "$@" \ - --tag registry.spin.nersc.gov/das/jupyter-nersc-off.$branch:latest . + --tag registry.spin.nersc.gov/das/web-offline.jupyter-nersc-$branch:latest . diff --git a/jupyter-nersc/off/static/logo.png b/jupyter-nersc/web-offline/static/logo.png similarity index 100% rename from jupyter-nersc/off/static/logo.png rename to jupyter-nersc/web-offline/static/logo.png diff --git a/jupyter-nersc/off/static/rectanglelogo-greytext-orangebody-greymoons.png b/jupyter-nersc/web-offline/static/rectanglelogo-greytext-orangebody-greymoons.png similarity index 100% rename from jupyter-nersc/off/static/rectanglelogo-greytext-orangebody-greymoons.png rename to jupyter-nersc/web-offline/static/rectanglelogo-greytext-orangebody-greymoons.png diff --git a/jupyter-nersc/off/templates/index.html b/jupyter-nersc/web-offline/templates/index.html similarity index 100% rename from jupyter-nersc/off/templates/index.html rename to jupyter-nersc/web-offline/templates/index.html From 85f38b99d3d118b6bc3d8ef8d249a2a7f4338d12 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Wed, 27 Mar 2019 13:11:23 -0700 Subject: [PATCH 04/19] Need to change app service host name --- jupyter-nersc/web-hub/jupyterhub_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter-nersc/web-hub/jupyterhub_config.py b/jupyter-nersc/web-hub/jupyterhub_config.py index 45f7320..38b1cb3 100644 --- a/jupyter-nersc/web-hub/jupyterhub_config.py +++ b/jupyter-nersc/web-hub/jupyterhub_config.py @@ -1022,7 +1022,7 @@ def comma_split(string): "sshspawner.sshspawner.SSHSpawner", { "cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh", "/opt/anaconda3/bin/jupyter-labhub"], - "remote_hosts": ["app"], + "remote_hosts": ["app-notebooks"], "remote_port_command": "/opt/anaconda3/bin/python /global/common/cori/das/jupyterhub/new-get-port.py --ip", "hub_api_url": "http://{}:8081/hub/api".format(ip), "path": "/opt/anaconda3/bin:/usr/bin:/usr/local/bin:/bin", From 559aeb92db238f1d31d94e98a89a82d1b0ceaac4 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Wed, 27 Mar 2019 13:12:53 -0700 Subject: [PATCH 05/19] Pedantic change --- jupyter-nersc/{web-hub => web-jupyterhub}/Dockerfile | 0 jupyter-nersc/{web-hub => web-jupyterhub}/announcement.py | 0 jupyter-nersc/{web-hub => web-jupyterhub}/build.sh | 0 jupyter-nersc/{web-hub => web-jupyterhub}/docker-entrypoint.sh | 0 .../{web-hub => web-jupyterhub}/hub-scripts/flush-certs.sh | 0 .../{web-hub => web-jupyterhub}/hub-scripts/test-user.sh | 0 jupyter-nersc/{web-hub => web-jupyterhub}/jupyterhub_config.py | 0 jupyter-nersc/{web-hub => web-jupyterhub}/templates/error.html | 0 jupyter-nersc/{web-hub => web-jupyterhub}/templates/home.html | 0 jupyter-nersc/{web-hub => web-jupyterhub}/templates/login.html | 0 jupyter-nersc/{web-hub => web-jupyterhub}/templates/page.html | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename jupyter-nersc/{web-hub => web-jupyterhub}/Dockerfile (100%) rename jupyter-nersc/{web-hub => web-jupyterhub}/announcement.py (100%) rename jupyter-nersc/{web-hub => web-jupyterhub}/build.sh (100%) rename jupyter-nersc/{web-hub => web-jupyterhub}/docker-entrypoint.sh (100%) rename jupyter-nersc/{web-hub => web-jupyterhub}/hub-scripts/flush-certs.sh (100%) rename jupyter-nersc/{web-hub => web-jupyterhub}/hub-scripts/test-user.sh (100%) rename jupyter-nersc/{web-hub => web-jupyterhub}/jupyterhub_config.py (100%) rename jupyter-nersc/{web-hub => web-jupyterhub}/templates/error.html (100%) rename jupyter-nersc/{web-hub => web-jupyterhub}/templates/home.html (100%) rename jupyter-nersc/{web-hub => web-jupyterhub}/templates/login.html (100%) rename jupyter-nersc/{web-hub => web-jupyterhub}/templates/page.html (100%) diff --git a/jupyter-nersc/web-hub/Dockerfile b/jupyter-nersc/web-jupyterhub/Dockerfile similarity index 100% rename from jupyter-nersc/web-hub/Dockerfile rename to jupyter-nersc/web-jupyterhub/Dockerfile diff --git a/jupyter-nersc/web-hub/announcement.py b/jupyter-nersc/web-jupyterhub/announcement.py similarity index 100% rename from jupyter-nersc/web-hub/announcement.py rename to jupyter-nersc/web-jupyterhub/announcement.py diff --git a/jupyter-nersc/web-hub/build.sh b/jupyter-nersc/web-jupyterhub/build.sh similarity index 100% rename from jupyter-nersc/web-hub/build.sh rename to jupyter-nersc/web-jupyterhub/build.sh diff --git a/jupyter-nersc/web-hub/docker-entrypoint.sh b/jupyter-nersc/web-jupyterhub/docker-entrypoint.sh similarity index 100% rename from jupyter-nersc/web-hub/docker-entrypoint.sh rename to jupyter-nersc/web-jupyterhub/docker-entrypoint.sh diff --git a/jupyter-nersc/web-hub/hub-scripts/flush-certs.sh b/jupyter-nersc/web-jupyterhub/hub-scripts/flush-certs.sh similarity index 100% rename from jupyter-nersc/web-hub/hub-scripts/flush-certs.sh rename to jupyter-nersc/web-jupyterhub/hub-scripts/flush-certs.sh diff --git a/jupyter-nersc/web-hub/hub-scripts/test-user.sh b/jupyter-nersc/web-jupyterhub/hub-scripts/test-user.sh similarity index 100% rename from jupyter-nersc/web-hub/hub-scripts/test-user.sh rename to jupyter-nersc/web-jupyterhub/hub-scripts/test-user.sh diff --git a/jupyter-nersc/web-hub/jupyterhub_config.py b/jupyter-nersc/web-jupyterhub/jupyterhub_config.py similarity index 100% rename from jupyter-nersc/web-hub/jupyterhub_config.py rename to jupyter-nersc/web-jupyterhub/jupyterhub_config.py diff --git a/jupyter-nersc/web-hub/templates/error.html b/jupyter-nersc/web-jupyterhub/templates/error.html similarity index 100% rename from jupyter-nersc/web-hub/templates/error.html rename to jupyter-nersc/web-jupyterhub/templates/error.html diff --git a/jupyter-nersc/web-hub/templates/home.html b/jupyter-nersc/web-jupyterhub/templates/home.html similarity index 100% rename from jupyter-nersc/web-hub/templates/home.html rename to jupyter-nersc/web-jupyterhub/templates/home.html diff --git a/jupyter-nersc/web-hub/templates/login.html b/jupyter-nersc/web-jupyterhub/templates/login.html similarity index 100% rename from jupyter-nersc/web-hub/templates/login.html rename to jupyter-nersc/web-jupyterhub/templates/login.html diff --git a/jupyter-nersc/web-hub/templates/page.html b/jupyter-nersc/web-jupyterhub/templates/page.html similarity index 100% rename from jupyter-nersc/web-hub/templates/page.html rename to jupyter-nersc/web-jupyterhub/templates/page.html From 687c67453a2a404f4e575c5ee5fd3336116e3588 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Wed, 27 Mar 2019 13:13:42 -0700 Subject: [PATCH 06/19] Changed to point at new name --- jupyter-nersc/app-monitoring/monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter-nersc/app-monitoring/monitor.py b/jupyter-nersc/app-monitoring/monitor.py index 9f8e3a7..e94b7a2 100644 --- a/jupyter-nersc/app-monitoring/monitor.py +++ b/jupyter-nersc/app-monitoring/monitor.py @@ -56,7 +56,7 @@ def parse_arguments(): parser.add_argument("--queue", "-q", default="text") parser.add_argument("--api-url", "-a", - default="http://web:8081/hub/api") + default="http://web-jupyterhub:8081/hub/api") parser.add_argument("--api-token", "-o", default=os.environ["MODS_JUPYTERHUB_API_TOKEN"]) parser.add_argument("--subcategory", "-s", From d099b7f21b1bb09eb349e2ebe5bc64decc9dca5c Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Wed, 27 Mar 2019 13:16:56 -0700 Subject: [PATCH 07/19] Foolish consistency --- jupyter-base/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter-base/build.sh b/jupyter-base/build.sh index bcae179..e6265ec 100644 --- a/jupyter-base/build.sh +++ b/jupyter-base/build.sh @@ -4,4 +4,4 @@ branch=$(git symbolic-ref --short HEAD) docker build \ --no-cache \ - --tag registry.spin.nersc.gov/das/jupyter-base.$branch:latest . + --tag registry.spin.nersc.gov/das/jupyter-base-$branch:latest . From cd4956828cd3598a3d08d40698cd386ff67fd8f6 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Wed, 27 Mar 2019 13:17:39 -0700 Subject: [PATCH 08/19] Follow base image name --- jupyter-nersc/app-notebooks/Dockerfile | 2 +- jupyter-nersc/web-jupyterhub/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter-nersc/app-notebooks/Dockerfile b/jupyter-nersc/app-notebooks/Dockerfile index 803d685..e0dca1a 100644 --- a/jupyter-nersc/app-notebooks/Dockerfile +++ b/jupyter-nersc/app-notebooks/Dockerfile @@ -1,6 +1,6 @@ ARG branch=unknown -FROM registry.spin.nersc.gov/das/jupyter-base.${branch}:latest +FROM registry.spin.nersc.gov/das/jupyter-base-${branch}:latest LABEL maintainer="Rollin Thomas " WORKDIR /srv diff --git a/jupyter-nersc/web-jupyterhub/Dockerfile b/jupyter-nersc/web-jupyterhub/Dockerfile index 41354a0..06eeb37 100644 --- a/jupyter-nersc/web-jupyterhub/Dockerfile +++ b/jupyter-nersc/web-jupyterhub/Dockerfile @@ -1,6 +1,6 @@ ARG branch=unknown -FROM registry.spin.nersc.gov/das/jupyter-base.${branch}:latest +FROM registry.spin.nersc.gov/das/jupyter-base-${branch}:latest LABEL maintainer="Rollin Thomas " WORKDIR /srv From 2139dc238339e6b8497e451fd98dc1f4147a4bb6 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Thu, 28 Mar 2019 08:11:07 -0700 Subject: [PATCH 09/19] Packages from conda, reset tag to 0.9.5 --- jupyter-base/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jupyter-base/Dockerfile b/jupyter-base/Dockerfile index f9ab8a5..69d98d2 100644 --- a/jupyter-base/Dockerfile +++ b/jupyter-base/Dockerfile @@ -37,10 +37,13 @@ RUN \ /opt/anaconda3/bin/conda install --yes \ alembic \ decorator \ + entrypoints \ jinja2 \ mako \ markupsafe \ nodejs \ + oauthlib=2 \ + pamela \ psycopg2 \ pyopenssl \ python-dateutil \ @@ -57,7 +60,7 @@ RUN \ npm install -g configurable-http-proxy && \ git clone https://github.com/jupyterhub/jupyterhub.git && \ cd jupyterhub && \ - git checkout tags/1.0.0b1 && \ + git checkout tags/0.9.5 && \ /opt/anaconda3/bin/python setup.py js && \ /opt/anaconda3/bin/pip --no-cache-dir install . && \ cp examples/cull-idle/cull_idle_servers.py /opt/anaconda3/bin/. && \ From 839324fbccfe6bd0b0db969b8db53a77e401b80b Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Sat, 30 Mar 2019 08:41:43 -0700 Subject: [PATCH 10/19] Change cull idle timeout to 18 hours --- jupyter-nersc/web-jupyterhub/jupyterhub_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter-nersc/web-jupyterhub/jupyterhub_config.py b/jupyter-nersc/web-jupyterhub/jupyterhub_config.py index 38b1cb3..ff17397 100644 --- a/jupyter-nersc/web-jupyterhub/jupyterhub_config.py +++ b/jupyter-nersc/web-jupyterhub/jupyterhub_config.py @@ -183,7 +183,7 @@ def comma_split(string): ## url for the database. e.g. `sqlite:///jupyterhub.sqlite` #c.JupyterHub.db_url = 'sqlite:///jupyterhub.sqlite' -c.JupyterHub.db_url = 'postgresql://jupyterhub:{}@db:5432/jupyterhub'.format( +c.JupyterHub.db_url = 'postgresql://jupyterhub:{}@db-jupyterhub:5432/jupyterhub'.format( os.getenv('POSTGRES_PASSWORD') ) @@ -388,7 +388,7 @@ def comma_split(string): { 'name': 'cull-idle', 'admin': True, - 'command': 'cull_idle_servers.py --timeout=43200'.split(), + 'command': 'cull_idle_servers.py --timeout=64800'.split(), }, { 'name': 'announcement', From 7e1109826f0f2458c3fcc324aa40291ecc97739a Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Sat, 30 Mar 2019 08:42:17 -0700 Subject: [PATCH 11/19] Stick to master for now --- jupyter-base/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/jupyter-base/Dockerfile b/jupyter-base/Dockerfile index 69d98d2..ee00a48 100644 --- a/jupyter-base/Dockerfile +++ b/jupyter-base/Dockerfile @@ -60,7 +60,6 @@ RUN \ npm install -g configurable-http-proxy && \ git clone https://github.com/jupyterhub/jupyterhub.git && \ cd jupyterhub && \ - git checkout tags/0.9.5 && \ /opt/anaconda3/bin/python setup.py js && \ /opt/anaconda3/bin/pip --no-cache-dir install . && \ cp examples/cull-idle/cull_idle_servers.py /opt/anaconda3/bin/. && \ From e66102c42f8abc36de739e87d8338d649127b5bb Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Sat, 30 Mar 2019 08:42:46 -0700 Subject: [PATCH 12/19] Require notebook 5.7.7 --- jupyter-nersc/app-notebooks/packages2.txt | 1 + jupyter-nersc/app-notebooks/packages3.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/jupyter-nersc/app-notebooks/packages2.txt b/jupyter-nersc/app-notebooks/packages2.txt index 6f668cf..a9b84d3 100644 --- a/jupyter-nersc/app-notebooks/packages2.txt +++ b/jupyter-nersc/app-notebooks/packages2.txt @@ -2,6 +2,7 @@ basemap beautiful-soup biopython netcdf4 +notebook=5.7.7 r-car r-essentials rpy2 diff --git a/jupyter-nersc/app-notebooks/packages3.txt b/jupyter-nersc/app-notebooks/packages3.txt index 23ad40c..bce3ee0 100644 --- a/jupyter-nersc/app-notebooks/packages3.txt +++ b/jupyter-nersc/app-notebooks/packages3.txt @@ -5,6 +5,7 @@ blaze jupyter jupyterlab netcdf4 +notebook=5.7.7 r-essentials rpy2 scipy From 95145998ae053f18e0e2f65f93fe88e8fb929d4d Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Sat, 30 Mar 2019 08:43:14 -0700 Subject: [PATCH 13/19] Fix image name --- jupyter-nersc/web-jupyterhub/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter-nersc/web-jupyterhub/build.sh b/jupyter-nersc/web-jupyterhub/build.sh index 9b109cd..b0340fc 100644 --- a/jupyter-nersc/web-jupyterhub/build.sh +++ b/jupyter-nersc/web-jupyterhub/build.sh @@ -5,4 +5,4 @@ branch=$(git symbolic-ref --short HEAD) docker build \ --build-arg branch=$branch \ "$@" \ - --tag registry.spin.nersc.gov/das/web-hub.jupyter-nersc-$branch:latest . + --tag registry.spin.nersc.gov/das/web-jupyterhub.jupyter-nersc-$branch:latest . From 45b9f6b807ee716110b117f2471d3a1396253732 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Sat, 30 Mar 2019 08:49:12 -0700 Subject: [PATCH 14/19] Add back pre-spawn hook myquota check This got dropped when we merged hubs because the base config was the old jupyter-nersc one. --- .../web-jupyterhub/jupyterhub_config.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/jupyter-nersc/web-jupyterhub/jupyterhub_config.py b/jupyter-nersc/web-jupyterhub/jupyterhub_config.py index ff17397..9cf72fa 100644 --- a/jupyter-nersc/web-jupyterhub/jupyterhub_config.py +++ b/jupyter-nersc/web-jupyterhub/jupyterhub_config.py @@ -3,6 +3,7 @@ import os import sys +import asyncssh import requests def comma_split(string): @@ -1031,6 +1032,41 @@ def comma_split(string): ) } +# Pre-spawn myquota check + +def space_error(home): + """Extra message pointing users to try spawning again from /hub/home. """ + home = url_path_join(home, 'home') + return ( + "There is insufficient space in your home directory; please clear up some files and then " + "navigate to the hub home and start your server.".format(home=home) + ) + +async def setup(spawner): + username = spawner.user.name + remote_host = "corijupyter.nersc.gov" + keyfile = spawner.ssh_keyfile.format(username=username) + certfile = keyfile + "-cert.pub" + k = asyncssh.read_private_key(keyfile) + c = asyncssh.read_certificate(certfile) + # print(username, remote_host, keyfile, certfile) + async with asyncssh.connect(remote_host, username=username, + client_keys=[(k,c)], known_hosts=None) as conn: + home = "/global/homes/{}/{}".format(username[0], username) + result = await conn.run("myquota -c {}".format(home)) + retcode = result.exit_status + # result = await conn.run(spawner.remote_port_command) + # remote_port = int(result.stdout) + if retcode: + e = web.HTTPError(507,reason="Insufficient Storage") + em = space_error(spawner.hub.base_url) + e.my_message = em + raise e + # spawner.remote_host = remote_host + # spawner.port = remote_port + +c.Spawner.pre_spawn_hook = setup + ## c.NERSCSpawner.spawners = [ ## ("spin", "sshspawner.sshspawner.SSHSpawner", { ## "remote_hosts" : ["jupyter"], From 7c99ec56598914efd98fd1294460beccce8c672f Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Wed, 10 Apr 2019 13:25:34 -0700 Subject: [PATCH 15/19] Let's make it a day --- jupyter-nersc/web-jupyterhub/jupyterhub_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter-nersc/web-jupyterhub/jupyterhub_config.py b/jupyter-nersc/web-jupyterhub/jupyterhub_config.py index 9cf72fa..4ecdb39 100644 --- a/jupyter-nersc/web-jupyterhub/jupyterhub_config.py +++ b/jupyter-nersc/web-jupyterhub/jupyterhub_config.py @@ -389,7 +389,7 @@ def comma_split(string): { 'name': 'cull-idle', 'admin': True, - 'command': 'cull_idle_servers.py --timeout=64800'.split(), + 'command': 'cull_idle_servers.py --timeout=86400'.split(), }, { 'name': 'announcement', From 2f362cabf6d9f0e7c3084e23465259b31ba774a5 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 12 Apr 2019 08:54:35 -0700 Subject: [PATCH 16/19] Fix pre-spawn hook but note issue --- jupyter-nersc/web-jupyterhub/jupyterhub_config.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/jupyter-nersc/web-jupyterhub/jupyterhub_config.py b/jupyter-nersc/web-jupyterhub/jupyterhub_config.py index 4ecdb39..7e4b8f8 100644 --- a/jupyter-nersc/web-jupyterhub/jupyterhub_config.py +++ b/jupyter-nersc/web-jupyterhub/jupyterhub_config.py @@ -1035,17 +1035,16 @@ def comma_split(string): # Pre-spawn myquota check def space_error(home): - """Extra message pointing users to try spawning again from /hub/home. """ - home = url_path_join(home, 'home') - return ( - "There is insufficient space in your home directory; please clear up some files and then " - "navigate to the hub home and start your server.".format(home=home) - ) + """Extra message pointing users to try spawning again from /hub/home. """ + home = url_path_join(home, 'home') + return ("There is insufficient space in your home directory; please clear up some files and then " + + "navigate to the hub home and start your server.".format(home=home)) async def setup(spawner): username = spawner.user.name remote_host = "corijupyter.nersc.gov" - keyfile = spawner.ssh_keyfile.format(username=username) +# keyfile = spawner.ssh_keyfile.format(username=username) + keyfile = "/certs/{username}.key".format(username=username) # NEED to have in NERSCSpawner now certfile = keyfile + "-cert.pub" k = asyncssh.read_private_key(keyfile) c = asyncssh.read_certificate(certfile) From 07e3c92e729f474cd58d5a2a40dbb1aacfafea60 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Thu, 25 Apr 2019 09:30:11 -0700 Subject: [PATCH 17/19] Pin pika, latest API has big changes --- jupyter-nersc/app-monitoring/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter-nersc/app-monitoring/Dockerfile b/jupyter-nersc/app-monitoring/Dockerfile index 6bcfc75..bbdd021 100644 --- a/jupyter-nersc/app-monitoring/Dockerfile +++ b/jupyter-nersc/app-monitoring/Dockerfile @@ -31,7 +31,7 @@ RUN \ echo "python 3.6.*" >> /opt/anaconda3/conda-meta/pinned && \ /opt/anaconda3/bin/conda update --yes conda && \ /opt/anaconda3/bin/pip install --no-cache-dir \ - pika + pika==0.13.1 ENV PATH=/opt/anaconda3/bin:$PATH From f7639ea8e003888a05d792a67cce3442fadfdc0a Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Thu, 25 Apr 2019 10:25:53 -0700 Subject: [PATCH 18/19] Force notebook 5.7.8 --- jupyter-nersc/app-notebooks/packages2.txt | 2 +- jupyter-nersc/app-notebooks/packages3.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter-nersc/app-notebooks/packages2.txt b/jupyter-nersc/app-notebooks/packages2.txt index a9b84d3..f4559a5 100644 --- a/jupyter-nersc/app-notebooks/packages2.txt +++ b/jupyter-nersc/app-notebooks/packages2.txt @@ -2,7 +2,7 @@ basemap beautiful-soup biopython netcdf4 -notebook=5.7.7 +notebook=5.7.8 r-car r-essentials rpy2 diff --git a/jupyter-nersc/app-notebooks/packages3.txt b/jupyter-nersc/app-notebooks/packages3.txt index bce3ee0..03df3a1 100644 --- a/jupyter-nersc/app-notebooks/packages3.txt +++ b/jupyter-nersc/app-notebooks/packages3.txt @@ -5,7 +5,7 @@ blaze jupyter jupyterlab netcdf4 -notebook=5.7.7 +notebook=5.7.8 r-essentials rpy2 scipy From b98e897b12eb8afdc457a3ae8c2e1af940ed0bfb Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Thu, 25 Apr 2019 10:26:28 -0700 Subject: [PATCH 19/19] Something was wrong with this, kick to 19-05 --- jupyter-nersc/web-jupyterhub/jupyterhub_config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jupyter-nersc/web-jupyterhub/jupyterhub_config.py b/jupyter-nersc/web-jupyterhub/jupyterhub_config.py index 7e4b8f8..3711dee 100644 --- a/jupyter-nersc/web-jupyterhub/jupyterhub_config.py +++ b/jupyter-nersc/web-jupyterhub/jupyterhub_config.py @@ -5,6 +5,9 @@ import asyncssh import requests +from tornado import web +from jupyterhub.utils import url_path_join + def comma_split(string): """Handle env variables that may be None, empty string, or have spaces""" @@ -1064,7 +1067,7 @@ async def setup(spawner): # spawner.remote_host = remote_host # spawner.port = remote_port -c.Spawner.pre_spawn_hook = setup +# c.Spawner.pre_spawn_hook = setup ## c.NERSCSpawner.spawners = [ ## ("spin", "sshspawner.sshspawner.SSHSpawner", {