diff --git a/jupyter-base/Dockerfile b/jupyter-base/Dockerfile index fa436b6..ee00a48 100644 --- a/jupyter-base/Dockerfile +++ b/jupyter-base/Dockerfile @@ -37,10 +37,13 @@ RUN \ /opt/anaconda3/bin/conda install --yes \ alembic \ decorator \ + entrypoints \ jinja2 \ mako \ markupsafe \ nodejs \ + oauthlib=2 \ + pamela \ psycopg2 \ pyopenssl \ python-dateutil \ @@ -57,7 +60,6 @@ RUN \ npm install -g configurable-http-proxy && \ git clone https://github.com/jupyterhub/jupyterhub.git && \ cd jupyterhub && \ -# git checkout tags/0.9.4 && \ /opt/anaconda3/bin/python setup.py js && \ /opt/anaconda3/bin/pip --no-cache-dir install . && \ cp examples/cull-idle/cull_idle_servers.py /opt/anaconda3/bin/. && \ diff --git a/jupyter-base/build.sh b/jupyter-base/build.sh index bcae179..e6265ec 100644 --- a/jupyter-base/build.sh +++ b/jupyter-base/build.sh @@ -4,4 +4,4 @@ branch=$(git symbolic-ref --short HEAD) docker build \ --no-cache \ - --tag registry.spin.nersc.gov/das/jupyter-base.$branch:latest . + --tag registry.spin.nersc.gov/das/jupyter-base-$branch:latest . diff --git a/jupyter-nersc/mods/Dockerfile b/jupyter-nersc/app-monitoring/Dockerfile similarity index 98% rename from jupyter-nersc/mods/Dockerfile rename to jupyter-nersc/app-monitoring/Dockerfile index 6bcfc75..bbdd021 100644 --- a/jupyter-nersc/mods/Dockerfile +++ b/jupyter-nersc/app-monitoring/Dockerfile @@ -31,7 +31,7 @@ RUN \ echo "python 3.6.*" >> /opt/anaconda3/conda-meta/pinned && \ /opt/anaconda3/bin/conda update --yes conda && \ /opt/anaconda3/bin/pip install --no-cache-dir \ - pika + pika==0.13.1 ENV PATH=/opt/anaconda3/bin:$PATH diff --git a/jupyter-nersc/mods/build.sh b/jupyter-nersc/app-monitoring/build.sh similarity index 51% rename from jupyter-nersc/mods/build.sh rename to jupyter-nersc/app-monitoring/build.sh index e1fbce5..a3a4efc 100644 --- a/jupyter-nersc/mods/build.sh +++ b/jupyter-nersc/app-monitoring/build.sh @@ -4,4 +4,4 @@ branch=$(git symbolic-ref --short HEAD) docker build \ "$@" \ - --tag registry.spin.nersc.gov/das/jupyter-nersc-mods.$branch:latest . + --tag registry.spin.nersc.gov/das/app-monitoring.jupyter-nersc-$branch:latest . diff --git a/jupyter-nersc/mods/docker-entrypoint.sh b/jupyter-nersc/app-monitoring/docker-entrypoint.sh similarity index 100% rename from jupyter-nersc/mods/docker-entrypoint.sh rename to jupyter-nersc/app-monitoring/docker-entrypoint.sh diff --git a/jupyter-nersc/mods/monitor.py b/jupyter-nersc/app-monitoring/monitor.py similarity index 99% rename from jupyter-nersc/mods/monitor.py rename to jupyter-nersc/app-monitoring/monitor.py index 9f8e3a7..e94b7a2 100644 --- a/jupyter-nersc/mods/monitor.py +++ b/jupyter-nersc/app-monitoring/monitor.py @@ -56,7 +56,7 @@ def parse_arguments(): parser.add_argument("--queue", "-q", default="text") parser.add_argument("--api-url", "-a", - default="http://web:8081/hub/api") + default="http://web-jupyterhub:8081/hub/api") parser.add_argument("--api-token", "-o", default=os.environ["MODS_JUPYTERHUB_API_TOKEN"]) parser.add_argument("--subcategory", "-s", diff --git a/jupyter-nersc/app/Dockerfile b/jupyter-nersc/app-notebooks/Dockerfile similarity index 97% rename from jupyter-nersc/app/Dockerfile rename to jupyter-nersc/app-notebooks/Dockerfile index 803d685..e0dca1a 100644 --- a/jupyter-nersc/app/Dockerfile +++ b/jupyter-nersc/app-notebooks/Dockerfile @@ -1,6 +1,6 @@ ARG branch=unknown -FROM registry.spin.nersc.gov/das/jupyter-base.${branch}:latest +FROM registry.spin.nersc.gov/das/jupyter-base-${branch}:latest LABEL maintainer="Rollin Thomas " WORKDIR /srv diff --git a/jupyter-nersc/app/NERSC-keys-api b/jupyter-nersc/app-notebooks/NERSC-keys-api similarity index 100% rename from jupyter-nersc/app/NERSC-keys-api rename to jupyter-nersc/app-notebooks/NERSC-keys-api diff --git a/jupyter-nersc/app/build.sh b/jupyter-nersc/app-notebooks/build.sh similarity index 65% rename from jupyter-nersc/app/build.sh rename to jupyter-nersc/app-notebooks/build.sh index b408866..8fec9e1 100644 --- a/jupyter-nersc/app/build.sh +++ b/jupyter-nersc/app-notebooks/build.sh @@ -5,4 +5,4 @@ branch=$(git symbolic-ref --short HEAD) docker build \ --build-arg branch=$branch \ "$@" \ - --tag registry.spin.nersc.gov/das/jupyter-nersc-app.$branch:latest . + --tag registry.spin.nersc.gov/das/app-notebooks.jupyter-nersc-$branch:latest . diff --git a/jupyter-nersc/app/docker-entrypoint.sh b/jupyter-nersc/app-notebooks/docker-entrypoint.sh similarity index 100% rename from jupyter-nersc/app/docker-entrypoint.sh rename to jupyter-nersc/app-notebooks/docker-entrypoint.sh diff --git a/jupyter-nersc/app/etc/ldap.conf b/jupyter-nersc/app-notebooks/etc/ldap.conf similarity index 100% rename from jupyter-nersc/app/etc/ldap.conf rename to jupyter-nersc/app-notebooks/etc/ldap.conf diff --git a/jupyter-nersc/app/etc/nslcd.conf b/jupyter-nersc/app-notebooks/etc/nslcd.conf similarity index 100% rename from jupyter-nersc/app/etc/nslcd.conf rename to jupyter-nersc/app-notebooks/etc/nslcd.conf diff --git a/jupyter-nersc/app/etc/nsswitch.conf b/jupyter-nersc/app-notebooks/etc/nsswitch.conf similarity index 100% rename from jupyter-nersc/app/etc/nsswitch.conf rename to jupyter-nersc/app-notebooks/etc/nsswitch.conf diff --git a/jupyter-nersc/app/etc/pam.d/jupyterhub b/jupyter-nersc/app-notebooks/etc/pam.d/jupyterhub similarity index 100% rename from jupyter-nersc/app/etc/pam.d/jupyterhub rename to jupyter-nersc/app-notebooks/etc/pam.d/jupyterhub diff --git a/jupyter-nersc/app/etc/pam.d/password-auth b/jupyter-nersc/app-notebooks/etc/pam.d/password-auth similarity index 100% rename from jupyter-nersc/app/etc/pam.d/password-auth rename to jupyter-nersc/app-notebooks/etc/pam.d/password-auth diff --git a/jupyter-nersc/app/etc/user_ca.pub b/jupyter-nersc/app-notebooks/etc/user_ca.pub similarity index 100% rename from jupyter-nersc/app/etc/user_ca.pub rename to jupyter-nersc/app-notebooks/etc/user_ca.pub diff --git a/jupyter-nersc/app/get_port.py b/jupyter-nersc/app-notebooks/get_port.py similarity index 100% rename from jupyter-nersc/app/get_port.py rename to jupyter-nersc/app-notebooks/get_port.py diff --git a/jupyter-nersc/app/packages2.txt b/jupyter-nersc/app-notebooks/packages2.txt similarity index 83% rename from jupyter-nersc/app/packages2.txt rename to jupyter-nersc/app-notebooks/packages2.txt index 6f668cf..f4559a5 100644 --- a/jupyter-nersc/app/packages2.txt +++ b/jupyter-nersc/app-notebooks/packages2.txt @@ -2,6 +2,7 @@ basemap beautiful-soup biopython netcdf4 +notebook=5.7.8 r-car r-essentials rpy2 diff --git a/jupyter-nersc/app/packages3.txt b/jupyter-nersc/app-notebooks/packages3.txt similarity index 85% rename from jupyter-nersc/app/packages3.txt rename to jupyter-nersc/app-notebooks/packages3.txt index 23ad40c..03df3a1 100644 --- a/jupyter-nersc/app/packages3.txt +++ b/jupyter-nersc/app-notebooks/packages3.txt @@ -5,6 +5,7 @@ blaze jupyter jupyterlab netcdf4 +notebook=5.7.8 r-essentials rpy2 scipy diff --git a/jupyter-nersc/app/supervisord.conf b/jupyter-nersc/app-notebooks/supervisord.conf similarity index 100% rename from jupyter-nersc/app/supervisord.conf rename to jupyter-nersc/app-notebooks/supervisord.conf diff --git a/jupyter-nersc/web/Dockerfile b/jupyter-nersc/web-jupyterhub/Dockerfile similarity index 92% rename from jupyter-nersc/web/Dockerfile rename to jupyter-nersc/web-jupyterhub/Dockerfile index 41354a0..06eeb37 100644 --- a/jupyter-nersc/web/Dockerfile +++ b/jupyter-nersc/web-jupyterhub/Dockerfile @@ -1,6 +1,6 @@ ARG branch=unknown -FROM registry.spin.nersc.gov/das/jupyter-base.${branch}:latest +FROM registry.spin.nersc.gov/das/jupyter-base-${branch}:latest LABEL maintainer="Rollin Thomas " WORKDIR /srv diff --git a/jupyter-nersc/web/announcement.py b/jupyter-nersc/web-jupyterhub/announcement.py similarity index 100% rename from jupyter-nersc/web/announcement.py rename to jupyter-nersc/web-jupyterhub/announcement.py diff --git a/jupyter-nersc/web/build.sh b/jupyter-nersc/web-jupyterhub/build.sh similarity index 65% rename from jupyter-nersc/web/build.sh rename to jupyter-nersc/web-jupyterhub/build.sh index b136616..b0340fc 100644 --- a/jupyter-nersc/web/build.sh +++ b/jupyter-nersc/web-jupyterhub/build.sh @@ -5,4 +5,4 @@ branch=$(git symbolic-ref --short HEAD) docker build \ --build-arg branch=$branch \ "$@" \ - --tag registry.spin.nersc.gov/das/jupyter-nersc-web.$branch:latest . + --tag registry.spin.nersc.gov/das/web-jupyterhub.jupyter-nersc-$branch:latest . diff --git a/jupyter-nersc/web/docker-entrypoint.sh b/jupyter-nersc/web-jupyterhub/docker-entrypoint.sh similarity index 100% rename from jupyter-nersc/web/docker-entrypoint.sh rename to jupyter-nersc/web-jupyterhub/docker-entrypoint.sh diff --git a/jupyter-nersc/web/hub-scripts/flush-certs.sh b/jupyter-nersc/web-jupyterhub/hub-scripts/flush-certs.sh similarity index 100% rename from jupyter-nersc/web/hub-scripts/flush-certs.sh rename to jupyter-nersc/web-jupyterhub/hub-scripts/flush-certs.sh diff --git a/jupyter-nersc/web/hub-scripts/test-user.sh b/jupyter-nersc/web-jupyterhub/hub-scripts/test-user.sh similarity index 94% rename from jupyter-nersc/web/hub-scripts/test-user.sh rename to jupyter-nersc/web-jupyterhub/hub-scripts/test-user.sh index 5dbab43..1f3c0ca 100644 --- a/jupyter-nersc/web/hub-scripts/test-user.sh +++ b/jupyter-nersc/web-jupyterhub/hub-scripts/test-user.sh @@ -2,8 +2,8 @@ # Test user's ability to ssh -hostname=app -username=$1 +hostname=$1 +username=$2 cert=/certs/$username.key echo $username $cert if [ ! -f $cert ]; then diff --git a/jupyter-nersc/web/jupyterhub_config.py b/jupyter-nersc/web-jupyterhub/jupyterhub_config.py similarity index 95% rename from jupyter-nersc/web/jupyterhub_config.py rename to jupyter-nersc/web-jupyterhub/jupyterhub_config.py index 45f7320..3711dee 100644 --- a/jupyter-nersc/web/jupyterhub_config.py +++ b/jupyter-nersc/web-jupyterhub/jupyterhub_config.py @@ -3,7 +3,11 @@ import os import sys +import asyncssh import requests +from tornado import web +from jupyterhub.utils import url_path_join + def comma_split(string): """Handle env variables that may be None, empty string, or have spaces""" @@ -183,7 +187,7 @@ def comma_split(string): ## url for the database. e.g. `sqlite:///jupyterhub.sqlite` #c.JupyterHub.db_url = 'sqlite:///jupyterhub.sqlite' -c.JupyterHub.db_url = 'postgresql://jupyterhub:{}@db:5432/jupyterhub'.format( +c.JupyterHub.db_url = 'postgresql://jupyterhub:{}@db-jupyterhub:5432/jupyterhub'.format( os.getenv('POSTGRES_PASSWORD') ) @@ -388,7 +392,7 @@ def comma_split(string): { 'name': 'cull-idle', 'admin': True, - 'command': 'cull_idle_servers.py --timeout=43200'.split(), + 'command': 'cull_idle_servers.py --timeout=86400'.split(), }, { 'name': 'announcement', @@ -1022,7 +1026,7 @@ def comma_split(string): "sshspawner.sshspawner.SSHSpawner", { "cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh", "/opt/anaconda3/bin/jupyter-labhub"], - "remote_hosts": ["app"], + "remote_hosts": ["app-notebooks"], "remote_port_command": "/opt/anaconda3/bin/python /global/common/cori/das/jupyterhub/new-get-port.py --ip", "hub_api_url": "http://{}:8081/hub/api".format(ip), "path": "/opt/anaconda3/bin:/usr/bin:/usr/local/bin:/bin", @@ -1031,6 +1035,40 @@ def comma_split(string): ) } +# Pre-spawn myquota check + +def space_error(home): + """Extra message pointing users to try spawning again from /hub/home. """ + home = url_path_join(home, 'home') + return ("There is insufficient space in your home directory; please clear up some files and then " + + "navigate to the hub home and start your server.".format(home=home)) + +async def setup(spawner): + username = spawner.user.name + remote_host = "corijupyter.nersc.gov" +# keyfile = spawner.ssh_keyfile.format(username=username) + keyfile = "/certs/{username}.key".format(username=username) # NEED to have in NERSCSpawner now + certfile = keyfile + "-cert.pub" + k = asyncssh.read_private_key(keyfile) + c = asyncssh.read_certificate(certfile) + # print(username, remote_host, keyfile, certfile) + async with asyncssh.connect(remote_host, username=username, + client_keys=[(k,c)], known_hosts=None) as conn: + home = "/global/homes/{}/{}".format(username[0], username) + result = await conn.run("myquota -c {}".format(home)) + retcode = result.exit_status + # result = await conn.run(spawner.remote_port_command) + # remote_port = int(result.stdout) + if retcode: + e = web.HTTPError(507,reason="Insufficient Storage") + em = space_error(spawner.hub.base_url) + e.my_message = em + raise e + # spawner.remote_host = remote_host + # spawner.port = remote_port + +# c.Spawner.pre_spawn_hook = setup + ## c.NERSCSpawner.spawners = [ ## ("spin", "sshspawner.sshspawner.SSHSpawner", { ## "remote_hosts" : ["jupyter"], diff --git a/jupyter-nersc/web/templates/error.html b/jupyter-nersc/web-jupyterhub/templates/error.html similarity index 100% rename from jupyter-nersc/web/templates/error.html rename to jupyter-nersc/web-jupyterhub/templates/error.html diff --git a/jupyter-nersc/web/templates/home.html b/jupyter-nersc/web-jupyterhub/templates/home.html similarity index 100% rename from jupyter-nersc/web/templates/home.html rename to jupyter-nersc/web-jupyterhub/templates/home.html diff --git a/jupyter-nersc/web/templates/login.html b/jupyter-nersc/web-jupyterhub/templates/login.html similarity index 100% rename from jupyter-nersc/web/templates/login.html rename to jupyter-nersc/web-jupyterhub/templates/login.html diff --git a/jupyter-nersc/web/templates/page.html b/jupyter-nersc/web-jupyterhub/templates/page.html similarity index 100% rename from jupyter-nersc/web/templates/page.html rename to jupyter-nersc/web-jupyterhub/templates/page.html diff --git a/jupyter-nersc/off/Dockerfile b/jupyter-nersc/web-offline/Dockerfile similarity index 100% rename from jupyter-nersc/off/Dockerfile rename to jupyter-nersc/web-offline/Dockerfile diff --git a/jupyter-nersc/off/README.md b/jupyter-nersc/web-offline/README.md similarity index 100% rename from jupyter-nersc/off/README.md rename to jupyter-nersc/web-offline/README.md diff --git a/jupyter-nersc/off/app.py b/jupyter-nersc/web-offline/app.py similarity index 100% rename from jupyter-nersc/off/app.py rename to jupyter-nersc/web-offline/app.py diff --git a/jupyter-nersc/off/build.sh b/jupyter-nersc/web-offline/build.sh similarity index 65% rename from jupyter-nersc/off/build.sh rename to jupyter-nersc/web-offline/build.sh index f21b653..b427ff9 100644 --- a/jupyter-nersc/off/build.sh +++ b/jupyter-nersc/web-offline/build.sh @@ -5,4 +5,4 @@ branch=$(git symbolic-ref --short HEAD) docker build \ --build-arg branch=$branch \ "$@" \ - --tag registry.spin.nersc.gov/das/jupyter-nersc-off.$branch:latest . + --tag registry.spin.nersc.gov/das/web-offline.jupyter-nersc-$branch:latest . diff --git a/jupyter-nersc/off/static/logo.png b/jupyter-nersc/web-offline/static/logo.png similarity index 100% rename from jupyter-nersc/off/static/logo.png rename to jupyter-nersc/web-offline/static/logo.png diff --git a/jupyter-nersc/off/static/rectanglelogo-greytext-orangebody-greymoons.png b/jupyter-nersc/web-offline/static/rectanglelogo-greytext-orangebody-greymoons.png similarity index 100% rename from jupyter-nersc/off/static/rectanglelogo-greytext-orangebody-greymoons.png rename to jupyter-nersc/web-offline/static/rectanglelogo-greytext-orangebody-greymoons.png diff --git a/jupyter-nersc/off/templates/index.html b/jupyter-nersc/web-offline/templates/index.html similarity index 100% rename from jupyter-nersc/off/templates/index.html rename to jupyter-nersc/web-offline/templates/index.html