Merge pull request #59 from NERSC/19-04

19 04

jupyter-base/Dockerfile

@@ -37,10 +37,13 @@ RUN \
     /opt/anaconda3/bin/conda install --yes                      \
         alembic         \
         decorator       \
+        entrypoints     \
         jinja2          \
         mako            \
         markupsafe      \
         nodejs          \
+        oauthlib=2      \
+        pamela          \
         psycopg2        \
         pyopenssl       \
         python-dateutil \
@@ -57,7 +60,6 @@ RUN \
     npm install -g configurable-http-proxy                                  &&  \
     git clone https://github.com/jupyterhub/jupyterhub.git                  &&  \
     cd jupyterhub                                                           &&  \
-#   git checkout tags/0.9.4                                                 &&  \
     /opt/anaconda3/bin/python setup.py js                                   &&  \
     /opt/anaconda3/bin/pip --no-cache-dir install .                         &&  \
     cp examples/cull-idle/cull_idle_servers.py /opt/anaconda3/bin/.         &&  \

jupyter-base/build.sh

@@ -4,4 +4,4 @@ branch=$(git symbolic-ref --short HEAD)
 
 docker build    \
     --no-cache  \
-    --tag registry.spin.nersc.gov/das/jupyter-base.$branch:latest .
+    --tag registry.spin.nersc.gov/das/jupyter-base-$branch:latest .

jupyter-nersc/mods/Dockerfile → jupyter-nersc/app-monitoring/Dockerfile

@@ -31,7 +31,7 @@ RUN \
     echo "python 3.6.*" >> /opt/anaconda3/conda-meta/pinned &&  \
     /opt/anaconda3/bin/conda update --yes conda             &&  \
     /opt/anaconda3/bin/pip install --no-cache-dir               \
-        pika
+        pika==0.13.1
 
 ENV PATH=/opt/anaconda3/bin:$PATH
 

jupyter-nersc/mods/build.sh → jupyter-nersc/app-monitoring/build.sh

@@ -4,4 +4,4 @@ branch=$(git symbolic-ref --short HEAD)
 
 docker build    \
     "$@"        \
-    --tag registry.spin.nersc.gov/das/jupyter-nersc-mods.$branch:latest .
+    --tag registry.spin.nersc.gov/das/app-monitoring.jupyter-nersc-$branch:latest .

jupyter-nersc/mods/monitor.py → jupyter-nersc/app-monitoring/monitor.py

@@ -56,7 +56,7 @@ def parse_arguments():
     parser.add_argument("--queue", "-q",
             default="text")
     parser.add_argument("--api-url", "-a",
-            default="http://web:8081/hub/api")
+            default="http://web-jupyterhub:8081/hub/api")
     parser.add_argument("--api-token", "-o",
             default=os.environ["MODS_JUPYTERHUB_API_TOKEN"])
     parser.add_argument("--subcategory", "-s",

jupyter-nersc/app/Dockerfile → jupyter-nersc/app-notebooks/Dockerfile

@@ -1,6 +1,6 @@
 ARG branch=unknown
 
-FROM registry.spin.nersc.gov/das/jupyter-base.${branch}:latest
+FROM registry.spin.nersc.gov/das/jupyter-base-${branch}:latest
 LABEL maintainer="Rollin Thomas <[email protected]>"
 WORKDIR /srv
 

jupyter-nersc/app/build.sh → jupyter-nersc/app-notebooks/build.sh

@@ -5,4 +5,4 @@ branch=$(git symbolic-ref --short HEAD)
 docker build                    \
     --build-arg branch=$branch  \
     "$@"                        \
-    --tag registry.spin.nersc.gov/das/jupyter-nersc-app.$branch:latest .
+    --tag registry.spin.nersc.gov/das/app-notebooks.jupyter-nersc-$branch:latest .

jupyter-nersc/app/packages2.txt → jupyter-nersc/app-notebooks/packages2.txt

@@ -2,6 +2,7 @@ basemap
 beautiful-soup
 biopython
 netcdf4
+notebook=5.7.8
 r-car
 r-essentials
 rpy2

jupyter-nersc/app/packages3.txt → jupyter-nersc/app-notebooks/packages3.txt

@@ -5,6 +5,7 @@ blaze
 jupyter
 jupyterlab
 netcdf4
+notebook=5.7.8
 r-essentials
 rpy2
 scipy

jupyter-nersc/web/Dockerfile → jupyter-nersc/web-jupyterhub/Dockerfile

@@ -1,6 +1,6 @@
 ARG branch=unknown
 
-FROM registry.spin.nersc.gov/das/jupyter-base.${branch}:latest
+FROM registry.spin.nersc.gov/das/jupyter-base-${branch}:latest
 LABEL maintainer="Rollin Thomas <[email protected]>"
 WORKDIR /srv
 

jupyter-nersc/web/build.sh → jupyter-nersc/web-jupyterhub/build.sh

@@ -5,4 +5,4 @@ branch=$(git symbolic-ref --short HEAD)
 docker build                    \
     --build-arg branch=$branch  \
     "$@"                        \
-    --tag registry.spin.nersc.gov/das/jupyter-nersc-web.$branch:latest .
+    --tag registry.spin.nersc.gov/das/web-jupyterhub.jupyter-nersc-$branch:latest .

jupyter-nersc/web/hub-scripts/test-user.sh → jupyter-nersc/web-jupyterhub/hub-scripts/test-user.sh

@@ -2,8 +2,8 @@
 
 # Test user's ability to ssh
 
-hostname=app
-username=$1
+hostname=$1
+username=$2
 cert=/certs/$username.key
 echo $username $cert
 if [ ! -f $cert ]; then

jupyter-nersc/web/jupyterhub_config.py → jupyter-nersc/web-jupyterhub/jupyterhub_config.py

@@ -3,7 +3,11 @@
 import os
 import sys
 
+import asyncssh
 import requests
+from tornado import web
+from jupyterhub.utils import url_path_join
+
 
 def comma_split(string):
     """Handle env variables that may be None, empty string, or have spaces"""
@@ -183,7 +187,7 @@ def comma_split(string):
 
 ## url for the database. e.g. `sqlite:///jupyterhub.sqlite`
 #c.JupyterHub.db_url = 'sqlite:///jupyterhub.sqlite'
-c.JupyterHub.db_url = 'postgresql://jupyterhub:{}@db:5432/jupyterhub'.format(
+c.JupyterHub.db_url = 'postgresql://jupyterhub:{}@db-jupyterhub:5432/jupyterhub'.format(
         os.getenv('POSTGRES_PASSWORD')
 )
 
@@ -388,7 +392,7 @@ def comma_split(string):
     {
         'name': 'cull-idle',
         'admin': True,
-        'command': 'cull_idle_servers.py --timeout=43200'.split(),
+        'command': 'cull_idle_servers.py --timeout=86400'.split(),
     },
     {
         'name': 'announcement',
@@ -1022,7 +1026,7 @@ def comma_split(string):
         "sshspawner.sshspawner.SSHSpawner", {
             "cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh",
                 "/opt/anaconda3/bin/jupyter-labhub"],
-            "remote_hosts": ["app"],
+            "remote_hosts": ["app-notebooks"],
             "remote_port_command": "/opt/anaconda3/bin/python /global/common/cori/das/jupyterhub/new-get-port.py --ip",
             "hub_api_url": "http://{}:8081/hub/api".format(ip),
             "path": "/opt/anaconda3/bin:/usr/bin:/usr/local/bin:/bin",
@@ -1031,6 +1035,40 @@ def comma_split(string):
     )
 }
 
+# Pre-spawn myquota check
+
+def space_error(home):
+    """Extra message pointing users to try spawning again from /hub/home.  """
+    home = url_path_join(home, 'home')
+    return ("There is insufficient space in your home directory; please clear up some files and then " +
+            "<a href='{home}'>navigate to the hub home</a> and start your server.".format(home=home))
+
+async def setup(spawner):
+    username = spawner.user.name
+    remote_host = "corijupyter.nersc.gov"
+#   keyfile = spawner.ssh_keyfile.format(username=username)
+    keyfile = "/certs/{username}.key".format(username=username) # NEED to have in NERSCSpawner now
+    certfile = keyfile + "-cert.pub"
+    k = asyncssh.read_private_key(keyfile)
+    c = asyncssh.read_certificate(certfile)
+    # print(username, remote_host, keyfile, certfile)
+    async with asyncssh.connect(remote_host, username=username, 
+            client_keys=[(k,c)], known_hosts=None) as conn:
+        home = "/global/homes/{}/{}".format(username[0], username)
+        result = await conn.run("myquota -c {}".format(home))
+        retcode = result.exit_status
+        # result = await conn.run(spawner.remote_port_command)
+        # remote_port = int(result.stdout)
+    if retcode:
+        e = web.HTTPError(507,reason="Insufficient Storage")
+        em = space_error(spawner.hub.base_url)
+        e.my_message = em
+        raise e
+    # spawner.remote_host = remote_host
+    # spawner.port = remote_port
+
+# c.Spawner.pre_spawn_hook = setup
+
 ## c.NERSCSpawner.spawners = [
 ##         ("spin", "sshspawner.sshspawner.SSHSpawner", {
 ##             "remote_hosts"          : ["jupyter"],

jupyter-nersc/off/build.sh → jupyter-nersc/web-offline/build.sh

@@ -5,4 +5,4 @@ branch=$(git symbolic-ref --short HEAD)
 docker build                    \
     --build-arg branch=$branch  \
     "$@"                        \
-    --tag registry.spin.nersc.gov/das/jupyter-nersc-off.$branch:latest .
+    --tag registry.spin.nersc.gov/das/web-offline.jupyter-nersc-$branch:latest .