Merge pull request #237 from OpenSourceBrain/release/0.3.0

Release/0.3.0
OpenSourceBrain · Jul 30, 2021 · 44b15fa · 44b15fa
2 parents da81308 + a3c4e0a
commit 44b15fa
Show file tree

Hide file tree

Showing 267 changed files with 20,345 additions and 45,494 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,7 @@
 cloud-harness
 .idea
-**/helm
+**/helm
+/build
+.vscode
+skaffold.yaml
+/deployment.yaml
diff --git a/README.md b/README.md
@@ -95,7 +95,7 @@ At least 6GB of ram and 4 processors are needed to run MNP
 
 To create a new cluster, run
 ```
-minikube start --memory="6000mb" --cpus=4 --disk-size=80mb
+minikube start --memory="6000mb" --cpus=4 --disk-size=60000mb
 ```
 
 Enable the ingress addon:

diff --git a/applications/.gitignore b/applications/.gitignore
@@ -0,0 +1 @@
+values-local.yaml
diff --git a/applications/accounts/deploy/values.yaml b/applications/accounts/deploy/values.yaml
diff --git a/applications/accounts/Docker-compose.yaml → applications/accounts/docker-compose.yaml b/applications/accounts/Docker-compose.yaml → applications/accounts/docker-compose.yaml
diff --git a/applications/accounts/themes/custom/login/resources/css/login.css b/applications/accounts/themes/custom/login/resources/css/login.css
@@ -26,6 +26,7 @@ body {
   position: absolute;
   right: 0;
   top: 20px;
+  cursor: pointer;
 }
 
 .login-pf-header {
@@ -40,22 +41,14 @@ body {
   font-weight: 700;
 }
 
-input:-webkit-autofill::first-line {
-  font-size: 14px;
-}
 
 .login-pf-page .has-error .form-control {
   border: 0;
   border-bottom: #eb5757 1px solid;
   box-shadow: none;
 }
 
-input:-webkit-autofill,
-input:-webkit-autofill:hover,
-input:-webkit-autofill:focus,
-input:-webkit-autofill:active {
-  -webkit-box-shadow: 0 0 0 30px #fff inset !important;
-}
+
 
 .form-group {
   margin-bottom: 9px;
@@ -351,14 +344,14 @@ div.kc-logo-text span {
   margin: 0 0 0 11px;
   color: rgba(255, 255, 255, 0.4);
   font-weight: 400;
-  text-decoration: underline;
+
 }
 .login-pf-header a,
 .login-pf-header a:hover {
+  text-decoration: underline;
   font-weight: 400;
-  color: rgba(255, 255, 255, 0.4);
+  color: inherit;
   outline: 0;
-  text-decoration: none;
 }
 
 .kc-register-form {

diff --git a/applications/accounts/themes/custom/login/template.ftl b/applications/accounts/themes/custom/login/template.ftl
@@ -34,6 +34,7 @@
             <script src="${script}" type="text/javascript"></script>
         </#list>
     </#if>
+
 </head>
 
 <body class="${properties.kcBodyClass!}">
@@ -147,6 +148,16 @@
 
     </div>
   </div>
+      <script>
+        const logo = document.getElementsByClassName("login-pf-logo");
+        for(const l of logo) {
+            l.addEventListener("click",function(e){
+
+                window.location.href = window.location.protocol + '//' + window.location.host.replace('accounts.', '')
+            },false);
+        }
+
+    </script>
 </body>
 </html>
 </#macro>
diff --git a/applications/common/deploy/values.yaml b/applications/common/deploy/values.yaml
@@ -0,0 +1,4 @@
+harness:
+  env:
+  - name: SENTRY_DSN
+    value: https://[email protected]/3
diff --git a/applications/jupyterhub/Dockerfile b/applications/jupyterhub/Dockerfile
@@ -1,14 +1,34 @@
-FROM jupyterhub/k8s-hub:0.9.0
+ARG CLOUDHARNESS_BASE
+FROM $CLOUDHARNESS_BASE as base
+RUN echo $CLOUDHARNESS_BASE
+
+FROM jupyter/base-notebook:hub-1.1.0
 USER root
 
-COPY src src
-RUN pip install ./src/harness_jupyter
-RUN pip install ./src/chauthenticator
-RUN pip install ./src/osb_jupyter
-RUN chmod 777 /usr/local/lib/python3.6/dist-packages/  -R
+RUN apt update && apt install nano
+COPY --from=base libraries/cloudharness-common/requirements.txt /libraries/cloudharness-common/requirements.txt
+RUN pip install -r /libraries/cloudharness-common/requirements.txt
+COPY --from=base libraries/client/cloudharness_cli/requirements.txt /libraries/client/cloudharness_cli/requirements.txt
+RUN pip install -r /libraries/client/cloudharness_cli/requirements.txt
+
+COPY --from=base /libraries/cloudharness-common /libraries/cloudharness-common
+COPY --from=base /libraries/client/cloudharness_cli /libraries/client/cloudharness_cli
+
+RUN pip install -e /libraries/cloudharness-common
+RUN pip install -e /libraries/client/cloudharness_cli
+
+COPY src /usr/src/app
+
+RUN pip install -e /usr/src/app/harness_jupyter
+RUN pip install -e /usr/src/app/chauthenticator
+RUN pip install -e /usr/src/app/osb_jupyter
+
+
+COPY --chown=1000:1000 theming/page.html /opt/conda/share/jupyterhub/templates/page.html
+COPY --chown=1000:1000 theming/spawn_pending.html /opt/conda/share/jupyterhub/templates/spawn_pending.html
+COPY --chown=1000:1000 theming/hot_fix_for_eventsource.js /opt/conda/share/jupyterhub/static/hot_fix_for_eventsource.js
+
+COPY hub/jupyter_notebook_config.py /etc/jupyter/jupyter_notebook_config.py
 
-COPY --chown=1000:1000 theming/page.html /usr/local/share/jupyterhub/templates/page.html
-COPY --chown=1000:1000 theming/spawn_pending.html /usr/local/share/jupyterhub/templates/spawn_pending.html
-COPY --chown=1000:1000 theming/hot_fix_for_eventsource.js /usr/local/share/jupyterhub/static/hot_fix_for_eventsource.js
+RUN chmod 777 /usr/src/app -R
 
-USER jovyan
diff --git a/applications/jupyterhub/deploy/values.yaml b/applications/jupyterhub/deploy/values.yaml
@@ -1,11 +1,15 @@
 harness:
-  subdomain: jupyter
+  subdomain: hub
+  secured: true
   service:
     auto: false
     port: 80
     name: proxy-public
   deployment:
     auto: false
+  dependencies:
+    build:
+      - cloudharness-base
 custom: {}
 hub:
   allowNamedServers: true
@@ -59,9 +63,9 @@ hub:
       c.Spawner.port = 8000
       c.Spawner.http_timeout = 300
       c.Spawner.start_timeout = 300
-      c.JupyterHub.tornado_settings = { "headers": { "Content-Security-Policy": "frame-ancestors 'self' *.osb.local *.opensourcebrain.org *.v2.opensourcebrain.org"}}
+      c.JupyterHub.tornado_settings = { "headers": { "Content-Security-Policy": "frame-ancestors 'self' localhost *.osb.local *.opensourcebrain.org *.v2.opensourcebrain.org"}}
     spawner: >-
-      c.Spawner.args = ["--port=8000"]
+      c.Spawner.args = []
   extraConfigMap: {}
   extraEnv: {}
   extraContainers: []
@@ -203,9 +207,9 @@ singleuser:
     type: dynamic
     capacity: 2Mi
     dynamic:
-      pvcNameTemplate: osb-jupyter-{userid}
-      volumeNameTemplate: osb-jupyter-{userid}
-    homeMountPath: /opt/home
+      pvcNameTemplate: osb-user-{userid}
+      volumeNameTemplate: osb-user-{userid}
+    homeMountPath: /opt/user
     extraLabels: {}
   image:
     name: jupyter/base-notebook

diff --git a/applications/jupyterhub/hub/jupyter_notebook_config.py b/applications/jupyterhub/hub/jupyter_notebook_config.py
@@ -0,0 +1,66 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+from jupyter_core.paths import jupyter_data_dir
+import subprocess
+import os
+import errno
+import stat
+
+c = get_config()
+c.NotebookApp.ip = '0.0.0.0'
+c.NotebookApp.port = 8888
+c.NotebookApp.open_browser = False
+
+# https://github.com/jupyter/notebook/issues/3130
+c.FileContentsManager.delete_to_trash = False
+
+print('*'*80)
+import notebook
+print(c.NotebookApp.tornado_settings)
+c.NotebookApp.tornado_settings = {
+    'headers': {
+        'Content-Security-Policy': "frame-ancestors 'self' localhost *.osb.local *.opensourcebrain.org",
+    }
+}
+print(c.NotebookApp.tornado_settings)
+print('*'*80)
+
+# Generate a self-signed certificate
+if 'GEN_CERT' in os.environ:
+    dir_name = jupyter_data_dir()
+    pem_file = os.path.join(dir_name, 'notebook.pem')
+    try:
+        os.makedirs(dir_name)
+    except OSError as exc:  # Python >2.5
+        if exc.errno == errno.EEXIST and os.path.isdir(dir_name):
+            pass
+        else:
+            raise
+
+    # Generate an openssl.cnf file to set the distinguished name
+    cnf_file = os.path.join(os.getenv('CONDA_DIR', '/usr/lib'), 'ssl', 'openssl.cnf')
+    if not os.path.isfile(cnf_file):
+        with open(cnf_file, 'w') as fh:
+            fh.write('''\
+[req]
+distinguished_name = req_distinguished_name
+[req_distinguished_name]
+''')
+
+    # Generate a certificate if one doesn't exist on disk
+    subprocess.check_call(['openssl', 'req', '-new',
+                           '-newkey', 'rsa:2048',
+                           '-days', '365',
+                           '-nodes', '-x509',
+                           '-subj', '/C=XX/ST=XX/L=XX/O=generated/CN=generated',
+                           '-keyout', pem_file,
+                           '-out', pem_file])
+    # Restrict access to the file
+    os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR)
+    c.NotebookApp.certfile = pem_file
+
+# Change default umask for all subprocesses of the notebook server if set in
+# the environment
+if 'NB_UMASK' in os.environ:
+    os.umask(int(os.environ['NB_UMASK'], 8))
diff --git a/applications/jupyterhub/src/osb_jupyter/osb_jupyter/jupyterhub.py b/applications/jupyterhub/src/osb_jupyter/osb_jupyter/jupyterhub.py
@@ -1,8 +1,30 @@
-import logging
 import sys
 
+import yaml
+from jupyterhub.user import User
 from kubespawner.spawner import KubeSpawner
 
+from cloudharness.auth import AuthClient
+from cloudharness import log
+
+
+def affinity_spec(key, value):
+    return {
+
+        'labelSelector':
+            {
+                'matchExpressions': [
+                    {
+                        'key': str(key),
+                        'operator': 'In',
+                        'values': [str(value)]
+                    },
+                ]
+            },
+        'topologyKey': 'kubernetes.io/hostname'
+    }
+
+
 def change_pod_manifest(self: KubeSpawner):
     """
     Application Hook to change the manifest of the notebook image
@@ -16,25 +38,66 @@ def change_pod_manifest(self: KubeSpawner):
     """
 
     # get the workspace cookie to determine the workspace id
-    workspace_cookie = self.handler.request.cookies.get('workspaceId', None)
-    if workspace_cookie:
-        workspace_id = workspace_cookie.value
+
+    def get_from_cookie(cookie_name):
+        cookie = self.handler.request.cookies.get(cookie_name, None)
+        if cookie is None:
+            raise Exception(
+                "Required cookie not found. Check that the cookie named '%s' is set.", cookie_name)
+        return cookie.value
+
+    try:
+        workspace_id = get_from_cookie('workspaceId')
         volume_name = f'workspace-{workspace_id}'
+        log.info('Mapping to volume %s', volume_name)
 
         # We found the workspace id and now we try to mount the
         # workspace persistent volume claim
 
-        # ToDo: get pvc from kubernetes!
-        ws_pvc = {'name': volume_name, 
+        ws_pvc = {
+            'name': volume_name,
             'persistentVolumeClaim': {
-                'claimName': volume_name, 
+                'claimName': volume_name,
                 'spec': {
-                    'accessModes': ['ReadWriteOne']
+                    'accessModes': ['ReadWriteOnce', 'ReadOnlyMany']
                 }
             }
         }
 
         # add the volume to the pod
-        self.volumes.append(ws_pvc)
-        # mount the workspace volume in the pod
-        self.volume_mounts.append({'name': volume_name, 'mountPath': '/opt/workspace'})
+        if not [v for v in self.volumes if v['name'] == volume_name]:
+            self.volumes.append(ws_pvc)
+
+        workspace_owner = get_from_cookie('workspaceOwner')
+
+        # Add labels to use for affinity
+        labels = {
+            'workspace': str(workspace_id),
+            'user': self.user.name
+        }
+
+        self.common_labels = labels
+        self.extra_labels = labels
+
+        self.pod_affinity_required.append(affinity_spec('user', self.user.name))
+        write_access = has_user_write_access(
+            workspace_id, self.user, workspace_owner)
+        if write_access:
+            # Pods with write access must be on the same node
+            self.pod_affinity_required.append(affinity_spec('workspace', workspace_id))
+        if not [v for v in self.volume_mounts if v['name'] == volume_name]:
+            self.volume_mounts.append({
+                'name': volume_name,
+                'mountPath': '/opt/workspace',
+                'readOnly': not write_access
+            })
+    except Exception as e:
+        log.error('Change pod manifest failed due to an error.', exc_info=True)
+
+
+def has_user_write_access(workspace_id, user: User, workspace_owner: str):
+    print('name:', user.name, workspace_owner)
+    if workspace_owner == user.name:
+        return True
+    auth_client = AuthClient()
+    return auth_client.user_has_realm_role(user.name, 'administrator')
diff --git a/applications/jupyterhub/src/osb_jupyter/setup.py b/applications/jupyterhub/src/osb_jupyter/setup.py
@@ -2,6 +2,7 @@
 
 REQUIREMENTS = [
     'jupyterhub-kubespawner',
+    'cloudharness'
 ]
 
 setup(
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,6 +2,7 @@ @@
     REQUIREMENTS = [
         'jupyterhub-kubespawner',
+        'cloudharness'
     ]
     setup(
@@ Expand Down @@