Merge pull request #92 from zitrosolrac/updatingDev

Migrating performance from public site
datajoint-company · Nov 2, 2021 · 45dc027 · 45dc027
2 parents 60dca99 + e1b4c60
commit 45dc027
Show file tree

Hide file tree

Showing 48 changed files with 524 additions and 353 deletions.
diff --git a/README.md b/README.md
@@ -19,6 +19,7 @@ AWS_ACCESS_KEY_ID=aws_key
 AWS_SECRET_ACCESS_KEY=aws_secret
 DEMO_PASSWORD=ibl_navigator_password
 JWT_SECRET=secret
+API_MODE=[public | private]
 # utilized for remote deployment
 SUBDOMAINS=sub
 URL=example.com
@@ -108,4 +109,4 @@ make sure to move over to the `master` branch by `git checkout master`
 `docker-compose -f docker-compose-deploy.yml pull` to get the ibl-navigator container
 `docker-compose -f docker-compose-deploy.yml up -d`
 
--------------------------------------
+-------------------------------------
diff --git a/backend/iblapi.py b/backend/iblapi.py
@@ -172,10 +172,12 @@ def do_req(subpath):
     # construct kwargs
     kwargs = {'as_dict': True}
     limit = int(request.values['__limit']) if '__limit' in values else None
-    order = request.values['__order'] if '__order' in values else None
+    order = request.values['__order'] if '__order' in values else 'KEY ASC'
+    page = int(request.values['__page']) if '__page' in values else 1
     proj = json.loads(request.values['__proj']) if '__proj' in values else None
-    special_fields = ['__json', '__limit', '__order', '__proj', '__json_kwargs']
-    for a in (v for v in values if v not in special_fields):
+    special_fields = ['__json', '__limit', '__order', '__proj', '__json_kwargs', 
+    '__page']
+    for a in (k for k, v in values.items() if k not in special_fields and v):
         # HACK: 'uuid' attrs -> UUID type (see also: datajoint-python #594)
         postargs[a] = UUID(values[a]) if 'uuid' in a else values[a]
     args = [postargs] if len(postargs) else []
@@ -186,9 +188,14 @@ def do_req(subpath):
     if '__json_kwargs' in values:
         json_kwargs = json.loads(request.values['__json_kwargs'])
     args = {} if not args else dj.AndList(args)
-    kwargs = {k: v for k, v in (('as_dict', True,),
-                                   ('limit', limit,),
-                                   ('order_by', order,)) if v is not None}
+    if limit == None:
+        kwargs = {k: v for k, v in (('as_dict', True,),
+                                    ('order_by', order,)) if v is not None}
+    else:
+        kwargs = {k: v for k, v in (('as_dict', True,),
+                                    ('limit', limit,),
+                                    ('order_by', order,),
+                                    ('offset', (page-1)*limit)) if v is not None}
     # 2) and dispatch
     app.logger.debug("args: '{}', kwargs: {}".format(args, kwargs))
     if obj not in reqmap:
@@ -216,8 +223,7 @@ def handle_q(subpath, args, proj, fetch_args=None, **kwargs):
         ((session * subject * lab * user) & arg).proj(flist)
     '''
     app.logger.info("handle_q: subpath: '{}', args: {}".format(subpath, args))
-    app.logger.info('key words: {}'.format(kwargs))
-
+
     fetch_args = {} if fetch_args is None else fetch_args
     ret = []
     post_process = None
@@ -266,10 +272,12 @@ def handle_q(subpath, args, proj, fetch_args=None, **kwargs):
 
         q = ((acquisition.Session() * sess_proj * psych_curve * ephys_data * subject.Subject() *
               subject.SubjectLab() * subject.SubjectUser() * trainingStatus) & args & brain_restriction)
-
+        q = q.proj(*proj) if proj else q
         dj.conn().query("SET SESSION max_join_size={}".format('18446744073709551615'))
-        q = q.proj(*proj).fetch(**fetch_args) if proj else q.fetch(**fetch_args)
+        ret_count = len(q)
+        ret = q.fetch(**fetch_args)
         dj.conn().query("SET SESSION max_join_size={}".format(original_max_join_size))
+        return dumps({"records_count": ret_count, "records": ret})
     elif subpath == 'subjpage':
         proj_restr = None
         for e in args:

diff --git a/docker-compose-base.yml b/docker-compose-base.yml
@@ -8,7 +8,7 @@ x-net: &net
 services:
   iblapi:
     <<: *net
-    image: registry.vathes.com/ibl-navigator/iblapi:v0.5.3 # for internal demo
+    image: registry.vathes.com/ibl-navigator/iblapi:v0.6.0 # for internal demo
     # image: registry.vathes.com/ibl-navigator/iblapi:v0.1.0-public # for public demo
     environment:
     - DJ_USER=maho
@@ -31,15 +31,16 @@ services:
     #cpu_period: 100ms
   ibl-navigator:
     <<: *net
-    image: registry.vathes.com/ibl-navigator/frontend:v0.4.2 # for internal demo
+    image: registry.vathes.com/ibl-navigator/frontend:v0.5.0 # for internal demo
     # image: registry.vathes.com/ibl-navigator/frontend:v0.1.0-public # for public demo
     healthcheck:
       test: curl --fail http://localhost:9000 || exit 1
       timeout: 3s
       retries: 20
   ibl-node-server:
     <<: *net
-    image: registry.vathes.com/ibl-navigator/node-server:v0.3.0 # for internal demo
+    platform: linux/amd64
+    image: registry.vathes.com/ibl-navigator/node-server:v0.4.0 # for internal demo
     # image: registry.vathes.com/ibl-navigator/node-server:v0.1.0-public # for public demo
     environment:
     - NODE_ENV=development
@@ -71,7 +72,7 @@ services:
     #     condition: service_healthy
   nginx:
     <<: *net
-    image: datajoint/nginx:v0.0.16
+    image: datajoint/nginx:v0.0.18
     environment:
       - ADD_zfrontend_TYPE=REST
       - ADD_zfrontend_ENDPOINT=ibl-navigator:9000

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
@@ -35,6 +35,9 @@ services:
       service: ibl-node-server
     image: ibl-node-server:v0.0.0
     # command: nodemon /src/server.js
+    # command: npm test
+    ports:
+      - "9222:9222"
     volumes:
       - ./node_server/app.js:/src/app.js
     <<: *net

diff --git a/ibl-frontend/Dockerfile b/ibl-frontend/Dockerfile
@@ -1,4 +1,10 @@
-FROM vathes/angulardev:angcli7.1.4-angbuild0.11.4
+# FROM vathes/angulardev:angcli7.1.4-angbuild0.11.4
+
+FROM node:12-bullseye
+
+RUN \
+    apt update && \
+    apt install nginx -y
 
 WORKDIR /app/dist/pipeline-viewer
 
@@ -10,10 +16,9 @@ COPY ./app.conf /etc/nginx/conf.d/default.conf
 CMD ["nginx", "-g", "daemon off;"]
 
 ADD ./frontend-content/package.json /app/
-ADD ./frontend-content/package-lock.json /app/
 RUN \
     cd /app && \
-    npm install --save-dev @angular-devkit/build-angular > /dev/null
+    npm install 
 
 
 ADD ./frontend-content /app
@@ -23,7 +28,7 @@ COPY ./frontend-content/src/assets/addons/plotly.js /app/node_modules/plotly.js-
 
 RUN \
     cd /app && \
-    node --max_old_space_size=8192 /usr/local/lib/node_modules/@angular/cli/bin/ng build --prod
+    node --max_old_space_size=5120 /app/node_modules/@angular/cli/bin/ng build --configuration production
 
 
 # CMD ["http-server","-p", "8080" ,"-a","0.0.0.0"]

diff --git a/ibl-frontend/dev.dockerfile b/ibl-frontend/dev.dockerfile
@@ -1,5 +1,7 @@
 # don't build
-FROM vathes/angulardev:angcli7.1.4-angbuild0.11.4
+# FROM vathes/angulardev:angcli7.1.4-angbuild0.11.4
+
+FROM node:12-bullseye
 
 HEALTHCHECK       \
     --timeout=3s \ 
@@ -10,12 +12,10 @@ HEALTHCHECK       \
 COPY ./entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]
-CMD ["ng","serve","--host","0.0.0.0","--port","9000","--disable-host-check"]
 
 WORKDIR /app
 
 ADD ./frontend-content/package.json /app/
-ADD ./frontend-content/package-lock.json /app/
 RUN \
     npm install && \
     npm install --only=dev
@@ -24,5 +24,6 @@ ADD ./frontend-content /app
 COPY ./frontend-content/src/assets/addons/indigo-pink-ibl.css /app/node_modules/\@angular/material/prebuilt-themes/
 COPY ./frontend-content/src/assets/addons/plotly.js /app/node_modules/plotly.js-dist/
 
+CMD ["node", "--max_old_space_size=5120", "/app/node_modules/@angular/cli/bin/ng", "serve", "--host", "0.0.0.0", "--port", "9000", "--disable-host-check"]
 
-
+# node --max_old_space_size=5120 /app/node_modules/@angular/cli/bin/ng serve  --host  0.0.0.0  --port  9000  --disable-host-check 1> /app/src/output.log 2> /app/src/error.log
diff --git a/ibl-frontend/frontend-content/angular.json b/ibl-frontend/frontend-content/angular.json
@@ -23,16 +23,25 @@
               "src/assets"
             ],
             "styles": [
-              "node_modules/bootstrap/dist/css/bootstrap.min.css",
+              {
+                "input": "node_modules/@angular/material/prebuilt-themes/indigo-pink-ibl.css",
+                "inject": true
+              },
+              {
+                "input": "src/assets/fonts/css/open-iconic-bootstrap.min.css",
+                "inject": true
+              },
               "src/styles.css"
             ],
             "scripts": [
-              // "node_modules/mathjax/MathJax.js",
-              // "node_modules/mathjax/jax/output/SVG/config.js",
-              // "node_modules/mathjax/extensions/MathMenu.js",
-              // "node_modules/mathjax/extensions/MathZoom.js",
               "node_modules/plotly.js-dist/plotly.js"
-            ]
+            ],
+            "vendorChunk": true,
+            "extractLicenses": false,
+            "buildOptimizer": false,
+            "sourceMap": true,
+            "optimization": false,
+            "namedChunks": true
           },
           "configurations": {
             "production": {
@@ -45,21 +54,24 @@
               "optimization": true,
               "outputHashing": "all",
               "sourceMap": false,
-              "extractCss": true,
               "namedChunks": false,
-              "aot": true,
               "extractLicenses": true,
               "vendorChunk": false,
               "buildOptimizer": true,
               "budgets": [
                 {
                   "type": "initial",
                   "maximumWarning": "2mb",
-                  "maximumError": "7mb"
+                  "maximumError": "10mb"
+                },
+                {
+                  "type": "anyComponentStyle",
+                  "maximumWarning": "6kb"
                 }
               ]
             }
-          }
+          },
+          "defaultConfiguration": ""
         },
         "serve": {
           "builder": "@angular-devkit/build-angular:dev-server",

diff --git a/ibl-frontend/frontend-content/package.json b/ibl-frontend/frontend-content/package.json
@@ -11,56 +11,30 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/animations": "^7.1.4",
-    "@angular/cdk": "^7.3.7",
-    "@angular/common": "~7.1.0",
-    "@angular/compiler": "~7.1.0",
-    "@angular/core": "~7.1.0",
-    "@angular/flex-layout": "^7.0.0-beta.24",
-    "@angular/forms": "~7.1.0",
-    "@angular/material": "^7.3.3",
-    "@angular/material-moment-adapter": "^8.1.2",
-    "@angular/platform-browser": "~7.1.0",
-    "@angular/platform-browser-dynamic": "~7.1.0",
-    "@angular/router": "~7.1.0",
-    "@types/mathjax": "0.0.35",
+    "@angular/animations": "^12.0.0",
+    "@angular/cdk": "^12.0.0",
+    "@angular/common": "^12.0.0",
+    "@angular/compiler": "^12.0.0",
+    "@angular/core": "^12.0.0",
+    "@angular/flex-layout": "^12.0.0-beta.35",
+    "@angular/forms": "^12.0.0",
+    "@angular/material": "^12.0.0",
+    "@angular/material-moment-adapter": "^12.0.0",
+    "@angular/platform-browser": "^12.0.0",
+    "@angular/platform-browser-dynamic": "^12.0.0",
+    "@angular/router": "^12.0.0",
     "@wizpanda/super-gif": "0.0.5",
-    "angular-plotly.js": "^1.3.2",
-    "body-parser": "^1.18.3",
+    "angular-plotly.js": "^4.0.4",
     "bootstrap": "^4.2.1",
-    "core-js": "^2.5.4",
-    "dotenv": "^8.0.0",
-    "express": "^4.16.4",
-    "jsonwebtoken": "^8.5.1",
-    "mathjax": "^2.7.5",
-    "moment": "^2.24.0",
-    "plotly.js": "^1.49.0",
-    "plotly.js-dist": "^1.44.4",
-    "request": "^2.88.0",
-    "rxjs": "~6.3.3",
-    "rxjs-compat": "^6.3.3",
-    "tslib": "^1.9.0",
-    "zone.js": "~0.8.26"
+    "moment": "^2.29.1",
+    "plotly.js": "^2.5.1",
+    "rxjs": "^7.4.0",
+    "rxjs-compat": "^6.6.7",
+    "zone.js": "^0.11.4"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "~0.11.0",
-    "@angular/cli": "~7.1.3",
-    "@angular/compiler-cli": "~7.1.0",
-    "@angular/language-service": "~7.1.0",
-    "@types/node": "~8.9.4",
-    "@types/jasmine": "~2.8.8",
-    "@types/jasminewd2": "~2.0.3",
-    "codelyzer": "~4.5.0",
-    "jasmine-core": "~2.99.1",
-    "jasmine-spec-reporter": "~4.2.1",
-    "karma": "~3.1.1",
-    "karma-chrome-launcher": "~2.2.0",
-    "karma-coverage-istanbul-reporter": "~2.0.1",
-    "karma-jasmine": "~1.1.2",
-    "karma-jasmine-html-reporter": "^0.2.2",
-    "protractor": "~5.4.0",
-    "ts-node": "~7.0.0",
-    "tslint": "~5.11.0",
-    "typescript": "~3.1.6"
+    "@angular-devkit/build-angular": "^12.0.0",
+    "@angular/cli": "^12.0.0",
+    "@angular/compiler-cli": "^12.0.0"
   }
 }
diff --git a/ibl-frontend/frontend-content/src/app/app.module.ts b/ibl-frontend/frontend-content/src/app/app.module.ts
@@ -6,30 +6,28 @@ import { PlotlyModule } from 'angular-plotly.js';
 import { FormsModule, ReactiveFormsModule } from '@angular/forms';
 import { HttpClientModule, HTTP_INTERCEPTORS } from '@angular/common/http';
 import { Routes, RouterModule } from '@angular/router';
-import { MatSelectModule,
-         MatAutocompleteModule,
-         MatIconModule,
-         MatInputModule,
-         MatCheckboxModule,
-         MatRadioModule,
-         MatNativeDateModule,
-         MatDatepickerModule,
-         MatSlideToggleModule,
-         MatCardModule,
-         MatButtonModule,
-         MatTableModule,
-         MatPaginatorModule,
-         MatSortModule,
-         MatSliderModule,
-         MatExpansionModule,
-         MatDialogModule,
-         MatTreeModule,
-         MatFormFieldModule } from '@angular/material';
 import { MatMomentDateModule, MAT_MOMENT_DATE_ADAPTER_OPTIONS } from '@angular/material-moment-adapter';
+import { MatNativeDateModule } from '@angular/material/core';
 import { FlexLayoutModule } from '@angular/flex-layout';
 import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
-
-
+import {MatSelectModule} from '@angular/material/select';
+import {MatAutocompleteModule} from '@angular/material/autocomplete';
+import {MatIconModule} from '@angular/material/icon';
+import {MatInputModule} from '@angular/material/input';
+import {MatCheckboxModule} from '@angular/material/checkbox';
+import {MatRadioModule} from '@angular/material/radio';
+import {MatDatepickerModule} from '@angular/material/datepicker';
+import {MatSlideToggleModule} from '@angular/material/slide-toggle';
+import {MatCardModule} from '@angular/material/card';
+import {MatButtonModule} from '@angular/material/button';
+import {MatTableModule} from '@angular/material/table';
+import {MatPaginatorModule} from '@angular/material/paginator';
+import {MatSortModule} from '@angular/material/sort';
+import {MatSliderModule} from '@angular/material/slider';
+import {MatExpansionModule} from '@angular/material/expansion';
+import {MatDialogModule} from '@angular/material/dialog';
+import {MatTreeModule} from '@angular/material/tree';
+import {MatFormFieldModule} from '@angular/material/form-field';
 import { AuthGuard } from './auth/auth-guard.service';
 import { AppComponent } from './app.component';
 import { EachBatchComponent } from './each-batch/each-batch.component';
@@ -64,6 +62,7 @@ import { PsthPlotsComponent } from './cell-list/cell/psth-plots/psth-plots.compo
 import { QualityControlComponent } from './quality-control/quality-control.component';
 import { DriftmapComponent } from './quality-control/driftmap/driftmap.component';
 import { SpinningBrainComponent } from './mouse-list/mouse/spinning-brain/spinning-brain.component';
+import {MatProgressSpinnerModule} from '@angular/material/progress-spinner';
 
 PlotlyModule.plotlyjs = PlotlyJS;
 
@@ -170,6 +169,7 @@ const appRoutes: Routes = [
     FormsModule,
     HttpClientModule,
     RouterModule.forRoot(appRoutes),
+    MatProgressSpinnerModule,
     MatSelectModule, MatAutocompleteModule, MatIconModule, MatInputModule,
     MatCheckboxModule, MatRadioModule, MatNativeDateModule, MatDatepickerModule, MatMomentDateModule, MatSlideToggleModule,
     MatCardModule, MatButtonModule, MatTableModule, MatPaginatorModule, MatSortModule, MatSliderModule, MatExpansionModule,

diff --git a/ibl-frontend/frontend-content/src/app/auth/auth-guard.service.ts b/ibl-frontend/frontend-content/src/app/auth/auth-guard.service.ts
@@ -1,5 +1,5 @@
 import { CanActivate, ActivatedRouteSnapshot, RouterStateSnapshot, Router, CanActivateChild } from '@angular/router';
-import { Observable } from 'rxjs/Observable';
+import { Observable } from 'rxjs';
 import { Injectable } from '@angular/core';
 import { AuthService } from './auth.service';