Merge pull request #39 from FREVA-CLINT/solr_api

Solr api

.github/workflows/build_job.yml

@@ -18,7 +18,7 @@ jobs:
     steps:
       -
         name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 

.github/workflows/test_job.yml

@@ -1,38 +1,34 @@
 name: Tests
 run-name: ${{ github.actor }} is doing some smoke tests
 
-on: [push, pull_request]
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
 jobs:
-  lint:
+  Smoke-tests:
     runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash -l {0}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: 'true'
       - name: Set up services
         run: docker-compose up -d --remove-orphans
       - name: Create conda env
-        uses: conda-incubator/setup-miniconda@v2
+        # https://github.com/marketplace/actions/setup-micromamba
+        uses: mamba-org/setup-micromamba@v1
         with:
-          activate-environment: freva-web
+          environment-name: freva-web
           environment-file: conda-env.yml
-          python-version: "3.11"
-          auto-activate-base: false
-      - name: Run checks in python
-        run: make tests
-      - name: Lint python
-        run: make lint
-      - name: Prepare node
-        run: npm install
-      - name: Lint js formatting
-        run: npm run lint-format
-      - name: Lint javascript
-        run: npm run lint
-      - name: Test building the prod system
-        run: npm run build-production
-      - name: Test building the dev system
-        run: npm run build
+          cache-environment: false
+          cache-downloads: false
+          init-shell: bash
+      - name: Lint js code and python
+        run: micromamba run -n freva-web make lint
+      - name: Run build checks for js and python smoke tests
+        run: micromamba run -n freva-web make tests

.gitignore

@@ -1,8 +1,10 @@
 *.pyc
 *.db
+*.log
 virt
 static_root/img/favicon.svg
 freva_web.conf
+docker/databrowser_api_config.toml
 db
 node_modules
 venv/*
@@ -22,3 +24,5 @@ activate_web
 static/*
 assets/bundles/*
 webpack-stats.json
+base/migrations/*
+

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "docker/config"]
 	path = docker/config
 	url = https://github.com/FREVA-CLINT/freva-service-config.git
+	branch = main

Makefile

@@ -2,16 +2,61 @@
 
 export EVALUATION_SYSTEM_CONFIG_FILE := $(PWD)/docker/local-eval-system.conf
 export EVALUATION_SYSTEM_DRS_CONFIG_FILE := $(PWD)/docker/drs_config.toml
-run:
-	python manage.py runserver
+export DJANGO_SUPERUSER_PASSWORD := secret
+export DEV_MODE := 1
+
+.PHONY: all run runserver runfrontend stopserver stopfrontend stop setup
+
+all: setup runserver runfrontend
+	@echo "All services are running in the background."
 
 dummy-data:
 	docker/dummy_plugin_runs.sh
-	python3 docker/solr/ingest_dummy_data.py
 
-lint:
+
+setup-django:
+	python manage.py makemigrations base
+	python manage.py migrate --fake-initial
+	python manage.py migrate --fake contenttypes
+	python manage.py createsuperuser --noinput --username admin --email [email protected] || echo
+
+setup-node:
+	npm install
+
+runserver:
+	@echo "Starting Django development server..."
+	python manage.py runserver > runserver.log 2>&1 &
+	@echo "Django development server is running..."
+	@echo "To watch the Django server logs, run 'tail -f runserver.log'"
+
+runfrontend:
+	@echo "Starting npm development server..."
+	npm run dev > npm.log 2>&1 &
+	@echo "npm development server is running..."
+	@echo "To watch the npm logs, run 'tail -f npm.log'"
+
+stopserver:
+	ps aux | grep '[m]anage.py runserver' | awk '{print $$2}' | xargs -r kill
+	echo "Stopped Django development server..." > runserver.log
+
+stopfrontend:
+	pkill -f "npm run dev"
+	echo "Stopped npm development server..." > npm.log
+
+stop: stopserver stopfrontend
+	@echo "All services have been stopped."
+
+setup: setup-node setup-django dummy-data
+
+run: runfrontend runserver
+
+lint: setup-node
+	npm run lint-format
+	npm run lint
 	black -t py310 --check .
 
-tests:
+tests: setup-node
+	npm run build-production
+	npm run build
 	rm -rf node_modules
 	pytest -vv $(PWD) tests/

README.md

@@ -25,8 +25,16 @@ conda env create -f conda-env.yml
 source .envrc
 ```
 
-The web ui will need a connection to a solr and mariadb service.
-This services can be deployed using
+> ``📝`` If conda has issues solving dependencies you can install and use
+         [mamba](https://mamba.readthedocs.io/en/latest/user_guide/mamba.html)
+         instead of anaconda. This is recommended because the dependency solvers
+         that ship with mamba are usually much faster than those conda uses.
+
+### Additional services running on docker
+
+The web ui will need a connection to a solr,
+[freva-databrowser](https://github.com/FREVA-CLINT/databrowserAPI/) and
+mariadb service. This services can be deployed using
 [`docker-compose`](https://docs.docker.com/compose/install/).
 
 ```console
@@ -43,69 +51,113 @@ docker-compose down
 
 There are some rudimentary tests that check the integration of `django` and the
 `nodejs` building process. Assuming you have followed steps mentioned above and
-created a `freva-dev` cona miniconda environment you can run the tests after
+created a `freva-dev` conda miniconda environment you can run the tests after
 activating this environment:
 
 ```console
 conda activate freva-web
 python -m pytest -vv tests
 ```
 
-## Django deployment
-
-You can check if django is working and correctly configured by:
-
-```console
-python manage.py check
-```
+## Using GNU `make`:
+We have created a Makefile that sets up a development version of the web. You
+can use:
 
-If checks are passing issue the following command
+- To *setup* / *initialise* the nodejs and django servers use:
+    ```console
+    make setup
+    ```
+- To *run* node and django servers use:
+    ```console
+    make run
+    ```
+- To use both `setup` and `run` command use just use make:
+    ```console
+    make
+    ```
+- To stop the servers use:
+    ```console
+    make stop
+    ```
 
-```console
-python manage.py makemigrations base
-python manage.py migrate --fake-initial
-python manage.py migrate --fake contenttypes
-python manage.py createsuperuser
-```
+The django and npm development servers will write output into `runserver.log` and
+`npm.log`. You can observe the output of the processes using `tail -f` or something
+similar.
 
-The `--fake-inital` flag tells `django` not to create the already existing
-database tables.
+> ``📝`` You need a Node version of at least 16.5 along a npm version of 8.19
 
-## Running the server in dev mode
+# The Production container
+This section only briefly describes the docker image that is automatically
+created (see the [next section](#create-a-new-web-release) on how to trigger a
+build of the image) and is not meant for actual application. Please install and
+use the [freva-deployment package](https://pypi.org/project/freva-deployment/)
+to deploy the web app in production mode.
 
-A development server can be set using the following command:
+A pre-build image of the web app is available via:
 
 ```console
-python manage.py runserver [port_number]
+docker pull ghcr.io/freva-clint/freva-web:latest
 ```
 
-Default port is 8000. If an application is already running on that port you
-can change the port number with help of a command line argument
-
-## Building the JS application :
+When running in production mode you should set the following container
+environment variables:
 
-**Please note:** You need a Node version of at least 16.5 along a npm version of 8.19
+- ``EVALUATION_SYSTEM_CONFIG_FILE`` : Path to the freva config file
+- ``LDAP_USER_DN``: the Distinguished Name within ldap for example
+                    `uid=jdoe,ou=users,dc=example,dc=com`.
+- ``LDAP_USER_PW``: password for the LDAP server connection.
+- ``DJANGO_SUPERUSER_PASSWORD``: the super user password for the django app.
 
-Install dependencies:
+The web app app is running on port 8000, hence you want to publish this port
+via the `-p` flag. Ideally the path to the `$EVALUATION_SYSTEM_CONFIG_FILE`
+should be mounted into the container as a volume.
 
-```console
+Since static files are served by the http web server and not the django web app
+you have to add the location of the static files (e.g. `/srv/static`) as a
+volume into the container to the `/opt/freva_web/static` location.
+On startup of the container django app will create all static files that will
+then be available through `/srv/static` on the docker host.
 
-npm install
-
-```
-
-Build project:
+All together a minimal working example looks like this:
 
 ```console
-npm run build
-npm run build-production   # optimized production build
+docker run -it -e EVALUATION_SYSTEM_CONFIG_FILE=/work/freva/evaluation_system.conf \
+        -e LDAP_USER_DN='uid=jdoe,ou=users,dc=example,dc=com' \
+        -e LDAP_USER_PW='secret' \
+        -e DJANGO_SUPERUSER_PASSWORD='more_secret' \
+        -v /work/freva:/work/freva:z \
+        -v /srv/static:/opt/freva_web/static:z \
+        -p 8000:8000 \
+        ghcr.io/freva-clint/freva-web:latest
 ```
+The web app is then available via port 8000 on the host system.
 
-Development:
+## Making the web app available on a web server.
+To be able to access the web app through a normal http web server you will need
+to setup a reverse proxy on your http web server to port 8000. Refer to the
+reverse proxy settings for your web server. Here is a minimal example for
+apache httpd (using the example from above where the static files are located
+in `/srv/static` on the docker host):
 
-```console
-npm run dev   # starts webpack-dev-server including hot-reloading
 ```
+Listen 80
+LoadModule proxy_html_module modules/mod_proxy_html.so
+LoadModule proxy_module modules/mod_proxy.so
+LoadModule proxy_connect_module modules/mod_proxy_connect.so
+LoadModule proxy_http_module modules/mod_proxy_http.so
+<VirtualHost *:80>
+     ProxyPass /static/ !
+     ProxyPass /api/databrowser/ http://freva-databrowser.example.com:7777/api/databrowser/
+     ProxyPassReverse /api/databrowser/ http://freva-databrowser.example.com:7777/api/databrowser/
+     ProxyPass / http://freva.example.com:8000/
+     ProxyPassReverse / http://freva.example.com:8000/
+     Alias /static /srv/static/
+     Alias /robots.txt /srv/static/robots.txt
+     Alias /favicon.ico /srv/static/favicon.ico
+</VirtualHost>
+```
+> ``📝`` This is a minimal example, in a real world scenario you should always
+         configure your web server to enable web encryption via port 443.
 
 # Create a new web release.
 The production systems are deployed in a docker image hosted on the GitHub

assets/js/Components/AccordionItemBody/index.js

@@ -45,7 +45,7 @@ Row.propTypes = {
   data: PropTypes.array,
   rowData: PropTypes.shape({
     value: PropTypes.string,
-    count: PropTypes.number,
+    count: PropTypes.oneOfType([PropTypes.number, PropTypes.string]),
   }),
   elemRef: PropTypes.oneOfType([
     PropTypes.func,
@@ -90,7 +90,9 @@ function AccordionItemBody(props) {
       let renderedElem;
       const badge = (
         <div>
-          <Badge bg="secondary d-flex align-items-center">{count}</Badge>
+          <Badge bg="secondary d-flex align-items-center">
+            {parseInt(count).toLocaleString("en-US")}
+          </Badge>
         </div>
       );
       const link = (
@@ -165,14 +167,15 @@ function AccordionItemBody(props) {
   const [windowWidth] = useWindowSize();
   const getSize = (index) => sizeMap.current[index] || 24;
 
+  const facetName = props.mappedName ?? eventKey;
   return (
     <div className={props.isFacetCentered ? "facet-container" : ""}>
       <div>
         <FormControl
           className="my-2"
           id="search"
           type="text"
-          placeholder={`Search ${eventKey} name`}
+          placeholder={`Search ${facetName} name`}
           onChange={handleChange}
         />
       </div>
@@ -212,6 +215,7 @@ function AccordionItemBody(props) {
 
 AccordionItemBody.propTypes = {
   eventKey: PropTypes.string,
+  mappedName: PropTypes.string,
   value: PropTypes.array,
   isFacetCentered: PropTypes.bool,
   metadata: PropTypes.object,