Added dockerfile, plugins, and nomad config

.gitignore

@@ -120,6 +120,7 @@ celerybeat.pid
 *.sage.py
 
 # Environments
+.pyenv
 .env
 .venv
 env/

Dockerfile

@@ -0,0 +1,7 @@
+FROM gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:develop
+USER root
+RUN apt-get update
+RUN apt-get -y install git
+COPY plugins.txt plugins.txt
+RUN pip install -r plugins.txt
+USER nomad

nomad-oasis/configs/nginx.conf

@@ -0,0 +1,82 @@
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    ''      close;
+}
+
+server {
+    listen        80;
+    server_name   localhost;
+    proxy_set_header Host $host;
+
+    gzip_min_length     1000;
+    gzip_buffers        4 8k;
+    gzip_http_version   1.0;
+    gzip_disable        "msie6";
+    gzip_vary           on;
+    gzip on;
+    gzip_proxied any;
+    gzip_types
+        text/css
+        text/javascript
+        text/xml
+        text/plain
+        application/javascript
+        application/x-javascript
+        application/json;
+
+    location / {
+        proxy_pass http://app:8000;
+    }
+
+    location ~ /nomad-oasis\/?(gui)?$ {
+        rewrite ^ /nomad-oasis/gui/ permanent;
+    }
+
+    location /nomad-oasis/gui/ {
+        proxy_intercept_errors on;
+        error_page 404 = @redirect_to_index;
+        proxy_pass http://app:8000;
+    }
+
+    location @redirect_to_index {
+        rewrite ^ /nomad-oasis/gui/index.html break;
+        proxy_pass http://app:8000;
+    }
+
+    location ~ \/gui\/(service-worker\.js|meta\.json)$ {
+        add_header Last-Modified $date_gmt;
+        add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0';
+        if_modified_since off;
+        expires off;
+        etag off;
+        proxy_pass http://app:8000;
+    }
+
+    location ~ /api/v1/uploads(/?$|.*/raw|.*/bundle?$)  {
+        client_max_body_size 35g;
+        proxy_request_buffering off;
+        proxy_pass http://app:8000;
+    }
+
+    location ~ /api/v1/.*/download {
+        proxy_buffering off;
+        proxy_pass http://app:8000;
+    }
+
+    location /nomad-oasis/north/ {
+        client_max_body_size 500m;
+        proxy_pass http://north:9000;
+
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+        # websocket headers
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+        proxy_set_header X-Scheme $scheme;
+
+        proxy_buffering off;
+    }
+}

nomad-oasis/configs/nomad.yaml

@@ -0,0 +1,34 @@
+services:
+  api_host: 'localhost'
+  api_base_path: '/nomad-oasis'
+
+oasis:
+  is_oasis: true
+  uses_central_user_management: true
+
+north:
+  jupyterhub_crypt_key: '978bfb2e13a8448a253c629d8dd84ff89587f30e635b753153960930cad9d36d'
+
+meta:
+  deployment: 'oasis'
+  deployment_url: 'https://my-oasis.org/api'
+  maintainer_email: '[email protected]'
+
+logstash:
+  enable: false
+
+mongo:
+    db_name: nomad_oasis_v1
+
+elastic:
+    entries_index: nomad_oasis_entries_v1
+    materials_index: nomad_oasis_materials_v1
+
+plugins:
+  options:
+    parsers/XRD:
+      python_package: nomad_measurements.xrd
+    schemas/nomad_measurements:
+      python_package: nomad_measurements
+    schemas/ikz_pld:
+      python_package: ikz_pld

nomad-oasis/docker-compose.yaml

@@ -0,0 +1,217 @@
+version: "3"
+
+services:
+  # broker for celery
+  rabbitmq:
+    restart: unless-stopped
+    image: rabbitmq:3.11.5
+    container_name: nomad_oasis_rabbitmq
+    environment:
+      - RABBITMQ_ERLANG_COOKIE=SWQOKODSQALRPCLNMEQG
+      - RABBITMQ_DEFAULT_USER=rabbitmq
+      - RABBITMQ_DEFAULT_PASS=rabbitmq
+      - RABBITMQ_DEFAULT_VHOST=/
+    volumes:
+      - rabbitmq:/var/lib/rabbitmq
+    healthcheck:
+      test: ["CMD", "rabbitmq-diagnostics", "--silent", "--quiet", "ping"]
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 10s
+
+  # the search engine
+  elastic:
+    restart: unless-stopped
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
+    container_name: nomad_oasis_elastic
+    environment:
+      - ES_JAVA_OPTS=-Xms512m -Xmx512m
+      - discovery.type=single-node
+    volumes:
+      - elastic:/usr/share/elasticsearch/data
+    healthcheck:
+      test:
+        - "CMD"
+        - "curl"
+        - "--fail"
+        - "--silent"
+        - "http://elastic:9200/_cat/health"
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 60s
+
+  # the user data db
+  mongo:
+    restart: unless-stopped
+    image: mongo:5.0.6
+    container_name: nomad_oasis_mongo
+    environment:
+      - MONGO_DATA_DIR=/data/db
+      - MONGO_LOG_DIR=/dev/null
+    volumes:
+      - mongo:/data/db
+      - ./.volumes/mongo:/backup
+    command: mongod --logpath=/dev/null # --quiet
+    healthcheck:
+      test:
+        - "CMD"
+        - "mongo"
+        - "mongo:27017/test"
+        - "--quiet"
+        - "--eval"
+        - "'db.runCommand({ping:1}).ok'"
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 10s
+
+  # nomad worker (processing)
+  worker:
+    restart: unless-stopped
+    image: ghcr.io/hampusnasstrom/nomad-test-image:main
+    container_name: nomad_oasis_worker
+    environment:
+      NOMAD_SERVICE: nomad_oasis_worker
+      NOMAD_RABBITMQ_HOST: rabbitmq
+      NOMAD_ELASTIC_HOST: elastic
+      NOMAD_MONGO_HOST: mongo
+      NOMAD_LOGSTASH_HOST: logtransfer
+    depends_on:
+      rabbitmq:
+        condition: service_healthy
+      elastic:
+        condition: service_healthy
+      mongo:
+        condition: service_healthy
+    volumes:
+      - ./configs/nomad.yaml:/app/nomad.yaml
+      - ./.volumes/fs:/app/.volumes/fs
+    command: python -m celery -A nomad.processing worker -l info -Q celery
+
+  # nomad app (api + proxy)
+  app:
+    restart: unless-stopped
+    image: ghcr.io/hampusnasstrom/nomad-test-image:main
+    container_name: nomad_oasis_app
+    environment:
+      NOMAD_SERVICE: nomad_oasis_app
+      NOMAD_SERVICES_API_PORT: 80
+      NOMAD_FS_EXTERNAL_WORKING_DIRECTORY: "$PWD"
+      NOMAD_RABBITMQ_HOST: rabbitmq
+      NOMAD_ELASTIC_HOST: elastic
+      NOMAD_MONGO_HOST: mongo
+      NOMAD_LOGSTASH_HOST: logtransfer
+      NOMAD_NORTH_HUB_HOST: north
+    depends_on:
+      rabbitmq:
+        condition: service_healthy
+      elastic:
+        condition: service_healthy
+      mongo:
+        condition: service_healthy
+      north:
+        condition: service_started
+    volumes:
+      - ./configs/nomad.yaml:/app/nomad.yaml
+      - ./.volumes/fs:/app/.volumes/fs
+    command: ./run.sh
+    healthcheck:
+      test:
+        - "CMD"
+        - "curl"
+        - "--fail"
+        - "--silent"
+        - "http://localhost:8000/-/health"
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 10s
+
+  # nomad remote tools hub (JupyterHUB, e.g. for AI Toolkit)
+  north:
+    restart: unless-stopped
+    image: ghcr.io/hampusnasstrom/nomad-test-image:main
+    container_name: nomad_oasis_north
+    environment:
+      NOMAD_SERVICE: nomad_oasis_north
+      NOMAD_NORTH_DOCKER_NETWORK: nomad_oasis_network
+      NOMAD_NORTH_HUB_CONNECT_IP: north
+      NOMAD_NORTH_HUB_IP: "0.0.0.0"
+      NOMAD_NORTH_HUB_HOST: north
+      NOMAD_SERVICES_API_HOST: app
+      NOMAD_FS_EXTERNAL_WORKING_DIRECTORY: "$PWD"
+      NOMAD_RABBITMQ_HOST: rabbitmq
+      NOMAD_ELASTIC_HOST: elastic
+      NOMAD_MONGO_HOST: mongo
+    volumes:
+      - ./configs/nomad.yaml:/app/nomad.yaml
+      - ./.volumes/fs:/app/.volumes/fs
+      - /var/run/docker.sock:/var/run/docker.sock
+    user: '1000:991'
+    command: python -m nomad.cli admin run hub
+    healthcheck:
+      test:
+        - "CMD"
+        - "curl"
+        - "--fail"
+        - "--silent"
+        - "http://localhost:8081/nomad-oasis/north/hub/health"
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 10s
+
+  # nomad logtransfer
+  # to enable the logtransfer service run "docker compose --profile with_logtransfer up"
+  logtransfer:
+    restart: unless-stopped
+    image: ghcr.io/hampusnasstrom/nomad-test-image:main
+    container_name: nomad_oasis_logtransfer
+    environment:
+      NOMAD_SERVICE: nomad_oasis_logtransfer
+      NOMAD_ELASTIC_HOST: elastic
+      NOMAD_MONGO_HOST: mongo
+    depends_on:
+      elastic:
+        condition: service_healthy
+      mongo:
+        condition: service_healthy
+    volumes:
+      - ./configs/nomad.yaml:/app/nomad.yaml
+      - ./.volumes/fs:/app/.volumes/fs
+    command: python -m nomad.cli admin run logtransfer
+    profiles: ["with_logtransfer"]
+
+  # nomad proxy (a reverse proxy for nomad)
+  proxy:
+    restart: unless-stopped
+    image: nginx:1.13.9-alpine
+    container_name: nomad_oasis_proxy
+    command: nginx -g 'daemon off;'
+    volumes:
+      - ./configs/nginx.conf:/etc/nginx/conf.d/default.conf
+    depends_on:
+      app:
+        condition: service_healthy
+      worker:
+        condition: service_started # TODO: service_healthy
+      north:
+        condition: service_healthy
+    ports:
+      - 80:80
+
+volumes:
+  mongo:
+    name: "nomad_oasis_mongo"
+  elastic:
+    name: "nomad_oasis_elastic"
+  rabbitmq:
+    name: "nomad_oasis_rabbitmq"
+  keycloak:
+    name: "nomad_oasis_keycloak"
+
+networks:
+  default:
+    name: nomad_oasis_network

plugins.txt

@@ -0,0 +1,3 @@
+nomad-material-processing
+git+https://github.com/FAIRmat-NFDI/nomad-measurements.git@71b7e8c9bb376ce9e8610aba9a20be0b5bce6775
+git+https://github.com/FAIRmat-NFDI/AreaA-data_modeling_and_schemas.git@6632dab23d325be221ae11519ad0a3b1a87c1c10#subdirectory=PVD/PLD/jeremy_ikz/ikz_pld_plugin