🐛 Fix: Make osparc.local functional again - various minor fixes

.pylintrc

@@ -58,7 +58,7 @@ ignore-paths=^.*\\generated_models\\.*$|^.*/generated_models/.*$
 # Files or directories matching the regex patterns are skipped. The regex
 # matches against base names, not paths. The default value ignores Emacs file
 # locks
-ignore-patterns=venv,.venv
+ignore-patterns=venv,.venv,jupyter_server_config.py
 
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime

.python-version

@@ -1 +1 @@
-3.11
+3.10

Makefile

@@ -26,7 +26,7 @@ certificates/domain.key:
 	# Done: Creating docker secrets
 
 .PHONY: up-local
-up-local: .install-fqdn certificates/domain.crt certificates/domain.key .create-secrets ## deploy osparc ops stacks and simcore, use minio_disabled=1 if minio s3 should not be started (if you have custom S3 set up)
+up-local: .init .venv .install-fqdn certificates/domain.crt certificates/domain.key .create-secrets ## deploy osparc ops stacks and simcore, use minio_disabled=1 if minio s3 should not be started (if you have custom S3 set up)
 	@bash scripts/deployments/deploy_everything_locally.bash --stack_target=local --minio_enabled=0 --vcs_check=1
 	@$(MAKE) info-local
 
@@ -71,15 +71,6 @@ down-maintenance: ## Stop the maintenance mode
 	fi \
 	,)
 
-
-.PHONY: venv
-venv: .venv ## Creates a python virtual environment with dev tools (pip, pylint, ...)
-.venv:
-	@python3 -m venv .venv
-	@.venv/bin/pip3 install --upgrade pip wheel setuptools
-	@.venv/bin/pip3 install typer
-	@echo "To activate the venv, execute 'source .venv/bin/activate'"
-
 # Misc: info & clean
 .PHONY: info info-vars info-local
 info: ## Displays some important info

scripts/common.Makefile

@@ -7,14 +7,12 @@ VERSION := $(shell uname -a)
 
 # Checks for handling various operating systems
 ifeq ($(filter Windows_NT,$(OS)),)
-IS_WSL  := $(if $(findstring microsoft,$(shell uname -a | tr '[:upper:]' '[:lower:]')),WSL,)
 IS_WSL2 := $(if $(findstring -microsoft-,$(shell uname -a)),WSL2,)
 IS_OSX  := $(filter Darwin,$(shell uname -a))
 IS_LINUX:= $(if $(or $(IS_WSL),$(IS_OSX)),,$(filter Linux,$(shell uname -a)))
 endif
 IS_WIN  := $(strip $(if $(or $(IS_LINUX),$(IS_OSX),$(IS_WSL)),,$(OS)))
 
-$(if $(IS_WIN),$(error Windows is not supported in all recipes. Use WSL2 instead. Follow instructions in README.md),)
 $(if $(IS_WSL2),,$(if $(IS_WSL),$(error WSL1 is not supported in all recipes. Use WSL2 instead. Follow instructions in README.md),))
 
 # Check that a valid location to a config file is set.
@@ -243,20 +241,21 @@ clean-default: .check_clean ## Cleans all outputs
 	# creating virtual environment with tooling (jinja, etc)
 	@python3 -m venv .venv
 	@.venv/bin/pip3 install --upgrade pip wheel setuptools
-	@.venv/bin/pip3 install jinja2 j2cli[yaml]
+	@.venv/bin/pip3 install jinja2 j2cli[yaml] typer
+	@echo "To activate the venv, execute 'source .venv/bin/activate'"
 
 
 # https://github.com/kolypto/j2cli?tab=readme-ov-file#customization
 ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes)
 
 define jinja
-	.venv/bin/j2 --format=env $(1) .env -o $(2) --customize j2cli_customization.py
+	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) .env -o $(2) --customize j2cli_customization.py
 endef
 
 else
 
 define jinja
-	.venv/bin/j2 --format=env $(1) .env -o $(2)
+	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) .env -o $(2)
 endef
 
 endif

scripts/create-s3-bucket.bash

@@ -11,5 +11,5 @@ IFS=$'\n\t'
 docker run \
 -v /etc/ssl/certs:/etc/ssl/certs:ro \
 --network host \
---env MC_HOST_local="https://${S3_ACCESS_KEY}:${S3_SECRET_KEY}@${S3_ENDPOINT}" \
-minio/mc:RELEASE.2023-06-19T19-31-19Z mb --ignore-existing local/"$1"
+--env MC_HOST_local="https://${S3_ACCESS_KEY}:${S3_SECRET_KEY}@${STORAGE_DOMAIN}" \
+minio/mc:RELEASE.2023-06-19T19-31-19Z mb --insecure --ignore-existing local/"$1"

scripts/deployments/deploy_everything_locally.sh → scripts/deployments/deploy_everything_locally.bash

@@ -235,6 +235,9 @@ if [ "$start_opsstack" -eq 0 ]; then
     # -------------------------------- ADMIN-PANELS -------------------------------
 
     log_info "starting admin-panels..."
+    # Check if the stack 'admin-panels' exists and delete it if it does
+    # shellcheck disable=2015
+    docker stack ls | grep -q admin-panels && docker stack rm admin-panels >/dev/null 2>&1 || true
     # Pushd because a call with call_make trigger a strange behavior
     pushd "${repo_basedir}"/services/admin-panels;
     call_make "." up-"$stack_target";

scripts/deployments/start_simcore_locally.bash

@@ -71,6 +71,7 @@ if [[ "$devel_repo_path" = "0" ]] ; then
     #
     #       IF GETREPO DOESNT EXIST
     if [ ! -d osparc-simcore ]; then
+        export GIT_SIMCORE_REPO_URL="https://github.com/ITISFoundation/osparc-simcore.git"
         git clone "$GIT_SIMCORE_REPO_URL"
     fi
     #       FI

services/graylog/scripts/configure.py

@@ -48,7 +48,9 @@
     before=before_log(logger, logging.INFO),
 )
 def wait_graylog_is_online():
-    _r = requests.get(GRAYLOG_BASE_DOMAIN + "/api/system", auth=REQUESTS_AUTH)
+    _r = requests.get(
+        GRAYLOG_BASE_DOMAIN + "/api/system", auth=REQUESTS_AUTH, verify=False
+    )
 
     if _r.status_code == 401:
         raise TypeError(f"Graylog unauthorized HTTP response: {_r}")
@@ -58,7 +60,9 @@ def wait_graylog_is_online():
 
 
 def validate_graylog_version_is_supported():
-    _r = requests.get(GRAYLOG_BASE_DOMAIN + "/api/system", auth=REQUESTS_AUTH)
+    _r = requests.get(
+        GRAYLOG_BASE_DOMAIN + "/api/system", auth=REQUESTS_AUTH, verify=False
+    )
     _r.raise_for_status()
 
     graylog_version = _r.json()["version"]

services/registry/Makefile

@@ -24,7 +24,7 @@ endef
 .PHONY: up-local
 up-local: .init .env ${TEMP_COMPOSE}-local  ## Deploys registry stack
 	@$(create-s3-bucket)
-	docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} ${STACK_NAME}
+	docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-local ${STACK_NAME}
 
 .PHONY: up-letsencrypt-http  ## Deploys registry stack using let's encrypt http challenge
 up-letsencrypt-http: .init .env ${TEMP_COMPOSE}-letsencrypt-http
@@ -48,8 +48,6 @@ up-master: up-dalco
 .PHONY: up-public ## Deploys registry on public cluster
 up-public: up-dalco
 
-.PHONY: up-local ## Deploys registry on local deployment
-up-local: up
 
 # Helpers -------------------------------------------------
 

services/registry/template.env

@@ -10,7 +10,7 @@ REGISTRY_S3_ACCESS_KEY=${REGISTRY_S3_ACCESS_KEY}
 REGISTRY_S3_SECRET_KEY=${REGISTRY_S3_SECRET_KEY}
 S3_ACCESS_KEY=${REGISTRY_S3_ACCESS_KEY}
 S3_SECRET_KEY=${REGISTRY_S3_SECRET_KEY}
-S3_ENDPOINT=${S3_ENDPOINT}
+STORAGE_DOMAIN=${STORAGE_DOMAIN}
 S3_BUCKET=${REGISTRY_DOMAIN}
 REGISTRY_S3_BUCKET=${REGISTRY_S3_BUCKET}
 REGISTRY_S3_ENDPOINT=${REGISTRY_S3_ENDPOINT}

services/traefik/docker-compose.local.yml

@@ -11,6 +11,13 @@ services:
       - "--api.dashboard=true"
       - "--log.level=${OPS_TRAEFIK_LOGLEVEL}"
       - "--accesslog=true"
+      - "--accesslog.format=json"
+      - "--accesslog.fields.defaultmode=keep"
+      - "--accesslog.fields.names.ClientUsername=keep"
+      - "--accesslog.fields.headers.defaultmode=keep"
+      - "--accesslog.fields.headers.names.User-Agent=keep"
+      - "--accesslog.fields.headers.names.Authorization=drop"
+      - "--accesslog.fields.headers.names.Content-Type=keep"
       - "--metrics.prometheus=true"
       - "--metrics.prometheus.addEntryPointsLabels=true"
       - "--metrics.prometheus.addServicesLabels=true"
@@ -26,6 +33,9 @@ services:
       - "--entryPoints.https.transport.respondingTimeouts.writeTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805
       - "--entryPoints.https.transport.respondingTimeouts.readTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805
       - "--entryPoints.master_postgres.address=:5432"
+      - "--entrypoints.http.http.redirections.entrypoint.to=https"
+      - "--entrypoints.http.http.redirections.entrypoint.scheme=https"
+      - "--entrypoints.http.http.redirections.entrypoint.permanent=true"
       - "--providers.swarm.endpoint=unix:///var/run/docker.sock"
       - "--providers.swarm.exposedByDefault=false"
       - "--providers.swarm.constraints=!LabelRegex(`io.simcore.zone`, `${TRAEFIK_SIMCORE_ZONE}`)"
@@ -45,12 +55,6 @@ services:
     deploy:
       placement:
         constraints: []
-      labels:
-        # redirect http to https
-        - traefik.http.middlewares.http_to_https.redirectScheme.scheme=https
-        - traefik.http.routers.http_to_https.rule=HostRegexp(`(?P<host>.+)`)
-        - traefik.http.routers.http_to_https.entrypoints=http
-        - traefik.http.routers.http_to_https.middlewares=http_to_https
     env_file:
       - .env
     configs: