From 42211ca07cad4c7e5b1a9bb3d04ff230f26f62a1 Mon Sep 17 00:00:00 2001 From: Phillip Maffettone Date: Wed, 10 Apr 2024 13:14:47 -0400 Subject: [PATCH 1/6] fix: changes to nginx and compose and publishing agents --- containers/mmm5-tax-day-compose.yaml | 18 +++++++++++++++-- containers/nginx/html/index.html | 20 ++++++++++++++----- pdf_agents/agents.py | 5 ++--- .../startup_scripts/mmm5-tax-day/gsas.py | 5 +++++ 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/containers/mmm5-tax-day-compose.yaml b/containers/mmm5-tax-day-compose.yaml index f77b394..0dfac9e 100644 --- a/containers/mmm5-tax-day-compose.yaml +++ b/containers/mmm5-tax-day-compose.yaml @@ -8,21 +8,35 @@ services: build: context: ../gsas dockerfile: Containerfile-conda - command: /bin/bash -c "export BS_AGENT_STARTUP_SCRIPT_PATH=/src/pdf-agents/pdf_agents/startup_scripts/mmm5-tax-day/gsas.py && exec conda run -n GSASII --no-capture-output uvicorn bluesky_adaptive.server:app" + command: conda run -n GSASII --no-capture-output uvicorn bluesky_adaptive.server:app --host 0.0.0.0 + environment: + - TILED_API_KEY=$TILED_API_KEY + - BS_AGENT_STARTUP_SCRIPT_PATH=/src/pdf-agents/pdf_agents/startup_scripts/mmm5-tax-day/gsas.py volumes: - type: bind source: ../pdf_agents target: /src/pdf-agents/pdf_agents read_only: true + - type: bind + source: /etc/bluesky/kafka.yml + target: /etc/bluesky/kafka.yml + read_only: true kmeans-gsas: image: bluesky:latest - command: /bin/bash -c "BS_AGENT_STARTUP_SCRIPT_PATH=/src/pdf-agents/pdf_agents/startup_scripts/mmm5-tax-day/kmeans-gsas.py uvicorn bluesky_adaptive.server:app" + command: uvicorn bluesky_adaptive.server:app --host 0.0.0.0 + environment: + - TILED_API_KEY=$TILED_API_KEY + - BS_AGENT_STARTUP_SCRIPT_PATH=/src/pdf-agents/pdf_agents/startup_scripts/mmm5-tax-day/gsas.py volumes: - type: bind source: ../pdf_agents target: /src/pdf-agents/pdf_agents read_only: true + - type: bind + source: /etc/bluesky/kafka.yml + target: /etc/bluesky/kafka.yml + read_only: true depends_on: gsas: condition: service_started diff --git a/containers/nginx/html/index.html b/containers/nginx/html/index.html index 97a00fc..e1e6561 100644 --- a/containers/nginx/html/index.html +++ b/containers/nginx/html/index.html @@ -57,16 +57,26 @@
-

Welcome!

-

This is the default landing page for our service. Please check back soon for updates.

+

Landing Page for Agent Services!

+

This is the default landing page for the agent services.

-

Our Services

+

Available Agents

+ +

Available Agent User Interfaces

+ diff --git a/pdf_agents/agents.py b/pdf_agents/agents.py index 3727d00..4be6b35 100644 --- a/pdf_agents/agents.py +++ b/pdf_agents/agents.py @@ -370,13 +370,12 @@ def generate_report(self, **kwargs): self.close_and_restart(clear_tell_cache=False, retell_all=False, reason="Per-Run Subscribers") @classmethod - def get_beamline_objects(cls) -> dict: - ret = super().get_beamline_objects() + def get_default_producer(cls) -> Publisher: beamline_tla = "pdf" kafka_config = nslsii.kafka_utils._read_bluesky_kafka_config_file( config_file_path="/etc/bluesky/kafka.yml" ) - ret["report_producer"] = Publisher( + ret = Publisher( topic=f"{beamline_tla}.mmm.bluesky.agents", bootstrap_servers=",".join(kafka_config["bootstrap_servers"]), key="{beamline_tla}.key", diff --git a/pdf_agents/startup_scripts/mmm5-tax-day/gsas.py b/pdf_agents/startup_scripts/mmm5-tax-day/gsas.py index dc1f910..9c7a683 100644 --- a/pdf_agents/startup_scripts/mmm5-tax-day/gsas.py +++ b/pdf_agents/startup_scripts/mmm5-tax-day/gsas.py @@ -20,11 +20,16 @@ } } +report_producer = RefinementAgent.get_default_producer() + + agent = RefinementAgent( # GSAS Args cif_paths=["/src/pdf-agents/assets/fcc.cif"], refinement_params=[param_dict_phase1], inst_param_path="/src/pdf-agents/assets/LaB6_IPF.instprm", + # Required report producer + report_producer=report_producer, # BS Adaptive Args ask_on_tell=False, report_on_tell=True, From 9e2ed95e82d356f65465c608a475493b8696c6f7 Mon Sep 17 00:00:00 2001 From: maffettone Date: Wed, 10 Apr 2024 11:09:30 -0700 Subject: [PATCH 2/6] feat: dash app and nginx --- containers/mmm5-tax-day-compose.yaml | 9 +++++++++ containers/nginx/locs.d/gsas.conf | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/containers/mmm5-tax-day-compose.yaml b/containers/mmm5-tax-day-compose.yaml index 0dfac9e..57ac6a8 100644 --- a/containers/mmm5-tax-day-compose.yaml +++ b/containers/mmm5-tax-day-compose.yaml @@ -22,6 +22,15 @@ services: target: /etc/bluesky/kafka.yml read_only: true + gsas-ui: + image: ghcr.io/maffettone/bluesky-adaptive-ui:latest + command: python3 /src/bluesky-adaptive-ui/bluesky_adaptive_ui/default_dash_app/app.py --agent-address gsas + volumes: + - type: bind + source: ../bluesky-adaptive-ui + target: /src/bluesky-adaptive-ui + read_only: true + kmeans-gsas: image: bluesky:latest command: uvicorn bluesky_adaptive.server:app --host 0.0.0.0 diff --git a/containers/nginx/locs.d/gsas.conf b/containers/nginx/locs.d/gsas.conf index 2f577e7..e86203a 100644 --- a/containers/nginx/locs.d/gsas.conf +++ b/containers/nginx/locs.d/gsas.conf @@ -5,3 +5,11 @@ location /gsas/ { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_read_timeout 60s; } + +location /gsas/ui/ { + proxy_pass http://gsas-ui:8050/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_read_timeout 60s; +} From e7dc548d5e7809e0e220dcde3804340374ca6c80 Mon Sep 17 00:00:00 2001 From: Phillip Maffettone Date: Wed, 10 Apr 2024 14:11:31 -0400 Subject: [PATCH 3/6] fix: root path behind proxy --- containers/mmm5-tax-day-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/mmm5-tax-day-compose.yaml b/containers/mmm5-tax-day-compose.yaml index 0dfac9e..1ef0af8 100644 --- a/containers/mmm5-tax-day-compose.yaml +++ b/containers/mmm5-tax-day-compose.yaml @@ -8,7 +8,7 @@ services: build: context: ../gsas dockerfile: Containerfile-conda - command: conda run -n GSASII --no-capture-output uvicorn bluesky_adaptive.server:app --host 0.0.0.0 + command: conda run -n GSASII --no-capture-output uvicorn bluesky_adaptive.server:app --host 0.0.0.0 --root-path /gsas environment: - TILED_API_KEY=$TILED_API_KEY - BS_AGENT_STARTUP_SCRIPT_PATH=/src/pdf-agents/pdf_agents/startup_scripts/mmm5-tax-day/gsas.py From c76b3b7705b9a600d74a906fafeb6c67978d2674 Mon Sep 17 00:00:00 2001 From: Phillip Maffettone Date: Wed, 10 Apr 2024 15:23:15 -0400 Subject: [PATCH 4/6] fix: nginx for ui and relative paths --- containers/mmm5-tax-day-compose.yaml | 5 ++++- containers/nginx/locs.d/gsas.conf | 9 +++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/containers/mmm5-tax-day-compose.yaml b/containers/mmm5-tax-day-compose.yaml index 35f7db0..2017731 100644 --- a/containers/mmm5-tax-day-compose.yaml +++ b/containers/mmm5-tax-day-compose.yaml @@ -27,7 +27,7 @@ services: command: python3 /src/bluesky-adaptive-ui/bluesky_adaptive_ui/default_dash_app/app.py --agent-address gsas volumes: - type: bind - source: ../bluesky-adaptive-ui + source: ../../bluesky-adaptive-ui target: /src/bluesky-adaptive-ui read_only: true @@ -71,6 +71,9 @@ services: depends_on: gsas: condition: service_started + gsas-ui: + condition: service_started kmeans-gsas: condition: service_started + diff --git a/containers/nginx/locs.d/gsas.conf b/containers/nginx/locs.d/gsas.conf index e86203a..9ee999d 100644 --- a/containers/nginx/locs.d/gsas.conf +++ b/containers/nginx/locs.d/gsas.conf @@ -1,13 +1,14 @@ -location /gsas/ { - proxy_pass http://gsas:8000/; + +location /gsas-ui/ { + proxy_pass http://gsas-ui:8050/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_read_timeout 60s; } -location /gsas/ui/ { - proxy_pass http://gsas-ui:8050/; +location /gsas/ { + proxy_pass http://gsas:8000/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; From 83bdb6b0d3773ec55afb423657cb68c82260220b Mon Sep 17 00:00:00 2001 From: maffettone Date: Wed, 10 Apr 2024 12:35:14 -0700 Subject: [PATCH 5/6] refactor: update addressing in nginx and dash --- containers/mmm5-tax-day-compose.yaml | 10 ++++++---- containers/nginx/html/index.html | 10 +++++----- containers/nginx/locs.d/gsas.conf | 6 +++--- containers/nginx/locs.d/kmeans-gsas.conf | 2 +- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/containers/mmm5-tax-day-compose.yaml b/containers/mmm5-tax-day-compose.yaml index 2017731..b1ace0b 100644 --- a/containers/mmm5-tax-day-compose.yaml +++ b/containers/mmm5-tax-day-compose.yaml @@ -3,12 +3,12 @@ version: '3' services: - gsas: + gsas-agent: image: gsas:conda build: context: ../gsas dockerfile: Containerfile-conda - command: conda run -n GSASII --no-capture-output uvicorn bluesky_adaptive.server:app --host 0.0.0.0 --root-path /gsas + command: conda run -n GSASII --no-capture-output uvicorn bluesky_adaptive.server:app --host 0.0.0.0 --root-path /gsas-agent environment: - TILED_API_KEY=$TILED_API_KEY - BS_AGENT_STARTUP_SCRIPT_PATH=/src/pdf-agents/pdf_agents/startup_scripts/mmm5-tax-day/gsas.py @@ -25,15 +25,17 @@ services: gsas-ui: image: ghcr.io/maffettone/bluesky-adaptive-ui:latest command: python3 /src/bluesky-adaptive-ui/bluesky_adaptive_ui/default_dash_app/app.py --agent-address gsas + environment: + - DASH_REQUEST_PATHNAME_PREFIX="/gsas/" volumes: - type: bind source: ../../bluesky-adaptive-ui target: /src/bluesky-adaptive-ui read_only: true - kmeans-gsas: + kmeans-gsas-agent: image: bluesky:latest - command: uvicorn bluesky_adaptive.server:app --host 0.0.0.0 + command: uvicorn bluesky_adaptive.server:app --host 0.0.0.0 --root-path /kmeans-gsas-agent environment: - TILED_API_KEY=$TILED_API_KEY - BS_AGENT_STARTUP_SCRIPT_PATH=/src/pdf-agents/pdf_agents/startup_scripts/mmm5-tax-day/gsas.py diff --git a/containers/nginx/html/index.html b/containers/nginx/html/index.html index e1e6561..0bafadf 100644 --- a/containers/nginx/html/index.html +++ b/containers/nginx/html/index.html @@ -64,19 +64,19 @@

Landing Page for Agent Services!

Available Agents

-

Available Agent User Interfaces

+

Available Agent User Interfaces

diff --git a/containers/nginx/locs.d/gsas.conf b/containers/nginx/locs.d/gsas.conf index 9ee999d..4a7b68d 100644 --- a/containers/nginx/locs.d/gsas.conf +++ b/containers/nginx/locs.d/gsas.conf @@ -1,5 +1,5 @@ -location /gsas-ui/ { +location /gsas/ { proxy_pass http://gsas-ui:8050/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -7,8 +7,8 @@ location /gsas-ui/ { proxy_read_timeout 60s; } -location /gsas/ { - proxy_pass http://gsas:8000/; +location /gsas-agent/ { + proxy_pass http://gsas-agent:8000/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/containers/nginx/locs.d/kmeans-gsas.conf b/containers/nginx/locs.d/kmeans-gsas.conf index 4d38c23..daefb96 100644 --- a/containers/nginx/locs.d/kmeans-gsas.conf +++ b/containers/nginx/locs.d/kmeans-gsas.conf @@ -1,4 +1,4 @@ -location /kmeans-gsas/ { +location /kmeans-gsas-agent/ { proxy_pass http://kmeans-gsas:8000/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; From 7785df1d06c288296d00ba57ad3ec8403d04eead Mon Sep 17 00:00:00 2001 From: Phillip Maffettone Date: Wed, 10 Apr 2024 17:26:46 -0400 Subject: [PATCH 6/6] fix: add dns override and address updates --- containers/mmm5-tax-day-compose.yaml | 6 ++++-- containers/modify_dns.sh | 13 +++++++++++++ containers/nginx/locs.d/kmeans-gsas.conf | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100755 containers/modify_dns.sh diff --git a/containers/mmm5-tax-day-compose.yaml b/containers/mmm5-tax-day-compose.yaml index b1ace0b..38bd87b 100644 --- a/containers/mmm5-tax-day-compose.yaml +++ b/containers/mmm5-tax-day-compose.yaml @@ -24,10 +24,12 @@ services: gsas-ui: image: ghcr.io/maffettone/bluesky-adaptive-ui:latest - command: python3 /src/bluesky-adaptive-ui/bluesky_adaptive_ui/default_dash_app/app.py --agent-address gsas + entrypoint: /modify_dns.sh + command: python3 /src/bluesky-adaptive-ui/bluesky_adaptive_ui/default_dash_app/app.py --agent-address gsas-agent --agent-port 8000 environment: - DASH_REQUEST_PATHNAME_PREFIX="/gsas/" volumes: + - ./modify_dns.sh:/modify_dns.sh - type: bind source: ../../bluesky-adaptive-ui target: /src/bluesky-adaptive-ui @@ -71,7 +73,7 @@ services: target: /var/www/html read_only: true depends_on: - gsas: + gsas-agent: condition: service_started gsas-ui: condition: service_started diff --git a/containers/modify_dns.sh b/containers/modify_dns.sh new file mode 100755 index 0000000..2358694 --- /dev/null +++ b/containers/modify_dns.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Extract the first nameserver entry +nameserver=$(grep -m 1 'nameserver' /etc/resolv.conf) + +# Recreate resolv.conf with the desired content +{ + echo "search dns.podman" + echo "$nameserver" +} > /etc/resolv.conf + +# Execute the original CMD passed to the script +exec "$@" diff --git a/containers/nginx/locs.d/kmeans-gsas.conf b/containers/nginx/locs.d/kmeans-gsas.conf index daefb96..a83d763 100644 --- a/containers/nginx/locs.d/kmeans-gsas.conf +++ b/containers/nginx/locs.d/kmeans-gsas.conf @@ -1,5 +1,5 @@ location /kmeans-gsas-agent/ { - proxy_pass http://kmeans-gsas:8000/; + proxy_pass http://kmeans-gsas-agent:8000/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;