feat: ensure that backend Sentry SDK requests are proxied as well (#715)

* feat: ensure that backend Sentry SDK requests are proxied as well

* Add better frontend and backend DSN options

docs/SENTRY.md

@@ -0,0 +1,28 @@
+# Sentry
+
+`visyn_core` has built in support for [Sentry](https://sentry.io/), including sending frontend and backend errors without exposing the underlying DSNs.
+
+## Getting started
+
+First, get a DSN from either [Sentry](https://sentry.io/) or a self-hosted version. It is recommended to create a frontend and backend project, i.e. as outlined in https://docs.sentry.io/product/sentry-basics/distributed-tracing/create-new-project/. Ultimately, you will have two DSNs similar to `https://<key>@<id>.ingest.us.sentry.io/<project-id>`, which you can then use to configure the backend:
+
+```.env
+# DSN for the backend
+VISYN_CORE__SENTRY__BACKEND_DSN=https://<key>@<id>.ingest.us.sentry.io/<backend-id>
+# DSN for the frontend
+VISYN_CORE__SENTRY__FRONTEND_DSN=https://<key>@<id>.ingest.us.sentry.io/<frontend-id>
+```
+
+## FAQ
+
+### What if my Sentry instance is not directly accessible from the browser?
+
+As long as the backend can directly access the Sentry instance (i.e. in k8s via `http://sentry-relay.sentry.svc.cluster.local:3000`), you can do this for both the frontend and backend.
+
+#### Frontend
+
+While the DSN stays the same (incl. the URL which is not directly accessible), you can set the `PROXY_TO` variable to an (internal) Sentry URL accessible from the backend like `http://sentry-relay.sentry.svc.cluster.local:3000`. The frontend will then automatically send all envelopes to the API route `/api/sentry` (instead of the original URL in the DSN), which then proxies it to the accessible URL via the `sentry_proxy_router.py`.
+
+#### Backend
+
+The same `PROXY_TO` applies to the backend DSN, although you can also just modify the URL part of the DSN, changing `https://<key>@<id>.ingest.us.sentry.io/<project-id>` to `http(s)://<key>@<internal-url>/<project-id>`.

package.json

@@ -104,7 +104,7 @@
     "@mantine/styles": "~6.0.22",
     "@mantine/tiptap": "~7.16.1",
     "@mantine6/core": "npm:@mantine/core@~6.0.22",
-    "@sentry/react": "^8.50.0",
+    "@sentry/react": "^9.1.0",
     "@types/d3-hexbin": "^0.2.5",
     "@types/d3v7": "npm:@types/d3@^7.4.3",
     "@types/plotly.js-dist-min": "^2.3.4",

requirements.txt

@@ -25,7 +25,7 @@ python-dateutil==2.9.0.post0
 requests==2.32.3
 # Redis versions according to https://github.com/celery/celery/blob/main/requirements/extras/redis.txt
 redis>=4.5.2,<6.0.0,!=4.5.5
-sentry-sdk~=2.19.2
+sentry-sdk~=2.22.0
 SQLAlchemy>=1.4.40,<=1.4.53
 starlette-context==0.3.6
 # Extras from fastapi[all], which we can't install because it requires pydantic v2: https://github.com/tiangolo/fastapi/blob/master/pyproject.toml#L79-L103

visyn_core/sentry/sentry_proxy_router.py

@@ -14,15 +14,15 @@
 
 @sentry_router.post("/")
 async def proxy_sentry_envelope(request: Request):  # pyright: ignore[reportUnusedFunction]
-    dsn = manager.settings.visyn_core.sentry.dsn
+    dsn = manager.settings.visyn_core.sentry.get_frontend_dsn()
     if dsn:
         async with httpx.AsyncClient(timeout=10) as client:
             # Example to parse a sentry envelope: https://github.com/getsentry/examples/blob/66da5f8c9559f64f1bfa57f8dd9b0731f75cd0e9/tunneling/python/app.py
             envelope = await request.body()
             piece = envelope.split(b"\n")[0].decode("utf-8")
             header = json.loads(piece)
             dsn = urlparse(header.get("dsn"))
-            proxy_to = manager.settings.visyn_core.sentry.proxy_to or f"{dsn.scheme}://{dsn.hostname}"
+            proxy_to = manager.settings.visyn_core.sentry.get_frontend_proxy_to() or f"{dsn.scheme}://{dsn.hostname}"
             project_id = dsn.path.strip("/")
             res = await client.post(
                 url=f"{proxy_to}/api/{project_id}/envelope/",

visyn_core/server/visyn_server.py

@@ -62,19 +62,42 @@ def filter(self, record: logging.LogRecord) -> bool:
 
         init_telemetry(app, settings=manager.settings.visyn_core.telemetry)
 
-    if manager.settings.visyn_core.sentry.dsn:
+    backend_dsn = manager.settings.visyn_core.sentry.get_backend_dsn()
+    if backend_dsn:
+        _log.info(f"Initializing Sentry with DSN {backend_dsn}")
         import sentry_sdk
 
+        # The sentry DSN usually contains the "public" URL of the sentry server, i.e. https://sentry.app-internal.datavisyn.io/...
+        # which is sometimes not accessible due to authentication. Therefore, we allow to proxy the sentry requests to a different URL,
+        # i.e. a cluster internal one without authentication. The same is happening for the frontend with the Sentry proxy router.
+        proxy_to = manager.settings.visyn_core.sentry.get_backend_proxy_to()
+
+        if proxy_to:
+            from urllib.parse import urlparse
+
+            parsed_dsn = urlparse(backend_dsn)
+            parsed_tunnel = urlparse(proxy_to)
+
+            # Replace the scheme and hostname of the dsn with the proxy_to URL, while keeping the rest intact.
+            # I.e. <scheme>://<token>@<url>/<project> becomes <scheme of proxy_to>://<token>:<url of proxy_to>/<project>
+            backend_dsn = backend_dsn.replace(parsed_dsn.scheme, parsed_tunnel.scheme)
+            if parsed_dsn.hostname and parsed_tunnel.hostname:
+                backend_dsn = backend_dsn.replace(parsed_dsn.hostname, parsed_tunnel.hostname)
+
+            _log.info(
+                f"Proxying Sentry from {parsed_dsn.scheme}://{parsed_dsn.hostname} to {parsed_tunnel.scheme}://{parsed_tunnel.hostname}"
+            )
+
         sentry_sdk.init(
-            dsn=manager.settings.visyn_core.sentry.dsn,
+            dsn=backend_dsn,
             # Set traces_sample_rate to 1.0 to capture 100%
             # of transactions for tracing.
             traces_sample_rate=1.0,
             # Set profiles_sample_rate to 1.0 to profile 100%
             # of sampled transactions.
             # We recommend adjusting this value in production.
             profiles_sample_rate=1.0,
-            **manager.settings.visyn_core.sentry.server_init_options,
+            **manager.settings.visyn_core.sentry.backend_init_options,
         )
 
     # Initialize global managers.

visyn_core/settings/client_config.py

@@ -41,8 +41,10 @@ def _get_model():
 @visyn_client_config
 class VisynCoreClientConfigModel(BaseModel):
     env: Literal["development", "production"] = Field(default_factory=lambda: manager.settings.env)
-    sentry_dsn: str | None = Field(default_factory=lambda: manager.settings.visyn_core.sentry.dsn)
-    sentry_proxy_to: str | None = Field(default_factory=lambda: "/api/sentry/" if manager.settings.visyn_core.sentry.proxy_to else None)
+    sentry_dsn: str | None = Field(default_factory=lambda: manager.settings.visyn_core.sentry.get_frontend_dsn())
+    sentry_proxy_to: str | None = Field(
+        default_factory=lambda: "/api/sentry/" if manager.settings.visyn_core.sentry.get_frontend_proxy_to() else None
+    )
 
 
 def init_client_config(app: FastAPI):

visyn_core/settings/model.py

@@ -171,14 +171,42 @@ class SentrySettings(BaseModel):
     """
     Public DSN of the Sentry project.
     """
+    frontend_dsn: str | None = None
+    """
+    Public DSN of the Sentry frontend project.
+    """
+    backend_dsn: str | None = None
+    """
+    Public DSN of the Sentry backend project.
+    """
+    backend_init_options: dict[str, Any] = {}
+    """
+    Options to be passed to the Sentry SDK during initialization.
+    """
     proxy_to: str | None = None
     """
     Proxy Sentry envelopes to this URL. Used if an internal Sentry server is used, otherwise the original DSN is used.
     """
-    server_init_options: dict[str, Any] = {}
+    frontend_proxy_to: str | None = None
     """
-    Options to be passed to the Sentry SDK during initialization.
+    Proxy Sentry frontend envelopes to this URL. Used if an internal Sentry server is used, otherwise the original DSN is used.
+    """
+    backend_proxy_to: str | None = None
     """
+    Proxy Sentry backend envelopes to this URL. Used if an internal Sentry server is used, otherwise the original DSN is used.
+    """
+
+    def get_frontend_dsn(self) -> str | None:
+        return self.frontend_dsn or self.dsn
+
+    def get_backend_dsn(self) -> str | None:
+        return self.backend_dsn or self.dsn
+
+    def get_frontend_proxy_to(self) -> str | None:
+        return self.frontend_proxy_to or self.proxy_to
+
+    def get_backend_proxy_to(self) -> str | None:
+        return self.backend_proxy_to or self.proxy_to
 
 
 class VisynCoreSettings(BaseModel):