From f965072839d6a51555f72f2def3a5bb734f5ed39 Mon Sep 17 00:00:00 2001 From: Jordi Bisbal Ansaldo Date: Tue, 23 Jan 2024 16:58:48 +0100 Subject: [PATCH 01/11] Add locust_plugins as a dependency --- src/loadgenerator/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/loadgenerator/requirements.txt b/src/loadgenerator/requirements.txt index 0f7bb7552e..c522facc86 100644 --- a/src/loadgenerator/requirements.txt +++ b/src/loadgenerator/requirements.txt @@ -16,6 +16,7 @@ idna==3.4 itsdangerous==2.1.2 jinja2==3.1.2 locust==2.18.2 +locust_plugins==3.4.0 markupsafe==2.1.3 msgpack==1.0.7 opentelemetry-api==1.21.0 From 94e143511d403d0df8e18f6f000b21bb2dfe8618 Mon Sep 17 00:00:00 2001 From: Jordi Bisbal Ansaldo Date: Tue, 23 Jan 2024 16:59:18 +0100 Subject: [PATCH 02/11] Increased memory constraints and introduced LOCUST_BROWSER_TRAFFIC_ENABLED environment variable --- .env | 1 + docker-compose.yml | 3 ++- kubernetes/opentelemetry-demo.yaml | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 32614ca3c0..8d53b699e3 100644 --- a/.env +++ b/.env @@ -67,6 +67,7 @@ LOCUST_HOST=http://${FRONTEND_PROXY_ADDR} LOCUST_WEB_HOST=loadgenerator LOCUST_AUTOSTART=true LOCUST_HEADLESS=false +LOCUST_BROWSER_TRAFFIC_ENABLED=false # Payment Service PAYMENT_SERVICE_PORT=50051 diff --git a/docker-compose.yml b/docker-compose.yml index 79fcfb69b0..fe97748646 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -371,7 +371,7 @@ services: deploy: resources: limits: - memory: 120M + memory: 1G restart: unless-stopped ports: - "${LOCUST_WEB_PORT}" @@ -381,6 +381,7 @@ services: - LOCUST_HOST - LOCUST_HEADLESS - LOCUST_AUTOSTART + - LOCUST_BROWSER_TRAFFIC_ENABLED - OTEL_EXPORTER_OTLP_ENDPOINT - OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE - OTEL_RESOURCE_ATTRIBUTES diff --git a/kubernetes/opentelemetry-demo.yaml b/kubernetes/opentelemetry-demo.yaml index 0916e91228..48a7450378 100644 --- a/kubernetes/opentelemetry-demo.yaml +++ b/kubernetes/opentelemetry-demo.yaml @@ -10041,6 +10041,8 @@ spec: value: "false" - name: LOCUST_AUTOSTART value: "true" + - name: LOCUST_BROWSER_TRAFFIC_ENABLED + value: "false" - name: PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION value: python - name: OTEL_EXPORTER_OTLP_ENDPOINT @@ -10049,7 +10051,7 @@ spec: value: service.name=$(OTEL_SERVICE_NAME),service.namespace=opentelemetry-demo resources: limits: - memory: 120Mi + memory: 1Gi --- # Source: opentelemetry-demo/templates/component.yaml apiVersion: apps/v1 From c4181f8f9d9c1b07e7b11512e5f88ffa985ae606 Mon Sep 17 00:00:00 2001 From: Jordi Bisbal Ansaldo Date: Tue, 23 Jan 2024 17:00:27 +0100 Subject: [PATCH 03/11] Added playwright on load generator docker image --- src/loadgenerator/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/loadgenerator/Dockerfile b/src/loadgenerator/Dockerfile index 92f4da7d90..816c9369a1 100644 --- a/src/loadgenerator/Dockerfile +++ b/src/loadgenerator/Dockerfile @@ -17,4 +17,6 @@ WORKDIR /usr/src/app/ COPY --from=builder /reqs /usr/local COPY ./src/loadgenerator/locustfile.py . COPY ./src/loadgenerator/people.json . +ENV LOCUST_PLAYWRIGHT=1 +RUN playwright install --with-deps chromium ENTRYPOINT locust From 7da7c1b42c3ba65e9358ba4919bf2a4ae0d6fb7b Mon Sep 17 00:00:00 2001 From: Jordi Bisbal Ansaldo Date: Tue, 23 Jan 2024 17:01:49 +0100 Subject: [PATCH 04/11] Inceased delay interval in milliseconds between two consecutive exports --- src/frontend/utils/telemetry/FrontendTracer.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/frontend/utils/telemetry/FrontendTracer.ts b/src/frontend/utils/telemetry/FrontendTracer.ts index 1918c9cde9..1720409b0c 100644 --- a/src/frontend/utils/telemetry/FrontendTracer.ts +++ b/src/frontend/utils/telemetry/FrontendTracer.ts @@ -34,7 +34,9 @@ const FrontendTracer = async (collectorString: string) => { new BatchSpanProcessor( new OTLPTraceExporter({ url: NEXT_PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT || collectorString || 'http://localhost:4318/v1/traces', - }) + }), { + scheduledDelayMillis : 500 + } ) ); From 50b06b7261102ac30c61c90fd4a6933cb2fcd80d Mon Sep 17 00:00:00 2001 From: Jordi Bisbal Ansaldo Date: Tue, 23 Jan 2024 17:02:22 +0100 Subject: [PATCH 05/11] Use different otel endpoint when request contains the synthetic_request=true header --- src/frontend/pages/_document.tsx | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/frontend/pages/_document.tsx b/src/frontend/pages/_document.tsx index db6dac7e91..5890885412 100644 --- a/src/frontend/pages/_document.tsx +++ b/src/frontend/pages/_document.tsx @@ -3,16 +3,9 @@ import Document, { DocumentContext, Html, Head, Main, NextScript } from 'next/document'; import { ServerStyleSheet } from 'styled-components'; +import {context, propagation} from "@opentelemetry/api"; -const { ENV_PLATFORM, WEB_OTEL_SERVICE_NAME, PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT } = process.env; - -const envString = ` -window.ENV = { - NEXT_PUBLIC_PLATFORM: '${ENV_PLATFORM}', - NEXT_PUBLIC_OTEL_SERVICE_NAME: '${WEB_OTEL_SERVICE_NAME}', - NEXT_PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: '${PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}', -}; -`; +const { ENV_PLATFORM, WEB_OTEL_SERVICE_NAME, PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, OTEL_COLLECTOR_NAME} = process.env; export default class MyDocument extends Document<{ envString: string }> { static async getInitialProps(ctx: DocumentContext) { @@ -26,6 +19,19 @@ export default class MyDocument extends Document<{ envString: string }> { }); const initialProps = await Document.getInitialProps(ctx); + const baggage = propagation.getBaggage(context.active()); + const isSyntheticRequest = baggage?.getEntry('synthetic_request')?.value === 'true'; + + const otlpTracesEndpoint = isSyntheticRequest + ? `http://${OTEL_COLLECTOR_NAME}:4318/v1/traces` + : PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT; + + const envString = ` + window.ENV = { + NEXT_PUBLIC_PLATFORM: '${ENV_PLATFORM}', + NEXT_PUBLIC_OTEL_SERVICE_NAME: '${WEB_OTEL_SERVICE_NAME}', + NEXT_PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: '${otlpTracesEndpoint}', + };`; return { ...initialProps, styles: [initialProps.styles, sheet.getStyleElement()], From b63991c3892d7aab2deec63dbbb7ea7a9323a5ef Mon Sep 17 00:00:00 2001 From: Jordi Bisbal Ansaldo Date: Tue, 23 Jan 2024 17:02:52 +0100 Subject: [PATCH 06/11] Add locust example using playwright. The header synthetic_request is set to true to identify them in the frontend --- src/loadgenerator/locustfile.py | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/loadgenerator/locustfile.py b/src/loadgenerator/locustfile.py index 8065f07e9e..42a6891694 100644 --- a/src/loadgenerator/locustfile.py +++ b/src/loadgenerator/locustfile.py @@ -5,9 +5,11 @@ import json +import os import random import uuid from locust import HttpUser, task, between +from locust_plugins.users.playwright import PlaywrightUser, pw, PageWithRetry, event from opentelemetry import context, baggage, trace from opentelemetry.metrics import set_meter_provider @@ -21,6 +23,7 @@ from opentelemetry.instrumentation.requests import RequestsInstrumentor from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor from opentelemetry.instrumentation.urllib3 import URLLib3Instrumentor +from playwright.async_api import Route, Request exporter = OTLPMetricExporter(insecure=True) set_meter_provider(MeterProvider([PeriodicExportingMetricReader(exporter)])) @@ -130,3 +133,42 @@ def on_start(self): context.attach(ctx) self.index() + +browser_traffic_enabled = os.environ.get('LOCUST_BROWSER_TRAFFIC_ENABLED', False) + +if browser_traffic_enabled: + class WebsiteBrowserUser(PlaywrightUser): + headless = True # to use a headless browser, without a GUI + + @task + @pw + async def open_cart_page_and_change_currency(self, page: PageWithRetry): + try: + page.on("console", lambda msg: print(msg.text)) + await page.route('**/*', add_baggage_header) + await page.goto("/cart", wait_until="domcontentloaded") + await page.select_option('[name="currency_code"]', 'CHF') + await page.wait_for_timeout(2000) # giving the browser time to export the traces + except: + pass + + @task + @pw + async def add_product_to_cart(self, page: PageWithRetry): + try: + page.on("console", lambda msg: print(msg.text)) + await page.route('**/*', add_baggage_header) + await page.goto("/", wait_until="domcontentloaded") + await page.click('p:has-text("Roof Binoculars")', wait_until="domcontentloaded") + await page.click('button:has-text("Add To Cart")', wait_until="domcontentloaded") + await page.wait_for_timeout(2000) # giving the browser time to export the traces + except: + pass + + +async def add_baggage_header(route: Route, request: Request): + headers = { + **request.headers, + 'baggage': 'synthetic_request=true' + } + await route.continue_(headers=headers) From 1d91a4fb6b2da0cab101da8eea3b23cb2b0f0689 Mon Sep 17 00:00:00 2001 From: Jordi Bisbal Ansaldo Date: Wed, 24 Jan 2024 09:27:55 +0100 Subject: [PATCH 07/11] Updated changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90e51e67d3..c532acd6b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ release. ([#1335](https://github.com/open-telemetry/opentelemetry-demo/pull/1335)) * [ffspostgres] define and use demo specific postgres image ([#1338](https://github.com/open-telemetry/opentelemetry-demo/pull/1338)) +* [loadgenerator, frontend] enable browser traffic in loadgenerator using playwright + ([#1345](https://github.com/open-telemetry/opentelemetry-demo/pull/1345)) ## 1.7.2 From d3894993273c829dc7d5ef0ca01e7e313385ea64 Mon Sep 17 00:00:00 2001 From: Jordi Bisbal Ansaldo Date: Fri, 26 Jan 2024 16:39:19 +0100 Subject: [PATCH 08/11] Use the same otelName (OTEL_COLLECTOR_HOST) for both k8s and docker --- docker-compose.yml | 1 + kubernetes/opentelemetry-demo.yaml | 2 ++ src/frontend/pages/_document.tsx | 4 ++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fe97748646..9e6de5731e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -296,6 +296,7 @@ services: - PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT - OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE - WEB_OTEL_SERVICE_NAME=frontend-web + - OTEL_COLLECTOR_HOST depends_on: adservice: condition: service_started diff --git a/kubernetes/opentelemetry-demo.yaml b/kubernetes/opentelemetry-demo.yaml index 48a7450378..54827c14bb 100644 --- a/kubernetes/opentelemetry-demo.yaml +++ b/kubernetes/opentelemetry-demo.yaml @@ -9824,6 +9824,8 @@ spec: value: http://localhost:8080/otlp-http/v1/traces - name: OTEL_RESOURCE_ATTRIBUTES value: service.name=$(OTEL_SERVICE_NAME),service.namespace=opentelemetry-demo + - name: OTEL_COLLECTOR_HOST + value: $(OTEL_COLLECTOR_NAME) resources: limits: memory: 200Mi diff --git a/src/frontend/pages/_document.tsx b/src/frontend/pages/_document.tsx index 5890885412..33dabe0a65 100644 --- a/src/frontend/pages/_document.tsx +++ b/src/frontend/pages/_document.tsx @@ -5,7 +5,7 @@ import Document, { DocumentContext, Html, Head, Main, NextScript } from 'next/do import { ServerStyleSheet } from 'styled-components'; import {context, propagation} from "@opentelemetry/api"; -const { ENV_PLATFORM, WEB_OTEL_SERVICE_NAME, PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, OTEL_COLLECTOR_NAME} = process.env; +const { ENV_PLATFORM, WEB_OTEL_SERVICE_NAME, PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, OTEL_COLLECTOR_HOST} = process.env; export default class MyDocument extends Document<{ envString: string }> { static async getInitialProps(ctx: DocumentContext) { @@ -23,7 +23,7 @@ export default class MyDocument extends Document<{ envString: string }> { const isSyntheticRequest = baggage?.getEntry('synthetic_request')?.value === 'true'; const otlpTracesEndpoint = isSyntheticRequest - ? `http://${OTEL_COLLECTOR_NAME}:4318/v1/traces` + ? `http://${OTEL_COLLECTOR_HOST}:4318/v1/traces` : PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT; const envString = ` From 4b93ae0c8449537de2b28f9bb714fa3c8e5087a9 Mon Sep 17 00:00:00 2001 From: Jordi Bisbal Ansaldo Date: Mon, 29 Jan 2024 16:27:45 +0100 Subject: [PATCH 09/11] Removed select option event --- src/loadgenerator/locustfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/loadgenerator/locustfile.py b/src/loadgenerator/locustfile.py index 42a6891694..13b538ad41 100644 --- a/src/loadgenerator/locustfile.py +++ b/src/loadgenerator/locustfile.py @@ -142,12 +142,12 @@ class WebsiteBrowserUser(PlaywrightUser): @task @pw - async def open_cart_page_and_change_currency(self, page: PageWithRetry): + async def open_cart_page_and_click_button(self, page: PageWithRetry): try: page.on("console", lambda msg: print(msg.text)) await page.route('**/*', add_baggage_header) await page.goto("/cart", wait_until="domcontentloaded") - await page.select_option('[name="currency_code"]', 'CHF') + await page.click('button:has-text("Continue Shopping")', wait_until="domcontentloaded") await page.wait_for_timeout(2000) # giving the browser time to export the traces except: pass From e556c4a7728bd51fbe04aa2753e7962e39fe5be8 Mon Sep 17 00:00:00 2001 From: Jordi Bisbal Ansaldo Date: Mon, 29 Jan 2024 16:37:15 +0100 Subject: [PATCH 10/11] Revert - Removed select option event --- src/loadgenerator/locustfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/loadgenerator/locustfile.py b/src/loadgenerator/locustfile.py index 13b538ad41..42a6891694 100644 --- a/src/loadgenerator/locustfile.py +++ b/src/loadgenerator/locustfile.py @@ -142,12 +142,12 @@ class WebsiteBrowserUser(PlaywrightUser): @task @pw - async def open_cart_page_and_click_button(self, page: PageWithRetry): + async def open_cart_page_and_change_currency(self, page: PageWithRetry): try: page.on("console", lambda msg: print(msg.text)) await page.route('**/*', add_baggage_header) await page.goto("/cart", wait_until="domcontentloaded") - await page.click('button:has-text("Continue Shopping")', wait_until="domcontentloaded") + await page.select_option('[name="currency_code"]', 'CHF') await page.wait_for_timeout(2000) # giving the browser time to export the traces except: pass From fd6bc2726da06164f83bb230bfd3927a52fa02e5 Mon Sep 17 00:00:00 2001 From: Jordi Bisbal Ansaldo Date: Tue, 30 Jan 2024 10:26:25 +0100 Subject: [PATCH 11/11] Added IS_SYNTHETIC_REQUEST window.env used to set the span web attribute in FrontendTracer.ts --- src/frontend/pages/_app.tsx | 1 + src/frontend/pages/_document.tsx | 1 + src/frontend/utils/telemetry/FrontendTracer.ts | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/frontend/pages/_app.tsx b/src/frontend/pages/_app.tsx index e5e1523277..6da83d3664 100755 --- a/src/frontend/pages/_app.tsx +++ b/src/frontend/pages/_app.tsx @@ -17,6 +17,7 @@ declare global { NEXT_PUBLIC_PLATFORM?: string; NEXT_PUBLIC_OTEL_SERVICE_NAME?: string; NEXT_PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT?: string; + IS_SYNTHETIC_REQUEST?: string; }; } } diff --git a/src/frontend/pages/_document.tsx b/src/frontend/pages/_document.tsx index 33dabe0a65..15e978d4d8 100644 --- a/src/frontend/pages/_document.tsx +++ b/src/frontend/pages/_document.tsx @@ -31,6 +31,7 @@ export default class MyDocument extends Document<{ envString: string }> { NEXT_PUBLIC_PLATFORM: '${ENV_PLATFORM}', NEXT_PUBLIC_OTEL_SERVICE_NAME: '${WEB_OTEL_SERVICE_NAME}', NEXT_PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: '${otlpTracesEndpoint}', + IS_SYNTHETIC_REQUEST: '${isSyntheticRequest}', };`; return { ...initialProps, diff --git a/src/frontend/utils/telemetry/FrontendTracer.ts b/src/frontend/utils/telemetry/FrontendTracer.ts index 1720409b0c..c7ccc83c8e 100644 --- a/src/frontend/utils/telemetry/FrontendTracer.ts +++ b/src/frontend/utils/telemetry/FrontendTracer.ts @@ -12,7 +12,7 @@ import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; import { SessionIdProcessor } from './SessionIdProcessor'; import { detectResourcesSync } from '@opentelemetry/resources/build/src/detect-resources'; -const { NEXT_PUBLIC_OTEL_SERVICE_NAME = '', NEXT_PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = '' } = +const { NEXT_PUBLIC_OTEL_SERVICE_NAME = '', NEXT_PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = '', IS_SYNTHETIC_REQUEST = '' } = typeof window !== 'undefined' ? window.ENV : {}; const FrontendTracer = async (collectorString: string) => { @@ -57,7 +57,7 @@ const FrontendTracer = async (collectorString: string) => { propagateTraceHeaderCorsUrls: /.*/, clearTimingResources: true, applyCustomAttributesOnSpan(span) { - span.setAttribute('app.synthetic_request', 'false'); + span.setAttribute('app.synthetic_request', IS_SYNTHETIC_REQUEST); }, }, }),