✨ Enable grafana for visual execution (#73)

lhjnilsson · Jan 10, 2025 · 35f79b0 · 35f79b0 · github-actions · Jan 11, 2025
1 parent 335d67f
commit 35f79b0
Show file tree

Hide file tree

Showing 10 changed files with 430 additions and 110 deletions.
diff --git a/.github/workflows/on_commit.yaml b/.github/workflows/on_commit.yaml
@@ -122,6 +122,30 @@ jobs:
           path: /tmp/foreverbull.tar
           retention-days: 1
 
+  build-docker-grafana:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    defaults:
+      run:
+        working-directory: ./grafana
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build Grafana
+        uses: docker/build-push-action@v6
+        with:
+          context: grafana
+          file: ./grafana/Dockerfile
+          tags: lhjnilsson/fb-grafana:on_commit
+          outputs: type=docker,dest=/tmp/fb_grafana.tar
+      - name: Upload Grafana
+        uses: actions/upload-artifact@v4
+        with:
+          name: fb-grafana
+          path: /tmp/fb_grafana.tar
+          retention-days: 1
+
   go-module-tests:
     needs: [build-docker-foreverbull, build-docker-zipline]
     runs-on: ubuntu-latest
@@ -233,7 +257,12 @@ jobs:
 
   end-to-end-tests:
     needs:
-      [build-docker-foreverbull, build-docker-zipline, build-python-packages]
+      [
+        build-docker-foreverbull,
+        build-docker-zipline,
+        build-python-packages,
+        build-docker-grafana,
+      ]
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -257,10 +286,17 @@ jobs:
         with:
           name: zipline
           path: /tmp
+      - name: Download Grafana
+        uses: actions/download-artifact@v4
+        with:
+          name: fb-grafana
+          path: /tmp
       - name: Load zipline
         run: docker load -i /tmp/zipline.tar
       - name: load foreverbull
         run: docker load -i /tmp/foreverbull.tar
+      - name: load fb-grafana
+        run: docker load -i /tmp/fb_grafana.tar
       - name: Download foreverbull pypi package
         uses: actions/download-artifact@v4
         with:
@@ -301,7 +337,7 @@ jobs:
           pip install ta-lib
       - name: Create environment
         run: |
-          fbull env start --broker-image lhjnilsson/foreverbull:on_commit --backtest-image lhjnilsson/zipline:on_commit
+          fbull env start --version on_commit
       - name: create backtest
         run: fbull backtest create /tmp/example_algorithms/nasdaq.json
       - name: ingest
@@ -337,7 +373,12 @@ jobs:
 
   run-manual-backtest:
     needs:
-      [build-docker-foreverbull, build-docker-zipline, build-python-packages]
+      [
+        build-docker-foreverbull,
+        build-docker-zipline,
+        build-python-packages,
+        build-docker-grafana,
+      ]
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -361,10 +402,17 @@ jobs:
         with:
           name: zipline
           path: /tmp
+      - name: Download Grafana
+        uses: actions/download-artifact@v4
+        with:
+          name: fb-grafana
+          path: /tmp
       - name: Load zipline
         run: docker load -i /tmp/zipline.tar
       - name: load foreverbull
         run: docker load -i /tmp/foreverbull.tar
+      - name: load grafana
+        run: docker load -i /tmp/fb_grafana.tar
       - name: Download foreverbull pypi package
         uses: actions/download-artifact@v4
         with:
@@ -396,7 +444,7 @@ jobs:
           pip install /tmp/foreverbull_testing-0.0.1-py3-none-any.whl
       - name: Create environment
         run: |
-          fbull env start --broker-image lhjnilsson/foreverbull:on_commit --backtest-image lhjnilsson/zipline:on_commit
+          fbull env start --version on_commit
       - name: create backtest
         run: fbull backtest create /tmp/example_algorithms/nasdaq.json
       - name: ingest

diff --git a/client/foreverbull_cli/src/foreverbull_cli/env.py b/client/foreverbull_cli/src/foreverbull_cli/env.py
@@ -17,8 +17,6 @@
 from foreverbull_cli.output import console
 
 
-version = "0.1.0"
-
 env = typer.Typer()
 
 
@@ -88,8 +86,9 @@ def setup_path(
 POSTGRES_IMAGE = "postgres:13.3-alpine"
 NATS_IMAGE = "nats:2.10-alpine"
 MINIO_IMAGE = "minio/minio:latest"
-BROKER_IMAGE = f"lhjnilsson/foreverbull:{version}"
-BACKTEST_IMAGE = f"lhjnilsson/zipline:{version}"
+BROKER_IMAGE = "lhjnilsson/foreverbull:{}"
+BACKTEST_IMAGE = "lhjnilsson/zipline:{}"
+GRAFANA_IMAGE = "lhjnilsson/fb-grafana:{}"
 
 
 @env.command()
@@ -116,6 +115,10 @@ def status():
         foreverbull_container = d.containers.get("foreverbull_foreverbull")
     except docker.errors.NotFound:
         foreverbull_container = None
+    try:
+        grafana_container = d.containers.get("foreverbull_grafana")
+    except docker.errors.NotFound:
+        grafana_container = None
 
     try:
         postgres_image = d.images.get(POSTGRES_IMAGE)
@@ -129,6 +132,11 @@ def status():
         minio_image = d.images.get(MINIO_IMAGE)
     except docker.errors.ImageNotFound:
         minio_image = None
+    try:
+        grafana_image = d.images.get(GRAFANA_IMAGE)
+    except docker.errors.ImageNotFound:
+        grafana_image = None
+
     try:
         foreverbull_image = d.images.get(BROKER_IMAGE)  # type: ignore
     except docker.errors.ImageNotFound:
@@ -154,6 +162,11 @@ def status():
         "Minio",
         minio_image.short_id if minio_image else "Not found",
     )
+    table.add_row(
+        grafana_container.status if grafana_container else "Not Found",
+        "Grafana",
+        grafana_image.short_id if grafana_image else "Not found",
+    )
     table.add_row(
         foreverbull_container.status if foreverbull_container else "Not Found",
         "Foreverbull",
@@ -164,8 +177,7 @@ def status():
 
 @env.command()
 def start(
-    broker_image: Annotated[str, typer.Option(help="Docker image name of broker")] = BROKER_IMAGE,
-    backtest_image: Annotated[str, typer.Option(help="Docker image name of backtest service")] = BACKTEST_IMAGE,
+    version: Annotated[str, typer.Option(help="Version of images to use")] = "latest",
     log_level: Annotated[str, typer.Option(help="Log level")] = "INFO",
 ):
     try:
@@ -192,6 +204,7 @@ def get_or_pull_image(image_name):
         postgres_task_id = progress.add_task("Postgres", total=2)
         nats_task_id = progress.add_task("NATS", total=2)
         minio_task_id = progress.add_task("Minio", total=2)
+        grafana_task_id = progress.add_task("Grafana", total=2)
         health_task_id = progress.add_task("Waiting for services to start", total=2)
         foreverbull_task_id = progress.add_task("Foreverbull", total=2)
 
@@ -202,8 +215,9 @@ def get_or_pull_image(image_name):
                 POSTGRES_IMAGE,
                 NATS_IMAGE,
                 MINIO_IMAGE,
-                broker_image,
-                backtest_image,
+                GRAFANA_IMAGE.format(version),
+                BROKER_IMAGE.format(version),
+                BACKTEST_IMAGE.format(version),
             ]:
                 futures.append(executor.submit(get_or_pull_image, image))
                 wait(futures)
@@ -369,7 +383,7 @@ def get_or_pull_image(image_name):
         except docker.errors.NotFound:
             try:
                 d.containers.run(
-                    broker_image,
+                    BROKER_IMAGE.format(version),
                     name="foreverbull_foreverbull",
                     detach=True,
                     network=NETWORK_NAME,
@@ -398,7 +412,7 @@ def get_or_pull_image(image_name):
                         "NATS_URL": "nats://nats:4222",
                         "MINIO_URL": "minio:9000",
                         "DOCKER_NETWORK": NETWORK_NAME,
-                        "BACKTEST_IMAGE": backtest_image,
+                        "BACKTEST_IMAGE": BACKTEST_IMAGE.format(version),
                         "LOG_LEVEL": log_level,
                     },  # type: ignore
                     volumes={
@@ -415,9 +429,31 @@ def get_or_pull_image(image_name):
                     completed=100,
                 )
                 exit(1)
-            time.sleep(2)
         progress.update(foreverbull_task_id, completed=2)
 
+        progress.update(grafana_task_id, completed=1)
+        try:
+            d.containers.get("foreverbull_grafana")
+        except docker.errors.NotFound:
+            try:
+                d.containers.run(
+                    GRAFANA_IMAGE.format(version),
+                    name="foreverbull_grafana",
+                    detach=True,
+                    network=NETWORK_NAME,
+                    hostname="grafana",
+                    ports={"3000/tcp": 3000},
+                    environment={"BROKER_URL": "foreverbull:50055"},
+                )
+            except Exception as e:
+                progress.update(
+                    grafana_task_id,
+                    description=f"[red]Failed to start grafana: {e}",
+                    completed=2,
+                )
+                exit(1)
+        progress.update(grafana_task_id, completed=2)
+
 
 @env.command()
 def stop():

diff --git a/grafana/Dockerfile b/grafana/Dockerfile
@@ -8,40 +8,37 @@ RUN apk add --no-cache git make
 WORKDIR /app
 
 # Copy Go module files
-COPY . .
+COPY lhjnilsson-foreverbull-datasource/ .
 RUN go mod download
 
 # Build the binary
-RUN CGO_ENABLED=0 GOOS=linux go build -o foreverbull_linux_arm64 .
+RUN go install github.com/magefile/mage@latest
+RUN mage
 
 # Build frontend
 FROM node:18 AS js-builder
 WORKDIR /app
-COPY package.json .
-COPY tsconfig.json .
-COPY src/ src/
-COPY README.md .
+COPY lhjnilsson-foreverbull-datasource/ .
 RUN npm install
 RUN npm run build
 
 # Second stage: Setup Grafana with the plugin
-FROM grafana/grafana-oss:latest
+FROM grafana/grafana-oss:11.3.2
 
 # Create plugin directory
-RUN mkdir -p /var/lib/grafana/plugins/foreverbull
+RUN mkdir -p /var/lib/grafana/plugins/lhjnilsson-foreverbull-datasource/
 
 # Copy binary from builder
-COPY --from=builder /app/foreverbull_linux_arm64 /var/lib/grafana/plugins/foreverbull/
-COPY --from=js-builder /app/dist /var/lib/grafana/plugins/foreverbull/
-COPY src/plugin.json /var/lib/grafana/plugins/foreverbull/
-
+COPY --from=builder /app/dist/* /var/lib/grafana/plugins/lhjnilsson-foreverbull-datasource/
+COPY --from=js-builder /app/dist/* /var/lib/grafana/plugins/lhjnilsson-foreverbull-datasource/
+COPY lhjnilsson-foreverbull-datasource/provisioning/ /etc/grafana/provisioning
 
 # Set permissions for Grafana user (uid 472)
 #RUN chown -R 472:472 /var/lib/grafana/plugins/foreverbull \
 #    && chmod +x /var/lib/grafana/plugins/foreverbull/foreverbull
 
 # Enable unsigned plugins
-ENV GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=foreverbull
+ENV GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=lhjnilsson-foreverbull-datasource
 ENV GF_AUTH_ANONYMOUS_ENABLED=true
 ENV GF_AUTH_DISABLE_LOGIN_FORM=true
 ENV GF_AUTH_ANONYMOUS_ORG_ROLE=Admin

diff --git a/grafana/lhjnilsson-foreverbull-datasource/pkg/datasource.go b/grafana/lhjnilsson-foreverbull-datasource/pkg/datasource.go
@@ -110,7 +110,7 @@ func (d *Datasource) HandleGetExecutionMetric(ctx context.Context, req *grafana.
 }
 
 type queryModel struct {
-	ExecutionId string `json:"execution_id"`
+	ExecutionId string `json:"executionId"`
 }
 
 func (d *Datasource) handleGetExecutionMetric(ctx context.Context, req grafana.QueryDataRequest, q grafana.DataQuery) grafana.DataResponse {
@@ -126,16 +126,33 @@ func (d *Datasource) handleGetExecutionMetric(ctx context.Context, req grafana.Q
 		return grafana.ErrDataResponse(grafana.StatusBadRequest, fmt.Sprintf("json unmarshal: %v", err.Error()))
 	}
 
-	execution, err := d.backend.GetExecution(ctx, &pb.GetExecutionRequest{ExecutionId: "0a7378c6-950b-4e8e-af40-4378c9927e9f"})
+	execution, err := d.backend.GetExecution(ctx, &pb.GetExecutionRequest{ExecutionId: qm.ExecutionId})
 	if err != nil {
 		return grafana.ErrDataResponse(grafana.StatusBadRequest, fmt.Sprintf("json unmarshal: %v", err.Error()))
 	}
 	times := make([]time.Time, len(execution.Periods))
-	values := make([]float64, len(execution.Periods))
+
+	pnl := make([]float64, len(execution.Periods))
+	returns := make([]float64, len(execution.Periods))
+	portfolio_value := make([]float64, len(execution.Periods))
+	longs_count := make([]int32, len(execution.Periods))
+	shorts_count := make([]int32, len(execution.Periods))
+	long_value := make([]float64, len(execution.Periods))
+	short_value := make([]float64, len(execution.Periods))
+	sharpe := make([]*float64, len(execution.Periods))
+	sortio := make([]*float64, len(execution.Periods))
 
 	for i, period := range execution.Periods {
 		times[i] = time.Date(int(period.Date.Year), time.Month(int(period.Date.Month)), int(period.Date.Day), 0, 0, 0, 0, time.UTC)
-		values[i] = period.PortfolioValue
+		pnl[i] = period.PNL
+		returns[i] = period.Returns
+		portfolio_value[i] = period.PortfolioValue
+		longs_count[i] = period.LongsCount
+		shorts_count[i] = period.ShortsCount
+		long_value[i] = period.LongValue
+		short_value[i] = period.ShortValue
+		sharpe[i] = period.Sharpe
+		sortio[i] = period.Sortino
 	}
 
 	// create data frame response.
@@ -146,7 +163,15 @@ func (d *Datasource) handleGetExecutionMetric(ctx context.Context, req grafana.Q
 	// add fields.
 	frame.Fields = append(frame.Fields,
 		data.NewField("time", nil, times),
-		data.NewField("values", nil, values),
+		data.NewField("profit & loss", nil, pnl),
+		data.NewField("returns", nil, portfolio_value),
+		data.NewField("portfolio value", nil, portfolio_value),
+		data.NewField("longs count", nil, longs_count),
+		data.NewField("shorts count", nil, shorts_count),
+		data.NewField("long value", nil, long_value),
+		data.NewField("short value", nil, short_value),
+		data.NewField("sharpe", nil, sharpe),
+		data.NewField("sortio", nil, sortio),
 	)
 
 	// add the frames to the response.

diff --git a/grafana/lhjnilsson-foreverbull-datasource/provisioning/dashboards/default.yaml b/grafana/lhjnilsson-foreverbull-datasource/provisioning/dashboards/default.yaml
@@ -0,0 +1,8 @@
+apiVersion: 1
+
+providers:
+  - name: Default # A uniquely identifiable name for the provider
+    folder: Foreverbull # The folder where to place the dashboards
+    type: file
+    options:
+      path: /etc/grafana/provisioning/dashboards/
File	Stmts	Miss	Cover	Missing
foreverbull/src/foreverbull
algorithm.py	87	21	76%	34–38, 54, 71–91, 96–99
models.py	314	71	77%	38, 50–68, 76, 100, 103, 117, 149, 166, 179–186, 191–192, 195, 198–202, 205–208, 211, 222, 323, 355–362, 373, 377, 384–391, 400, 404, 421–432
worker.py	247	43	83%	31, 35, 85–87, 102, 110–123, 126–133, 170–172, 229–244, 259, 267, 279, 308, 310, 328, 330
foreverbull/src/foreverbull/broker
__init__.py	3	3	0%	1–5
backtest.py	65	65	0%	1–118
finance.py	21	21	0%	1–42
storage.py	23	23	0%	1–36
foreverbull/src/foreverbull/pb
pb_utils.py	36	15	58%	15–21, 37, 40–47
foreverbull/src/foreverbull/pb/buf/validate
validate_pb2.py	439	424	3%	36–459
foreverbull/src/foreverbull/pb/foreverbull
common_pb2.py	20	8	60%	33–40
foreverbull/src/foreverbull/pb/foreverbull/backtest
backtest_pb2.py	22	8	64%	35–42
backtest_service_pb2.py	58	42	28%	37–78
backtest_service_pb2_grpc.py	78	31	60%	15–16, 19, 79–81, 85–87, 91–93, 97–99, 103–105, 109–111, 115–117, 179, 206, 233, 260, 287, 314, 341
execution_pb2.py	25	10	60%	36–45
session_pb2.py	21	8	62%	34–41
session_service_pb2.py	45	28	38%	38–65
session_service_pb2_grpc.py	54	19	65%	15–16, 19, 64–66, 70–72, 76–78, 82–84, 131, 158, 185, 212
foreverbull/src/foreverbull/pb/foreverbull/finance
finance_pb2.py	25	12	52%	34–45
foreverbull/src/foreverbull/pb/foreverbull/service
worker_pb2.py	26	14	46%	33–46
worker_service_pb2.py	44	28	36%	37–64
foreverbull/src/foreverbull/pb/foreverbull/strategy
strategy_service_pb2.py	33	18	45%	36–53
strategy_service_pb2_grpc.py	30	12	60%	15–16, 19, 37, 49–51, 55–65, 83
TOTAL	1726	924	46%
File	Stmts	Miss	Cover	Missing
foreverbull_zipline/src/foreverbull_zipline
__main__.py	21	21	0%	1–27
engine.py	264	38	86%	67, 87, 107, 124, 149, 176, 188, 198, 203, 207, 209–211, 213, 218, 222, 224, 228–232, 261–263, 268–270, 383–385, 405–406, 424–425, 436–437, 440–442
service.py	103	23	78%	67–72, 84–89, 98, 108, 111, 122–129
session_service.py	13	1	92%	19
foreverbull_zipline/src/foreverbull_zipline/data_bundles
foreverbull.py	54	2	96%	25, 71
TOTAL	457	85	81%
File	Stmts	Miss	Cover	Missing
foreverbull_testing/src/foreverbull_testing
data.py	81	54	33%	1–26, 40, 46, 49–50, 53–54, 58–59, 80, 86, 89–90, 93–98, 102–133
database.py	55	35	36%	1–43, 52, 55, 60–63
plugin.py	51	30	41%	1–22, 32–35, 46, 65–76
TOTAL	187	119	36%
File	Stmts	Miss	Cover	Missing
foreverbull_cli/src/foreverbull_cli
__init__.py	5	2	60%	6–7
backtest.py	127	52	59%	118–120, 130–171, 179–195, 202–225
env.py	270	49	82%	28, 98–100, 185–187, 195–198, 226–230, 243–246, 282–288, 294–297, 321–327, 350–356, 364, 367, 371–376, 381–382, 425–431, 448–454, 462–464, 475, 483, 491, 499
main.py	28	28	0%	1–49
strategy.py	27	27	0%	1–47
TOTAL	475	158	67%