Telemetry follow-up (#299)

* update telemetry message to include docs link

* chore: changeset

* add telemetry docs, fix event construction

* add payload to app lifecycle events

* fix event construction

* tweaks

.changeset/stale-lies-smoke.md

@@ -0,0 +1,5 @@
+---
+"@ponder/core": patch
+---
+
+Added anonymized telemetry. See `https://ponder.sh/advanced/telemetry` for details.

docs/pages/advanced/telemetry.mdx

@@ -16,3 +16,9 @@ To opt out of telemetry, set the `PONDER_TELEMETRY_DISABLED` environment variabl
 ```js filename=".env.local"
 PONDER_TELEMETRY_DISABLED = true
 ```
+
+## Implementation
+
+Ponder's telemetry implementation is 100% open-source. The [telemetry service](https://github.com/0xOlias/ponder/blob/main/packages/core/src/telemetry/service.ts) (part of `@ponder/core`) runs on the user's device and submits event data via HTTP POST requests to the [telemetry collection endpoint](https://github.com/0xOlias/ponder/blob/main/docs/pages/api/telemetry/index.ts) hosted at `https://ponder.sh/api/telemetry`.
+
+The implementation generates a stable anonymous unique identifier for the user's device and stores it at the [system default user config directory](https://github.com/sindresorhus/env-paths#pathsconfig). This config also stores the user's opt-out preference and a stable salt used to hash potentially sensitive data such as file paths and the git remote URL.

docs/pages/api/telemetry/index.ts

@@ -7,6 +7,11 @@ if (!process.env.SEGMENT_WRITE_KEY) {
 
 const analytics = new Analytics({
   writeKey: process.env.SEGMENT_WRITE_KEY,
+  /**
+   * Disable batching so that event are submitted immediately.
+   * See https://segment.com/docs/connections/sources/catalog/libraries/server/node/#batching
+   */
+  maxEventsInBatch: 1,
 });
 
 export default async function forwardTelemetry(
@@ -23,5 +28,5 @@ export default async function forwardTelemetry(
     message: "Telemetry data processed successfully.",
   });
 
-  await analytics.track(req.body);
+  analytics.track(req.body);
 }

packages/core/src/Ponder.ts

@@ -172,10 +172,6 @@ export class Ponder {
       )}`,
     });
 
-    this.common.telemetry.record({
-      event: "App Started",
-    });
-
     this.registerServiceDependencies();
 
     // If any of the provided networks do not have a valid RPC url,
@@ -216,6 +212,17 @@ export class Ponder {
 
   async dev() {
     const setupError = await this.setup();
+
+    this.common.telemetry.record({
+      event: "App Started",
+      properties: {
+        command: "ponder dev",
+        hasSetupError: !!setupError,
+        logFilterCount: this.logFilters.length,
+        databaseKind: this.eventStore.kind,
+      },
+    });
+
     if (setupError) {
       this.common.logger.error({
         service: "app",
@@ -242,6 +249,17 @@ export class Ponder {
 
   async start() {
     const setupError = await this.setup();
+
+    this.common.telemetry.record({
+      event: "App Started",
+      properties: {
+        command: "ponder start",
+        hasSetupError: !!setupError,
+        logFilterCount: this.logFilters.length,
+        databaseKind: this.eventStore.kind,
+      },
+    });
+
     if (setupError) {
       this.common.logger.error({
         service: "app",
@@ -282,7 +300,7 @@ export class Ponder {
 
     this.common.telemetry.record({
       event: "App Killed",
-      payload: {
+      properties: {
         processDuration: process.uptime(),
       },
     });

packages/core/src/telemetry/service.test.ts

@@ -1,5 +1,6 @@
 import child_process from "node:child_process";
 import fs from "node:fs";
+import { tmpdir } from "node:os";
 import path from "path";
 import process from "process";
 import { afterAll, beforeEach, expect, test, vi } from "vitest";
@@ -39,18 +40,17 @@ test("should be disabled if PONDER_TELEMETRY_DISABLED flag is set", async () =>
 test("events are processed", async ({ common: { options } }) => {
   const telemetry = new TelemetryService({ options });
 
-  telemetry.record({ event: "test" });
+  telemetry.record({ event: "test", properties: { test: "data" } });
   await telemetry.flush();
 
   expect(fetchSpy).toHaveBeenCalled();
 
   const fetchBody = JSON.parse(fetchSpy.mock.calls[0][1]["body"]);
   expect(fetchBody).toMatchObject({
+    anonymousId: expect.any(String),
+    context: expect.anything(),
     event: "test",
-    meta: expect.anything(),
-    sessionId: expect.anything(),
-    anonymousId: expect.anything(),
-    projectId: expect.anything(),
+    properties: { test: "data" },
   });
 });
 
@@ -85,16 +85,17 @@ test("events are put back in queue if telemetry service is killed", async ({
 test("kill method should persist events and trigger detached flush", async ({
   common: { options },
 }) => {
+  const options_ = { ...options, ponderDir: tmpdir() };
+
   const spawn = vi.spyOn(child_process, "spawn");
-  const telemetry = new TelemetryService({ options });
-  const fileName = path.join(options.ponderDir, "telemetry-events.json");
+  const telemetry = new TelemetryService({ options: options_ });
+  const fileName = path.join(options_.ponderDir, "telemetry-events.json");
 
   const writeFileSyncSpy = vi
     .spyOn(fs, "writeFileSync")
     .mockImplementationOnce(() => vi.fn());
 
-  // we need to mock the fetch call to throw an AbortError so that the event is
-  // put back in the queue
+  // Mock the fetch call to throw an AbortError so that the event is put back in the queue.
   fetchSpy.mockImplementation(() => {
     throw { name: "AbortError" };
   });
@@ -103,12 +104,11 @@ test("kill method should persist events and trigger detached flush", async ({
     telemetry.record({ event: "test" });
   }
   // Note that we are not flushing here, because we want to test the detachedFlush flow.
-  await telemetry.flush();
 
   await telemetry.kill();
+
   const fileNameArgument = writeFileSyncSpy.mock.calls[0][0];
 
   expect(spawn).toHaveBeenCalled();
   expect(fileNameArgument).toBe(fileName);
-  expect(fetchSpy).toHaveBeenCalledTimes(10);
 });

packages/core/src/telemetry/service.ts

@@ -15,7 +15,7 @@ import { getGitRemoteUrl } from "@/telemetry/remote";
 
 type TelemetryEvent = {
   event: string;
-  payload?: object;
+  properties?: any;
 };
 
 type TelemetryDeviceConfig = {
@@ -26,30 +26,27 @@ type TelemetryDeviceConfig = {
 };
 
 type TelemetryEventContext = {
-  sessionId: string;
-  anonymousId: string;
   projectId: string;
-  meta: {
-    // Software information
-    systemPlatform: NodeJS.Platform;
-    systemRelease: string;
-    systemArchitecture: string;
-    // Machine information
-    cpuCount: number;
-    cpuModel: string | null;
-    cpuSpeed: number | null;
-    memoryInMb: number;
-    // package.json information
-    ponderVersion: string;
-    nodeVersion: string;
-    packageManager: string;
-    packageManagerVersion: string;
-  };
+  sessionId: string;
+  // package.json information
+  nodeVersion: string;
+  packageManager: string;
+  packageManagerVersion: string;
+  ponderVersion: string;
+  // Software information
+  systemPlatform: NodeJS.Platform;
+  systemRelease: string;
+  systemArchitecture: string;
+  // Machine information
+  cpuCount: number;
+  cpuModel: string | null;
+  cpuSpeed: number | null;
+  memoryInMb: number;
 };
 
 export class TelemetryService {
-  private conf: Conf<TelemetryDeviceConfig>;
   private options: Options;
+  private conf: Conf<TelemetryDeviceConfig>;
 
   private queue = new PQueue({ concurrency: 1 });
   private events: TelemetryEvent[] = [];
@@ -58,8 +55,8 @@ export class TelemetryService {
   private context?: TelemetryEventContext;
 
   constructor({ options }: { options: Options }) {
-    this.conf = new Conf({ projectName: "ponder" });
     this.options = options;
+    this.conf = new Conf({ projectName: "ponder" });
     this.notify();
   }
 
@@ -77,8 +74,19 @@ export class TelemetryService {
     const event = this.events.pop();
     if (!event) return;
 
-    const context = await this.getContext();
-    const serializedEvent = { ...event, ...context };
+    // Build the context. If it's already been built, this will return immediately.
+    try {
+      await this.getContext();
+    } catch (e) {
+      // Do nothing
+    }
+
+    // See https://segment.com/docs/connections/spec/track
+    const serializedEvent = {
+      ...event,
+      anonymousId: this.anonymousId,
+      context: this.context,
+    };
 
     try {
       await fetch(this.options.telemetryUrl, {
@@ -119,17 +127,21 @@ export class TelemetryService {
     console.log(
       `${pc.magenta(
         "Attention"
-      )}: Ponder now collects completely anonymous telemetry regarding usage.`
-    );
-    console.log(
-      "This information is used to shape Ponder's roadmap and prioritize features."
+      )}: Ponder now collects completely anonymous telemetry regarding usage. This data helps shape Ponder's roadmap and prioritize features. See https://ponder.sh/advanced/telemetry for more information.`
     );
   }
 
   private flushDetached() {
     if (this.events.length === 0) return;
 
-    const serializedEvents = JSON.stringify(this.events);
+    const eventsWithContext = this.events.map((event) => ({
+      ...event,
+      anonymousId: this.anonymousId,
+      // Note that it's possible for the context to be undefined here.
+      context: this.context,
+    }));
+    const serializedEvents = JSON.stringify(eventsWithContext);
+
     const telemetryEventsFilePath = path.join(
       this.options.ponderDir,
       "telemetry-events.json"
@@ -143,59 +155,13 @@ export class TelemetryService {
     ]);
   }
 
-  oneWayHash(value: string) {
-    const hash = createHash("sha256");
-    // Always prepend the payload value with salt. This ensures the hash is truly
-    // one-way.
-    hash.update(this.salt);
-    hash.update(value);
-    return hash.digest("hex");
-  }
-
   get disabled() {
     return (
       this.options.telemetryDisabled ||
       (this.conf.has("enabled") && !this.conf.get("enabled"))
     );
   }
 
-  private async getContext() {
-    if (this.context) return this.context;
-
-    const projectId = (await getGitRemoteUrl()) ?? process.cwd();
-    const cpus = os.cpus() || [];
-    const packageJson = JSON.parse(fs.readFileSync("package.json", "utf8"));
-    let packageManager: any = "unknown";
-    let packageManagerVersion: any = "unknown";
-    try {
-      packageManager = await detect();
-      packageManagerVersion = await getNpmVersion(packageManager);
-    } catch (e) {
-      // Ignore
-    }
-
-    this.context = {
-      anonymousId: this.anonymousId,
-      sessionId: randomBytes(32).toString("hex"),
-      projectId: this.oneWayHash(projectId),
-      meta: {
-        systemPlatform: os.platform(),
-        systemRelease: os.release(),
-        systemArchitecture: os.arch(),
-        cpuCount: cpus.length,
-        cpuModel: cpus.length ? cpus[0].model : null,
-        cpuSpeed: cpus.length ? cpus[0].speed : null,
-        memoryInMb: Math.trunc(os.totalmem() / Math.pow(1024, 2)),
-        ponderVersion: packageJson["version"],
-        nodeVersion: process.version,
-        packageManager,
-        packageManagerVersion,
-      },
-    };
-
-    return this.context;
-  }
-
   private get anonymousId() {
     const storedAnonymousId = this.conf.get("anonymousId");
     if (storedAnonymousId) return storedAnonymousId;
@@ -213,4 +179,64 @@ export class TelemetryService {
     this.conf.set("salt", createdSalt);
     return createdSalt;
   }
+
+  private oneWayHash(value: string) {
+    const hash = createHash("sha256");
+    // Always prepend the payload value with salt. This ensures the hash is truly
+    // one-way.
+    hash.update(this.salt);
+    hash.update(value);
+    return hash.digest("hex");
+  }
+
+  private async getContext() {
+    if (this.context) return this.context;
+
+    const sessionId = randomBytes(32).toString("hex");
+    const projectIdRaw = (await getGitRemoteUrl()) ?? process.cwd();
+    const projectId = this.oneWayHash(projectIdRaw);
+
+    let packageManager: any = "unknown";
+    let packageManagerVersion: any = "unknown";
+    try {
+      packageManager = await detect();
+      packageManagerVersion = await getNpmVersion(packageManager);
+    } catch (e) {
+      // Ignore
+    }
+
+    const packageJsonCwdPath = path.join(process.cwd(), "package.json");
+    const packageJsonRootPath = path.join(this.options.rootDir, "package.json");
+    const packageJsonPath = fs.existsSync(packageJsonCwdPath)
+      ? packageJsonCwdPath
+      : fs.existsSync(packageJsonRootPath)
+      ? packageJsonRootPath
+      : undefined;
+    const packageJson = packageJsonPath
+      ? JSON.parse(fs.readFileSync("package.json", "utf8"))
+      : undefined;
+    const ponderVersion = packageJson
+      ? packageJson["dependencies"]["@ponder/core"]
+      : "unknown";
+
+    const cpus = os.cpus() || [];
+
+    this.context = {
+      sessionId,
+      projectId,
+      nodeVersion: process.version,
+      packageManager,
+      packageManagerVersion,
+      ponderVersion,
+      systemPlatform: os.platform(),
+      systemRelease: os.release(),
+      systemArchitecture: os.arch(),
+      cpuCount: cpus.length,
+      cpuModel: cpus.length ? cpus[0].model : null,
+      cpuSpeed: cpus.length ? cpus[0].speed : null,
+      memoryInMb: Math.trunc(os.totalmem() / Math.pow(1024, 2)),
+    } satisfies TelemetryEventContext;
+
+    return this.context;
+  }
 }