refactor: do not send audit logs to tinybird (unkeyed#2206)

* refactor: do not send audit logs to tinybird * refactor: use database for audit logs in trigger app * chore: remove accidentally created file * fix: cache and return properly
AkshayBandi027 · Oct 5, 2024 · 2288ebf · 2288ebf
1 parent ea60984
commit 2288ebf
Show file tree

Hide file tree

Showing 57 changed files with 779 additions and 1,987 deletions.
diff --git a/apps/dashboard/app/(app)/settings/billing/stripe/success/page.tsx b/apps/dashboard/app/(app)/settings/billing/stripe/success/page.tsx
@@ -5,7 +5,6 @@ import { getTenantId } from "@/lib/auth";
 import { db, eq, schema } from "@/lib/db";
 import { stripeEnv } from "@/lib/env";
 import { PostHogClient } from "@/lib/posthog";
-import { ingestAuditLogsTinybird } from "@/lib/tinybird";
 import { currentUser } from "@clerk/nextjs";
 import { defaultProSubscriptions } from "@unkey/billing";
 import { headers } from "next/headers";
@@ -114,22 +113,6 @@ export default async function StripeSuccess(props: Props) {
           userAgent: h.get("user-agent") ?? undefined,
         },
       });
-      await ingestAuditLogsTinybird({
-        workspaceId: ws.id,
-        actor: { type: "user", id: user.id },
-        event: "workspace.update",
-        description: "Changed plan to 'pro'",
-        resources: [
-          {
-            type: "workspace",
-            id: ws.id,
-          },
-        ],
-        context: {
-          location: h.get("x-forwarded-for") ?? process.env.VERCEL_REGION ?? "unknown",
-          userAgent: h.get("user-agent") ?? undefined,
-        },
-      });
     }
   });
 

diff --git a/apps/dashboard/app/new/page.tsx b/apps/dashboard/app/new/page.tsx
@@ -3,7 +3,6 @@ import { Button } from "@/components/ui/button";
 import { Separator } from "@/components/ui/separator";
 import { insertAuditLogs } from "@/lib/audit";
 import { db, schema } from "@/lib/db";
-import { ingestAuditLogsTinybird } from "@/lib/tinybird";
 import { auth } from "@clerk/nextjs";
 import { newId } from "@unkey/id";
 import { ArrowRight, DatabaseZap, GlobeLock, KeySquare } from "lucide-react";
@@ -251,26 +250,6 @@ export default async function (props: Props) {
           },
         });
       });
-      await ingestAuditLogsTinybird({
-        workspaceId: workspaceId,
-        event: "workspace.create",
-        actor: {
-          type: "user",
-          id: userId,
-        },
-        description: `Created ${workspaceId}`,
-        resources: [
-          {
-            type: "workspace",
-            id: workspaceId,
-          },
-        ],
-
-        context: {
-          userAgent: headers().get("user-agent") ?? undefined,
-          location: headers().get("x-forwarded-for") ?? process.env.VERCEL_REGION ?? "unknown",
-        },
-      });
 
       return redirect(`/new?workspaceId=${workspaceId}`);
     }

diff --git a/apps/dashboard/lib/tinybird.ts b/apps/dashboard/lib/tinybird.ts
@@ -1,10 +1,7 @@
-import { time } from "node:console";
 import { env } from "@/lib/env";
 import { NoopTinybird, Tinybird } from "@chronark/zod-bird";
-import { newId } from "@unkey/id";
-import { auditLogSchemaV1, unkeyAuditLogEvents } from "@unkey/schema/src/auditlog";
+import type { unkeyAuditLogEvents } from "@unkey/schema/src/auditlog";
 import { z } from "zod";
-import type { MaybeArray } from "./types";
 
 const token = env().TINYBIRD_TOKEN;
 const tb = token ? new Tinybird({ token }) : new NoopTinybird();
@@ -504,32 +501,6 @@ export type UnkeyAuditLog = {
   };
 };
 
-export function ingestAuditLogsTinybird(logs: MaybeArray<UnkeyAuditLog>) {
-  if (Array.isArray(logs) && logs.length === 0) {
-    return Promise.resolve();
-  }
-  return tb.buildIngestEndpoint({
-    datasource: "audit_logs__v2",
-    event: auditLogSchemaV1
-      .merge(
-        z.object({
-          event: unkeyAuditLogEvents,
-          auditLogId: z.string().default(newId("auditLog")),
-          bucket: z.string().default("unkey_mutations"),
-          time: z.number().default(Date.now()),
-        }),
-      )
-      .transform((l) => ({
-        ...l,
-        actor: {
-          ...l.actor,
-          meta: l.actor.meta ? JSON.stringify(l.actor.meta) : undefined,
-        },
-        resources: JSON.stringify(l.resources),
-      })),
-  })(logs);
-}
-
 export const getRatelimitsHourly = tb.buildPipe({
   pipe: "get_ratelimits_hourly__v1",
   parameters: z.object({

diff --git a/apps/dashboard/lib/trpc/routers/api/create.ts b/apps/dashboard/lib/trpc/routers/api/create.ts
@@ -3,7 +3,6 @@ import { z } from "zod";
 
 import { insertAuditLogs } from "@/lib/audit";
 import { db, schema } from "@/lib/db";
-import { ingestAuditLogsTinybird } from "@/lib/tinybird";
 import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure";
 import { newId } from "@unkey/id";
 
@@ -90,25 +89,6 @@ export const createApi = rateLimitedProcedure(ratelimit.create)
           userAgent: ctx.audit.userAgent,
         },
       });
-      await ingestAuditLogsTinybird({
-        workspaceId: ws.id,
-        actor: {
-          type: "user",
-          id: ctx.user.id,
-        },
-        event: "api.create",
-        description: `Created ${apiId}`,
-        resources: [
-          {
-            type: "api",
-            id: apiId,
-          },
-        ],
-        context: {
-          location: ctx.audit.location,
-          userAgent: ctx.audit.userAgent,
-        },
-      });
     });
 
     return {

diff --git a/apps/dashboard/lib/trpc/routers/api/delete.ts b/apps/dashboard/lib/trpc/routers/api/delete.ts
@@ -3,7 +3,6 @@ import { z } from "zod";
 
 import { insertAuditLogs } from "@/lib/audit";
 import { db, eq, schema } from "@/lib/db";
-import { ingestAuditLogsTinybird } from "@/lib/tinybird";
 import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure";
 
 export const deleteApi = rateLimitedProcedure(ratelimit.delete)
@@ -66,25 +65,6 @@ export const deleteApi = rateLimitedProcedure(ratelimit.delete)
             userAgent: ctx.audit.userAgent,
           },
         });
-        await ingestAuditLogsTinybird({
-          workspaceId: api.workspaceId,
-          actor: {
-            type: "user",
-            id: ctx.user.id,
-          },
-          event: "api.delete",
-          description: `Deleted ${api.id}`,
-          resources: [
-            {
-              type: "api",
-              id: api.id,
-            },
-          ],
-          context: {
-            location: ctx.audit.location,
-            userAgent: ctx.audit.userAgent,
-          },
-        });
 
         const keyIds = await tx.query.keys.findMany({
           where: eq(schema.keys.keyAuthId, api.keyAuthId!),
@@ -122,34 +102,6 @@ export const deleteApi = rateLimitedProcedure(ratelimit.delete)
               },
             })),
           );
-          await ingestAuditLogsTinybird(
-            keyIds.map(({ id }) => ({
-              workspaceId: api.workspace.id,
-              actor: {
-                type: "user",
-                id: ctx.user.id,
-              },
-              event: "key.delete",
-              description: `Deleted ${id} as part of the ${api.id} deletion`,
-              resources: [
-                {
-                  type: "api",
-                  id: api.id,
-                },
-                {
-                  type: "key",
-                  id: id,
-                },
-              ],
-              context: {
-                location: ctx.audit.location,
-                userAgent: ctx.audit.userAgent,
-              },
-            })),
-          ).catch((err) => {
-            tx.rollback();
-            throw err;
-          });
         }
       });
     } catch (_err) {

diff --git a/apps/dashboard/lib/trpc/routers/api/updateDeleteProtection.ts b/apps/dashboard/lib/trpc/routers/api/updateDeleteProtection.ts
@@ -3,7 +3,6 @@ import { z } from "zod";
 
 import { insertAuditLogs } from "@/lib/audit";
 import { db, eq, schema } from "@/lib/db";
-import { ingestAuditLogsTinybird } from "@/lib/tinybird";
 import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure";
 
 export const updateAPIDeleteProtection = rateLimitedProcedure(ratelimit.update)
@@ -76,28 +75,4 @@ export const updateAPIDeleteProtection = rateLimitedProcedure(ratelimit.update)
         },
       });
     });
-    await ingestAuditLogsTinybird({
-      workspaceId: api.workspace.id,
-      actor: {
-        type: "user",
-        id: ctx.user.id,
-      },
-      event: "api.update",
-      description: `API ${api.name} delete protection is now ${
-        input.enabled ? "enabled" : "disabled"
-      }.}`,
-      resources: [
-        {
-          type: "api",
-          id: api.id,
-          meta: {
-            deleteProtection: input.enabled,
-          },
-        },
-      ],
-      context: {
-        location: ctx.audit.location,
-        userAgent: ctx.audit.userAgent,
-      },
-    });
   });
diff --git a/apps/dashboard/lib/trpc/routers/api/updateIpWhitelist.ts b/apps/dashboard/lib/trpc/routers/api/updateIpWhitelist.ts
@@ -3,7 +3,6 @@ import { z } from "zod";
 
 import { insertAuditLogs } from "@/lib/audit";
 import { db, eq, schema } from "@/lib/db";
-import { ingestAuditLogsTinybird } from "@/lib/tinybird";
 import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure";
 
 export const updateApiIpWhitelist = rateLimitedProcedure(ratelimit.update)
@@ -88,23 +87,4 @@ export const updateApiIpWhitelist = rateLimitedProcedure(ratelimit.update)
         },
       });
     });
-    await ingestAuditLogsTinybird({
-      workspaceId: api.workspace.id,
-      actor: {
-        type: "user",
-        id: ctx.user.id,
-      },
-      event: "api.update",
-      description: `Changed ${api.id} IP whitelist from ${api.ipWhitelist} to ${newIpWhitelist}`,
-      resources: [
-        {
-          type: "api",
-          id: api.id,
-        },
-      ],
-      context: {
-        location: ctx.audit.location,
-        userAgent: ctx.audit.userAgent,
-      },
-    });
   });
diff --git a/apps/dashboard/lib/trpc/routers/api/updateName.ts b/apps/dashboard/lib/trpc/routers/api/updateName.ts
@@ -3,7 +3,6 @@ import { z } from "zod";
 
 import { insertAuditLogs } from "@/lib/audit";
 import { db, eq, schema } from "@/lib/db";
-import { ingestAuditLogsTinybird } from "@/lib/tinybird";
 import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure";
 
 export const updateApiName = rateLimitedProcedure(ratelimit.update)
@@ -71,23 +70,4 @@ export const updateApiName = rateLimitedProcedure(ratelimit.update)
         },
       });
     });
-    await ingestAuditLogsTinybird({
-      workspaceId: api.workspace.id,
-      actor: {
-        type: "user",
-        id: ctx.user.id,
-      },
-      event: "api.update",
-      description: `Changed ${api.id} name from ${api.name} to ${input.name}`,
-      resources: [
-        {
-          type: "api",
-          id: api.id,
-        },
-      ],
-      context: {
-        location: ctx.audit.location,
-        userAgent: ctx.audit.userAgent,
-      },
-    });
   });
diff --git a/apps/dashboard/lib/trpc/routers/key/create.ts b/apps/dashboard/lib/trpc/routers/key/create.ts
@@ -1,6 +1,5 @@
 import { insertAuditLogs } from "@/lib/audit";
 import { db, schema } from "@/lib/db";
-import { ingestAuditLogsTinybird } from "@/lib/tinybird";
 import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure";
 import { TRPCError } from "@trpc/server";
 import { newId } from "@unkey/id";
@@ -134,22 +133,5 @@ export const createKey = rateLimitedProcedure(ratelimit.create)
         });
       });
 
-    await ingestAuditLogsTinybird({
-      workspaceId: workspace.id,
-      actor: { type: "user", id: ctx.user.id },
-      event: "key.create",
-      description: `Created ${keyId}`,
-      resources: [
-        {
-          type: "key",
-          id: keyId,
-        },
-      ],
-      context: {
-        location: ctx.audit.location,
-        userAgent: ctx.audit.userAgent,
-      },
-    });
-
     return { keyId, key };
   });
diff --git a/apps/dashboard/lib/trpc/routers/key/createRootKey.ts b/apps/dashboard/lib/trpc/routers/key/createRootKey.ts
@@ -1,6 +1,6 @@
 import { db, eq, schema } from "@/lib/db";
 import { env } from "@/lib/env";
-import { type UnkeyAuditLog, ingestAuditLogsTinybird } from "@/lib/tinybird";
+import type { UnkeyAuditLog } from "@/lib/tinybird";
 import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure";
 import { TRPCError } from "@trpc/server";
 import { newId } from "@unkey/id";
@@ -197,7 +197,5 @@ export const createRootKey = rateLimitedProcedure(ratelimit.create)
       });
     }
 
-    await ingestAuditLogsTinybird(auditLogs);
-
     return { key, keyId };
   });
diff --git a/apps/dashboard/lib/trpc/routers/key/delete.ts b/apps/dashboard/lib/trpc/routers/key/delete.ts
@@ -1,6 +1,5 @@
 import { insertAuditLogs } from "@/lib/audit";
 import { and, db, eq, inArray, schema } from "@/lib/db";
-import { ingestAuditLogsTinybird } from "@/lib/tinybird";
 import { rateLimitedProcedure, ratelimit } from "@/lib/trpc/ratelimitProcedure";
 import { TRPCError } from "@trpc/server";
 import { z } from "zod";
@@ -82,23 +81,4 @@ export const deleteKeys = rateLimitedProcedure(ratelimit.delete)
             "We are unable to delete the key. Please contact support using [email protected]",
         });
       });
-
-    await ingestAuditLogsTinybird(
-      workspace.keys.map((key) => ({
-        workspaceId: workspace.id,
-        actor: { type: "user", id: ctx.user.id },
-        event: "key.delete",
-        description: `Deleted ${key.id}`,
-        resources: [
-          {
-            type: "key",
-            id: key.id,
-          },
-        ],
-        context: {
-          location: ctx.audit.location,
-          userAgent: ctx.audit.userAgent,
-        },
-      })),
-    );
   });