feat: Replay ingestion with EFS (#20487)

.vscode/launch.json

@@ -72,7 +72,8 @@
                 "DATABASE_URL": "postgres://posthog:posthog@localhost:5432/posthog",
                 "SKIP_SERVICE_VERSION_REQUIREMENTS": "1",
                 "PRINT_SQL": "1",
-                "BILLING_SERVICE_URL": "https://billing.dev.posthog.dev"
+                "BILLING_SERVICE_URL": "https://billing.dev.posthog.dev",
+                "RECORDINGS_INGESTER_URL": "http://localhost:6738"
             },
             "console": "integratedTerminal",
             "python": "${workspaceFolder}/env/bin/python",

bin/start

@@ -7,6 +7,7 @@ trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
 export DEBUG=${DEBUG:-1}
 export SKIP_SERVICE_VERSION_REQUIREMENTS=1
 export BILLING_SERVICE_URL=${BILLING_SERVICE_URL:-https://billing.dev.posthog.dev}
+export RECORDINGS_INGESTER_URL=${RECORDINGS_INGESTER_URL:-http://localhost:6738}
 
 service_warning() {
   echo -e "\033[0;31m$1 isn't ready. You can run the stack with:\ndocker compose -f docker-compose.dev.yml up\nIf you have already ran that, just make sure that services are starting properly, and sit back.\nWaiting for $1 to start...\033[0m"

docker-compose.hobby.yml

@@ -74,6 +74,8 @@ services:
             SENTRY_DSN: $SENTRY_DSN
             SITE_URL: https://$DOMAIN
             SECRET_KEY: $POSTHOG_SECRET
+            RECORDINGS_INGESTER_URL: http://plugins:6738
+
     plugins:
         extends:
             file: docker-compose.base.yml

plugin-server/jest.config.js

@@ -8,4 +8,5 @@ module.exports = {
     setupFilesAfterEnv: ['./jest.setup.fetch-mock.js'],
     testMatch: ['<rootDir>/tests/**/*.test.ts'],
     testTimeout: 60000,
+    modulePathIgnorePatterns: ['<rootDir>/.tmp/'],
 }

plugin-server/src/capabilities.ts

@@ -1,5 +1,5 @@
 import { PluginServerCapabilities, PluginServerMode, PluginsServerConfig, stringToPluginServerMode } from './types'
-import { isTestEnv } from './utils/env-utils'
+import { isDevEnv, isTestEnv } from './utils/env-utils'
 
 export function getPluginServerCapabilities(config: PluginsServerConfig): PluginServerCapabilities {
     const mode: PluginServerMode | null = config.PLUGIN_SERVER_MODE
@@ -19,6 +19,7 @@ export function getPluginServerCapabilities(config: PluginsServerConfig): Plugin
                 processAsyncOnEventHandlers: true,
                 processAsyncWebhooksHandlers: true,
                 sessionRecordingBlobIngestion: true,
+                sessionRecordingV3Ingestion: isDevEnv(),
                 personOverrides: true,
                 appManagementSingleton: true,
                 preflightSchedules: true,
@@ -55,6 +56,12 @@ export function getPluginServerCapabilities(config: PluginsServerConfig): Plugin
                 sessionRecordingBlobIngestion: true,
                 ...sharedCapabilities,
             }
+
+        case PluginServerMode.recordings_ingestion_v3:
+            return {
+                sessionRecordingV3Ingestion: true,
+                ...sharedCapabilities,
+            }
         case PluginServerMode.async_onevent:
             return {
                 processAsyncOnEventHandlers: true,

plugin-server/src/main/ingestion-queues/session-recording/process-event.ts

@@ -1,14 +1,7 @@
-import { Properties } from '@posthog/plugin-scaffold'
 import { captureException } from '@sentry/node'
 import { DateTime } from 'luxon'
 
-import {
-    ClickHouseTimestamp,
-    PerformanceEventReverseMapping,
-    RawPerformanceEvent,
-    RRWebEvent,
-    TimestampFormat,
-} from '../../../types'
+import { ClickHouseTimestamp, RRWebEvent, TimestampFormat } from '../../../types'
 import { status } from '../../../utils/status'
 import { castTimestampOrNow } from '../../../utils/utils'
 import { activeMilliseconds } from './snapshot-segmenter'
@@ -271,23 +264,3 @@ export const createSessionReplayEvent = (
 
     return data
 }
-
-export function createPerformanceEvent(uuid: string, team_id: number, distinct_id: string, properties: Properties) {
-    const data: Partial<RawPerformanceEvent> = {
-        uuid,
-        team_id: team_id,
-        distinct_id: distinct_id,
-        session_id: properties['$session_id'],
-        window_id: properties['$window_id'],
-        pageview_id: properties['$pageview_id'],
-        current_url: properties['$current_url'],
-    }
-
-    Object.entries(PerformanceEventReverseMapping).forEach(([key, value]) => {
-        if (key in properties) {
-            data[value] = properties[key]
-        }
-    })
-
-    return data
-}