Skip to content

Commit

Permalink
feat: default replay partial compression on (#1445)
Browse files Browse the repository at this point in the history
pauldambra authored Oct 1, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent f09f334 commit 8c83b1f
Showing 3 changed files with 62 additions and 17 deletions.
58 changes: 46 additions & 12 deletions src/__tests__/extensions/replay/sessionrecording.test.ts
Original file line number Diff line number Diff line change
@@ -105,9 +105,9 @@ const createIncrementalMouseEvent = () => {
})
}

const createIncrementalMutationEvent = () => {
const createIncrementalMutationEvent = (mutations?: { texts: any[] }) => {
const mutationData = {
texts: [],
texts: mutations?.texts || [],
attributes: [],
removes: [],
adds: [],
@@ -121,15 +121,15 @@ const createIncrementalMutationEvent = () => {
})
}

const createIncrementalStyleSheetEvent = () => {
const createIncrementalStyleSheetEvent = (mutations?: { adds: any[] }) => {
return createIncrementalSnapshot({
data: {
// doesn't need to be a valid style sheet event
source: 8,
id: 1,
styleId: 1,
removes: [],
adds: [],
adds: mutations.adds || [],
replace: 'something',
replaceSync: 'something',
},
@@ -1009,6 +1009,7 @@ describe('SessionRecording', () => {
})

it('can emit when there are circular references', () => {
posthog.config.session_recording.compress_events = false
sessionRecording.afterDecideResponse(makeDecideResponse({ sessionRecording: { endpoint: '/s/' } }))
sessionRecording.startIfEnabledOrStop()

@@ -1271,6 +1272,10 @@ describe('SessionRecording', () => {
type: INCREMENTAL_SNAPSHOT_EVENT_TYPE,
data: {
source: 0,
adds: [],
attributes: [],
removes: [],
texts: [],
},
timestamp: activityTimestamp,
}
@@ -1490,7 +1495,7 @@ describe('SessionRecording', () => {
expect(sessionRecording['buffer']).toEqual({
data: [firstSnapshotEvent, secondSnapshot],
sessionId: firstSessionId,
size: 136,
size: 186,
windowId: expect.any(String),
})

@@ -1514,7 +1519,7 @@ describe('SessionRecording', () => {
{
$snapshot_data: [firstSnapshotEvent, secondSnapshot],
$session_id: firstSessionId,
$snapshot_bytes: 136,
$snapshot_bytes: 186,
$window_id: expect.any(String),
},
{
@@ -1576,7 +1581,7 @@ describe('SessionRecording', () => {
expect(sessionRecording['buffer']).toEqual({
data: [firstSnapshotEvent, secondSnapshot],
sessionId: firstSessionId,
size: 136,
size: 186,
windowId: expect.any(String),
})

@@ -1604,7 +1609,7 @@ describe('SessionRecording', () => {
{
$snapshot_data: [firstSnapshotEvent, secondSnapshot],
$session_id: firstSessionId,
$snapshot_bytes: 136,
$snapshot_bytes: 186,
$window_id: expect.any(String),
},
{
@@ -1630,7 +1635,7 @@ describe('SessionRecording', () => {
{
$snapshot_data: [firstSnapshotEvent, secondSnapshot],
$session_id: firstSessionId,
$snapshot_bytes: 136,
$snapshot_bytes: 186,
$window_id: expect.any(String),
},
{
@@ -1969,7 +1974,13 @@ describe('SessionRecording', () => {
})

it('compresses full snapshot data', () => {
_emit(createFullSnapshot())
_emit(
createFullSnapshot({
data: {
content: Array(30).fill(uuidv7()).join(''),
},
})
)
sessionRecording['_flushBuffer']()

expect(posthog.capture).toHaveBeenCalledWith(
@@ -1990,8 +2001,29 @@ describe('SessionRecording', () => {
)
})

it('does not compress small full snapshot data', () => {
_emit(createFullSnapshot({ data: { content: 'small' } }))
sessionRecording['_flushBuffer']()

expect(posthog.capture).toHaveBeenCalledWith(
'$snapshot',
{
$snapshot_data: [
{
data: { content: 'small' },
type: 2,
},
],
$session_id: sessionId,
$snapshot_bytes: expect.any(Number),
$window_id: 'windowId',
},
captureOptions
)
})

it('compresses incremental snapshot mutation data', () => {
_emit(createIncrementalMutationEvent())
_emit(createIncrementalMutationEvent({ texts: [Array(30).fill(uuidv7()).join('')] }))
sessionRecording['_flushBuffer']()

expect(posthog.capture).toHaveBeenCalledWith(
@@ -2020,7 +2052,7 @@ describe('SessionRecording', () => {
})

it('compresses incremental snapshot style data', () => {
_emit(createIncrementalStyleSheetEvent())
_emit(createIncrementalStyleSheetEvent({ adds: [Array(30).fill(uuidv7()).join('')] }))
sessionRecording['_flushBuffer']()

expect(posthog.capture).toHaveBeenCalledWith(
@@ -2049,6 +2081,8 @@ describe('SessionRecording', () => {
)
})

it('does not compress small incremental snapshot data', () => {})

it('does not compress incremental snapshot non full data', () => {
const mouseEvent = createIncrementalMouseEvent()
_emit(mouseEvent)
15 changes: 11 additions & 4 deletions src/extensions/replay/sessionrecording.ts
Original file line number Diff line number Diff line change
@@ -39,7 +39,9 @@ const BASE_ENDPOINT = '/s/'
const FIVE_MINUTES = 1000 * 60 * 5
const TWO_SECONDS = 2000
export const RECORDING_IDLE_THRESHOLD_MS = FIVE_MINUTES
export const RECORDING_MAX_EVENT_SIZE = 1024 * 1024 * 0.9 // ~1mb (with some wiggle room)
const ONE_KB = 1024
const PARTIAL_COMPRESSION_THRESHOLD = ONE_KB
export const RECORDING_MAX_EVENT_SIZE = ONE_KB * ONE_KB * 0.9 // ~1mb (with some wiggle room)
export const RECORDING_BUFFER_TIMEOUT = 2000 // 2 seconds
export const SESSION_RECORDING_BATCH_KEY = 'recordings'

@@ -144,6 +146,11 @@ function gzipToString(data: unknown): string {
// but we want to be able to inspect metadata during ingestion, and don't want to compress the entire event
// so we have a custom packer that only compresses part of some events
function compressEvent(event: eventWithTime, ph: PostHog): eventWithTime | compressedEventWithTime {
const originalSize = estimateSize(event)
if (originalSize < PARTIAL_COMPRESSION_THRESHOLD) {
return event
}

try {
if (event.type === EventType.FullSnapshot) {
return {
@@ -937,10 +944,10 @@ export class SessionRecording {
}
}

const eventToSend = this.instance.config.session_recording.compress_events
? compressEvent(event, this.instance)
: event
const eventToSend =
this.instance.config.session_recording.compress_events ?? true ? compressEvent(event, this.instance) : event
const size = estimateSize(eventToSend)

const properties = {
$snapshot_bytes: size,
$snapshot_data: eventToSend,
6 changes: 5 additions & 1 deletion src/types.ts
Original file line number Diff line number Diff line change
@@ -277,7 +277,11 @@ export interface SessionRecordingOptions {
recordBody?: boolean
// ADVANCED: while a user is active we take a full snapshot of the browser every interval. For very few sites playback performance might be better with different interval. Set to 0 to disable
full_snapshot_interval_millis?: number
// PREVIEW: whether to compress part of the events before sending them to the server, this is a preview feature and may change without notice
/*
ADVANCED: whether to partially compress rrweb events before sending them to the server,
defaults to true, can be set to false to disable partial compression
NB requests are still compressed when sent to the server regardless of this setting
*/
compress_events?: boolean
/*
ADVANCED: alters the threshold before a recording considers a user has become idle.

0 comments on commit 8c83b1f

Please sign in to comment.