Skip to content

Commit

Permalink
feat(suite): distribution for ab testing
Browse files Browse the repository at this point in the history
  • Loading branch information
adderpositive committed Nov 14, 2024
1 parent 06ad1fd commit cd14491
Show file tree
Hide file tree
Showing 5 changed files with 212 additions and 0 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import { ReactElement } from 'react';

import { useExperiment } from 'src/hooks/experiment/useExperiment';

interface ExperimentWrapperProps {
id: string;
components: Array<{
variant: string;
element: ReactElement;
}>;
}

/**
* @param components last item in components is default
*/
export const ExperimentWrapper = ({
id,
components,
}: ExperimentWrapperProps): ReactElement | null => {
const { experiment, activeExperimentVariant } = useExperiment(id);
const areComponentEmpty = !components.length;

if (areComponentEmpty) return null;

const defaultComponent = components[components.length - 1];
const experimentOrVariantNotFound = !experiment || !activeExperimentVariant;
const experimentAndComponentsMismatch = experiment?.groups.length !== components.length;

if (experimentOrVariantNotFound || experimentAndComponentsMismatch)
return defaultComponent.element;

const activeComponent = components.find(
component => component.variant === activeExperimentVariant.variant,
);

return activeComponent?.element ?? defaultComponent.element;
};
22 changes: 22 additions & 0 deletions packages/suite/src/hooks/experiment/useExperiment.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
import { useMemo } from 'react';

import { selectAnalyticsInstanceId } from '@suite-common/analytics';
import { selectExperimentById } from '@suite-common/message-system';

import { useSelector } from 'src/hooks/suite';
import { selectActiveExperimentGroup } from 'src/utils/suite/experiment';

export const useExperiment = (id: string) => {
const state = useSelector(state => state);
const instanceId = selectAnalyticsInstanceId(state);
const experiment = useSelector(selectExperimentById(id));
const activeExperimentVariant = useMemo(
() => selectActiveExperimentGroup({ instanceId, experiment }),
[instanceId, experiment],
);

return {
experiment,
activeExperimentVariant,
};
};
19 changes: 19 additions & 0 deletions packages/suite/src/utils/suite/__fixtures__/experiment.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import { getWeakRandomId } from '@trezor/utils';

// getWeakRandomId is also used for generating instanceId
export const getArrayOfInstanceIds = (count: number) =>
Array.from({ length: count }, () => getWeakRandomId(10));

export const experimentTest = {
id: 'experiment-test',
groups: [
{
variant: 'A',
percentage: 20,
},
{
variant: 'B',
percentage: 80,
},
],
};
70 changes: 70 additions & 0 deletions packages/suite/src/utils/suite/__tests__/experiment.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
import { experimentTest, getArrayOfInstanceIds } from 'src/utils/suite/__fixtures__/experiment';
import {
getExperimentGroupByInclusion,
getInclusionFromInstanceId,
selectActiveExperimentGroup,
} from 'src/utils/suite/experiment';

describe('testing experiment utils', () => {
it('test getInclusionFromInstanceId whether returns percentage between 0 and 99', () => {
const arrayOfIds = getArrayOfInstanceIds(100);
const isExistNumberOutOfRange = arrayOfIds.some(id => {
const percentage = getInclusionFromInstanceId(id);

return percentage < 0 || percentage > 99;
});

expect(isExistNumberOutOfRange).toEqual(false);
});

it('test getExperimentGroupByInclusion whether instanceId is not in range of variants', () => {
const arrayOfIds = getArrayOfInstanceIds(100);
const isExistInstanceIdNotInVariantRange = arrayOfIds.some(id => {
const inclusion = getInclusionFromInstanceId(id);
const group = getExperimentGroupByInclusion({
groups: experimentTest.groups,
inclusion,
});

return group === undefined;
});

expect(isExistInstanceIdNotInVariantRange).toEqual(false);
});

it('test selectActiveExperimentGroup share of variant inclusion', () => {
const deviation = 0.05;
const sampleSize = 1000;
let groupACount = 0;
let groupBCount = 0;

const arrayOfIds = getArrayOfInstanceIds(sampleSize);

arrayOfIds.forEach(id => {
const selectedGroup = selectActiveExperimentGroup({
experiment: experimentTest,
instanceId: id,
});

if (selectedGroup?.variant === 'A') {
groupACount += 1;
}

if (selectedGroup?.variant === 'B') {
groupBCount += 1;
}
});

const shareA = groupACount / sampleSize;
const shareB = groupBCount / sampleSize;

expect(shareA).toBeGreaterThanOrEqual(
experimentTest.groups[0].percentage / 100 - deviation,
);
expect(shareA).toBeLessThanOrEqual(experimentTest.groups[0].percentage / 100 + deviation);
expect(shareB).toBeGreaterThanOrEqual(
experimentTest.groups[1].percentage / 100 - deviation,
);
expect(shareB).toBeLessThanOrEqual(experimentTest.groups[1].percentage / 100 + deviation);
});
});
64 changes: 64 additions & 0 deletions packages/suite/src/utils/suite/experiment.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import { createHash } from 'crypto';

import { ExperimentsItem } from '@suite-common/suite-types';

type ExperimentCategoriesProps = {
experiment: ExperimentsItem | undefined;
instanceId: string | undefined;
};

type ExperimentsGroupsType = ExperimentsItem['groups'];
type ExperimentsGroupType = ExperimentsGroupsType[number];

type ExperimentGetGroupByInclusion = {
groups: ExperimentsGroupsType;
inclusion: number;
};

/**
* @returns number between 0 and 99 generated from instanceId
*/
export const getInclusionFromInstanceId = (instanceId: string) => {
const hash = createHash('sha256').update(instanceId).digest('hex').slice(0, 8);

return parseInt(hash, 16) % 100;
};

export const getExperimentGroupByInclusion = ({
groups,
inclusion,
}: ExperimentGetGroupByInclusion): ExperimentsGroupType | undefined => {
let currentPercentage = 0;

const extendedExperiment = groups.map(group => {
const result = {
group,
range: [currentPercentage, currentPercentage + group.percentage - 1],
};

currentPercentage += group.percentage;

return result;
});

return extendedExperiment.find(
group => group.range[0] <= inclusion && group.range[1] >= inclusion,
)?.group;
};

export const selectActiveExperimentGroup = ({
experiment,
instanceId,
}: ExperimentCategoriesProps): ExperimentsGroupType | undefined => {
if (!instanceId || !experiment) return undefined;

const inclusionFromInstanceId = getInclusionFromInstanceId(instanceId);
const { groups } = experiment;

const experimentRange = getExperimentGroupByInclusion({
groups,
inclusion: inclusionFromInstanceId,
});

return experimentRange;
};

0 comments on commit cd14491

Please sign in to comment.