Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Bootstrap common EppoClient #251

Open
wants to merge 15 commits into
base: main
Choose a base branch
from
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ help: Makefile
testDataDir := test/data/
tempDir := ${testDataDir}temp/
gitDataDir := ${tempDir}sdk-test-data/
branchName := main
branchName := tp/bootstrap-config
githubRepoLink := https://github.com/Eppo-exp/sdk-test-data.git
repoName := sdk-test-data
.PHONY: test-data
Expand Down
41 changes: 37 additions & 4 deletions src/client/eppo-client-with-bandits.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,21 @@ import {
testCasesByFileName,
BanditTestCase,
BANDIT_TEST_DATA_DIR,
readMockConfigurationWireResponse,
SHARED_BOOTSTRAP_BANDIT_FLAGS_FILE,
} from '../../test/testHelpers';
import ApiEndpoints from '../api-endpoints';
import { IAssignmentEvent, IAssignmentLogger } from '../assignment-logger';
import { BanditEvaluation, BanditEvaluator } from '../bandit-evaluator';
import { IBanditEvent, IBanditLogger } from '../bandit-logger';
import ConfigurationRequestor from '../configuration-requestor';
import { ConfigurationManager } from '../configuration-store/configuration-manager';
import { MemoryOnlyConfigurationStore } from '../configuration-store/memory.store';
import {
IConfigurationWire,
IPrecomputedConfiguration,
IObfuscatedPrecomputedConfigurationResponse,
ConfigurationWireV1,
} from '../configuration-wire/configuration-wire-types';
import { Evaluator, FlagEvaluation } from '../evaluator';
import {
Expand Down Expand Up @@ -64,12 +68,12 @@ describe('EppoClient Bandits E2E test', () => {
},
});
const httpClient = new FetchHttpClient(apiEndpoints, 1000);
const configurationRequestor = new ConfigurationRequestor(
httpClient,
const configManager = new ConfigurationManager(
flagStore,
banditVariationStore,
banditModelStore,
);
const configurationRequestor = new ConfigurationRequestor(httpClient, configManager, true);
await configurationRequestor.fetchAndStoreConfigurations();
});

Expand All @@ -93,8 +97,8 @@ describe('EppoClient Bandits E2E test', () => {
describe('Shared test cases', () => {
const testCases = testCasesByFileName<BanditTestCase>(BANDIT_TEST_DATA_DIR);

it.each(Object.keys(testCases))('Shared bandit test case - %s', async (fileName: string) => {
const { flag: flagKey, defaultValue, subjects } = testCases[fileName];
function testBanditCaseAgainstClient(client: EppoClient, testCase: BanditTestCase) {
const { flag: flagKey, defaultValue, subjects } = testCase;
let numAssignmentsChecked = 0;
subjects.forEach((subject) => {
// test files have actions as an array, so we convert them to a map as expected by the client
Expand Down Expand Up @@ -130,6 +134,35 @@ describe('EppoClient Bandits E2E test', () => {
});
// Ensure that this test case correctly checked some test assignments
expect(numAssignmentsChecked).toBeGreaterThan(0);
}

describe('bootstrapped client', () => {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔥 LOVE that you did this 💪

const banditFlagsConfig = ConfigurationWireV1.fromString(
readMockConfigurationWireResponse(SHARED_BOOTSTRAP_BANDIT_FLAGS_FILE),
);

let client: EppoClient;
beforeAll(async () => {
client = new EppoClient({
flagConfigurationStore: new MemoryOnlyConfigurationStore(),
banditVariationConfigurationStore: new MemoryOnlyConfigurationStore(),
banditModelConfigurationStore: new MemoryOnlyConfigurationStore(),
});
client.setIsGracefulFailureMode(false);

// Bootstrap using the bandit flag config.
await client.bootstrap(banditFlagsConfig);
});

it.each(Object.keys(testCases))('Shared bandit test case - %s', async (fileName: string) => {
testBanditCaseAgainstClient(client, testCases[fileName]);
});
});

describe('traditional client', () => {
it.each(Object.keys(testCases))('Shared bandit test case - %s', async (fileName: string) => {
testBanditCaseAgainstClient(client, testCases[fileName]);
});
});
});

Expand Down
237 changes: 155 additions & 82 deletions src/client/eppo-client.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,10 @@
IAssignmentTestCase,
MOCK_UFC_RESPONSE_FILE,
OBFUSCATED_MOCK_UFC_RESPONSE_FILE,
readMockConfigurationWireResponse,
readMockUFCResponse,
SHARED_BOOTSTRAP_FLAGS_FILE,
SHARED_BOOTSTRAP_FLAGS_OBFUSCATED_FILE,
SubjectTestCase,
testCasesByFileName,
validateTestAssignments,
Expand All @@ -18,6 +21,7 @@
import { IConfigurationStore } from '../configuration-store/configuration-store';
import { MemoryOnlyConfigurationStore } from '../configuration-store/memory.store';
import {
ConfigurationWireV1,
IConfigurationWire,
IObfuscatedPrecomputedConfigurationResponse,
ObfuscatedPrecomputedConfigurationResponse,
Expand Down Expand Up @@ -317,109 +321,178 @@
});
});

const testCases = testCasesByFileName<IAssignmentTestCase>(ASSIGNMENT_TEST_DATA_DIR);

Check warning on line 324 in src/client/eppo-client.spec.ts

View workflow job for this annotation

GitHub Actions / lint-test-sdk (18)

'testCases' is assigned a value but never used

Check warning on line 324 in src/client/eppo-client.spec.ts

View workflow job for this annotation

GitHub Actions / lint-test-sdk (20)

'testCases' is assigned a value but never used

Check warning on line 324 in src/client/eppo-client.spec.ts

View workflow job for this annotation

GitHub Actions / lint-test-sdk (22)

'testCases' is assigned a value but never used

Check warning on line 324 in src/client/eppo-client.spec.ts

View workflow job for this annotation

GitHub Actions / lint-test-sdk (23)

'testCases' is assigned a value but never used

function testCasesAgainstClient(client: EppoClient, testCase: IAssignmentTestCase) {
const { flag, variationType, defaultValue, subjects } = testCase;

let assignments: {
subject: SubjectTestCase;
assignment: string | boolean | number | null | object;
}[] = [];

const typeAssignmentFunctions = {
[VariationType.BOOLEAN]: client.getBooleanAssignment.bind(client),
[VariationType.NUMERIC]: client.getNumericAssignment.bind(client),
[VariationType.INTEGER]: client.getIntegerAssignment.bind(client),
[VariationType.STRING]: client.getStringAssignment.bind(client),
[VariationType.JSON]: client.getJSONAssignment.bind(client),
};

const assignmentFn = typeAssignmentFunctions[variationType] as (
flagKey: string,
subjectKey: string,
subjectAttributes: Record<string, AttributeType>,
defaultValue: boolean | string | number | object,
) => never;
if (!assignmentFn) {
throw new Error(`Unknown variation type: ${variationType}`);
}

assignments = getTestAssignments({ flag, variationType, defaultValue, subjects }, assignmentFn);

validateTestAssignments(assignments, flag);
}

describe('UFC Shared Test Cases', () => {
const testCases = testCasesByFileName<IAssignmentTestCase>(ASSIGNMENT_TEST_DATA_DIR);

describe('Not obfuscated', () => {
beforeAll(async () => {
global.fetch = jest.fn(() => {
return Promise.resolve({
ok: true,
status: 200,
json: () => Promise.resolve(readMockUFCResponse(MOCK_UFC_RESPONSE_FILE)),
describe('boostrapped client', () => {
const bootstrapFlagsConfig = ConfigurationWireV1.fromString(
readMockConfigurationWireResponse(SHARED_BOOTSTRAP_FLAGS_FILE),
);
const bootstrapFlagsObfuscatedConfig = ConfigurationWireV1.fromString(
readMockConfigurationWireResponse(SHARED_BOOTSTRAP_FLAGS_OBFUSCATED_FILE),
);

describe('Not obfuscated', () => {
let client: EppoClient;
beforeAll(() => {
client = new EppoClient({
flagConfigurationStore: new MemoryOnlyConfigurationStore(),
});
}) as jest.Mock;
client.setIsGracefulFailureMode(false);

await initConfiguration(storage);
});
// Bootstrap using the flags config.
client.bootstrap(bootstrapFlagsConfig);
});

it('contains some key flags', () => {
const flagKeys = client.getFlagConfigurations();

afterAll(() => {
jest.restoreAllMocks();
expect(Object.keys(flagKeys)).toContain('numeric_flag');
expect(Object.keys(flagKeys)).toContain('kill-switch');
});

it.each(Object.keys(testCases))('test variation assignment splits - %s', (fileName) => {
testCasesAgainstClient(client, testCases[fileName]);
});
});

it.each(Object.keys(testCases))('test variation assignment splits - %s', async (fileName) => {
const { flag, variationType, defaultValue, subjects } = testCases[fileName];
const client = new EppoClient({ flagConfigurationStore: storage });
client.setIsGracefulFailureMode(false);
describe('Obfuscated', () => {
let client: EppoClient;
beforeAll(async () => {
client = new EppoClient({
flagConfigurationStore: new MemoryOnlyConfigurationStore(),
});
client.setIsGracefulFailureMode(false);

let assignments: {
subject: SubjectTestCase;
assignment: string | boolean | number | null | object;
}[] = [];

const typeAssignmentFunctions = {
[VariationType.BOOLEAN]: client.getBooleanAssignment.bind(client),
[VariationType.NUMERIC]: client.getNumericAssignment.bind(client),
[VariationType.INTEGER]: client.getIntegerAssignment.bind(client),
[VariationType.STRING]: client.getStringAssignment.bind(client),
[VariationType.JSON]: client.getJSONAssignment.bind(client),
};

const assignmentFn = typeAssignmentFunctions[variationType] as (
flagKey: string,
subjectKey: string,
subjectAttributes: Record<string, AttributeType>,
defaultValue: boolean | string | number | object,
) => never;
if (!assignmentFn) {
throw new Error(`Unknown variation type: ${variationType}`);
}
// Bootstrap using the obfuscated flags config.
await client.bootstrap(bootstrapFlagsObfuscatedConfig);
});

assignments = getTestAssignments(
{ flag, variationType, defaultValue, subjects },
assignmentFn,
);
it('contains some key flags', () => {
const flagKeys = client.getFlagConfigurations();

validateTestAssignments(assignments, flag);
expect(Object.keys(flagKeys)).toContain('73fcc84c69e49e31fe16a29b2b1f803b');
expect(Object.keys(flagKeys)).toContain('69d2ea567a75b7b2da9648bf312dc3a5');
});

it.each(Object.keys(testCases))('test variation assignment splits - %s', (fileName) => {
testCasesAgainstClient(client, testCases[fileName]);
});
});
});

describe('Obfuscated', () => {
beforeAll(async () => {
global.fetch = jest.fn(() => {
return Promise.resolve({
ok: true,
status: 200,
json: () => Promise.resolve(readMockUFCResponse(OBFUSCATED_MOCK_UFC_RESPONSE_FILE)),
});
}) as jest.Mock;
describe('traditional client', () => {
describe('Not obfuscated', () => {
beforeAll(async () => {
global.fetch = jest.fn(() => {
return Promise.resolve({
ok: true,
status: 200,
json: () => Promise.resolve(readMockUFCResponse(MOCK_UFC_RESPONSE_FILE)),
});
}) as jest.Mock;

await initConfiguration(storage);
});

await initConfiguration(storage);
});
afterAll(() => {
jest.restoreAllMocks();
});

afterAll(() => {
jest.restoreAllMocks();
it.each(Object.keys(testCases))(
'test variation assignment splits - %s',
async (fileName) => {
const client = new EppoClient({ flagConfigurationStore: storage });
client.setIsGracefulFailureMode(false);

testCasesAgainstClient(client, testCases[fileName]);
},
);
});

it.each(Object.keys(testCases))('test variation assignment splits - %s', async (fileName) => {
const { flag, variationType, defaultValue, subjects } = testCases[fileName];
const client = new EppoClient({ flagConfigurationStore: storage, isObfuscated: true });
client.setIsGracefulFailureMode(false);
describe('Obfuscated', () => {
beforeAll(async () => {
global.fetch = jest.fn(() => {
return Promise.resolve({
ok: true,
status: 200,
json: () => Promise.resolve(readMockUFCResponse(OBFUSCATED_MOCK_UFC_RESPONSE_FILE)),
});
}) as jest.Mock;

await initConfiguration(storage);
});

const typeAssignmentFunctions = {
[VariationType.BOOLEAN]: client.getBooleanAssignment.bind(client),
[VariationType.NUMERIC]: client.getNumericAssignment.bind(client),
[VariationType.INTEGER]: client.getIntegerAssignment.bind(client),
[VariationType.STRING]: client.getStringAssignment.bind(client),
[VariationType.JSON]: client.getJSONAssignment.bind(client),
};

const assignmentFn = typeAssignmentFunctions[variationType] as (
flagKey: string,
subjectKey: string,
subjectAttributes: Record<string, AttributeType>,
defaultValue: boolean | string | number | object,
) => never;
if (!assignmentFn) {
throw new Error(`Unknown variation type: ${variationType}`);
}
afterAll(() => {
jest.restoreAllMocks();
});

const assignments = getTestAssignments(
{ flag, variationType, defaultValue, subjects },
assignmentFn,
it.each(Object.keys(testCases))(
'test variation assignment splits - %s',
async (fileName) => {
const { flag, variationType, defaultValue, subjects } = testCases[fileName];
const client = new EppoClient({ flagConfigurationStore: storage, isObfuscated: true });
client.setIsGracefulFailureMode(false);

const typeAssignmentFunctions = {
[VariationType.BOOLEAN]: client.getBooleanAssignment.bind(client),
[VariationType.NUMERIC]: client.getNumericAssignment.bind(client),
[VariationType.INTEGER]: client.getIntegerAssignment.bind(client),
[VariationType.STRING]: client.getStringAssignment.bind(client),
[VariationType.JSON]: client.getJSONAssignment.bind(client),
};

const assignmentFn = typeAssignmentFunctions[variationType] as (
flagKey: string,
subjectKey: string,
subjectAttributes: Record<string, AttributeType>,
defaultValue: boolean | string | number | object,
) => never;
if (!assignmentFn) {
throw new Error(`Unknown variation type: ${variationType}`);
}

const assignments = getTestAssignments(
{ flag, variationType, defaultValue, subjects },
assignmentFn,
);

validateTestAssignments(assignments, flag);
},
);

validateTestAssignments(assignments, flag);
});
});
});
Expand Down
Loading
Loading