diff --git a/package-lock.json b/package-lock.json index 424f6df57..212a75eb5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@adobe/spacecat-audit-worker", - "version": "1.183.2", + "version": "1.183.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@adobe/spacecat-audit-worker", - "version": "1.183.2", + "version": "1.183.1", "license": "Apache-2.0", "dependencies": { "@adobe/fetch": "4.2.3", @@ -25,7 +25,7 @@ "@adobe/spacecat-shared-rum-api-client": "2.37.6", "@adobe/spacecat-shared-rum-api-client-v1": "npm:@adobe/spacecat-shared-rum-api-client@1.8.4", "@adobe/spacecat-shared-scrape-client": "2.1.4", - "@adobe/spacecat-shared-utils": "1.50.7", + "@adobe/spacecat-shared-utils": "https://gitpkg.now.sh/adobe/spacecat-shared/packages/spacecat-shared-utils?2be677c1bf7457fe55fd6ca875ef4d7b61acf0f7", "@adobe/structured-data-validator": "1.5.0", "@aws-sdk/client-athena": "3.888.0", "@aws-sdk/client-lambda": "3.888.0", @@ -36955,8 +36955,8 @@ }, "node_modules/@adobe/spacecat-shared-utils": { "version": "1.50.7", - "resolved": "https://registry.npmjs.org/@adobe/spacecat-shared-utils/-/spacecat-shared-utils-1.50.7.tgz", - "integrity": "sha512-ZoCtGA1ENZYfBPYg7R+4CwraNmrP3QgP8H++U8+7GG99KESLUfQvWetwsc/09/L+dAtUZLBnz+0dLqVnH0RBDw==", + "resolved": "https://gitpkg.now.sh/adobe/spacecat-shared/packages/spacecat-shared-utils?2be677c1bf7457fe55fd6ca875ef4d7b61acf0f7", + "integrity": "sha512-iHqoCuTBuZSv0LRBbNKzLsxLGpqvjt81lm9XAEWqjnnKnEXbr5+s/9f6NNAmBmHrWOHdQ5OcmeAkz657tVnnSw==", "license": "Apache-2.0", "dependencies": { "@adobe/fetch": "4.2.3", @@ -36968,6 +36968,7 @@ "@json2csv/plainjs": "7.0.6", "aws-xray-sdk": "3.10.3", "date-fns": "4.1.0", + "jstat-esm": "2.0.2", "validator": "^13.15.15" }, "engines": { @@ -58886,6 +58887,12 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/jstat-esm": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/jstat-esm/-/jstat-esm-2.0.2.tgz", + "integrity": "sha512-D/TKdqsiS4Xa1QfzXsllvOtVArr6xlNkZ9JdboCRuaQh5Uj85+lER4NAJON73tphu2bqGc5FIm9l3WDbDR4lOw==", + "license": "MIT" + }, "node_modules/jszip": { "version": "3.10.1", "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", diff --git a/package.json b/package.json index 3c1401a90..3ce4092d7 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@adobe/spacecat-audit-worker", - "version": "1.183.2", + "version": "1.183.1", "description": "SpaceCat Audit Worker", "main": "src/index.js", "type": "module", @@ -84,7 +84,7 @@ "@adobe/spacecat-shared-http-utils": "1.17.3", "@adobe/spacecat-shared-rum-api-client": "2.37.6", "@adobe/spacecat-shared-rum-api-client-v1": "npm:@adobe/spacecat-shared-rum-api-client@1.8.4", - "@adobe/spacecat-shared-utils": "1.50.7", + "@adobe/spacecat-shared-utils": "https://gitpkg.now.sh/adobe/spacecat-shared/packages/spacecat-shared-utils?2be677c1bf7457fe55fd6ca875ef4d7b61acf0f7", "@adobe/spacecat-shared-scrape-client": "2.1.4", "@adobe/structured-data-validator": "1.5.0", "@aws-sdk/client-athena": "3.888.0", diff --git a/src/common/base-audit.js b/src/common/base-audit.js index f2df1b05e..bc882b047 100644 --- a/src/common/base-audit.js +++ b/src/common/base-audit.js @@ -144,6 +144,7 @@ export class BaseAudit { async processAuditResult(result, params, context) { const { type, site } = params; + const { log } = context; const { auditResult, fullAuditRef } = result; const auditData = { @@ -156,6 +157,8 @@ export class BaseAudit { }; const audit = await this.persister(auditData, context); + log.info(`Audit data from processAuditResult: ${JSON.stringify(auditData, null, 2)}`); + log.info(`Site id [${site.getId()}] audit metadata persisted: ${audit.getId()}`); context.audit = audit; return this.runPostProcessors( audit, diff --git a/src/experimentation-ess/all.js b/src/experimentation-ess/all.js index 5fdeb61df..255e69153 100644 --- a/src/experimentation-ess/all.js +++ b/src/experimentation-ess/all.js @@ -24,10 +24,12 @@ async function persistOnlyMetadata(auditData, context) { // whole audit result will be bigger than the allowed size in dynamo const { dataAccess } = context; const { Audit } = dataAccess; - await Audit.create({ + const audit = await Audit.create({ ...auditData, auditResult: [], // deliberately overrides the result }); + log.info(`ESS Experimentation All Audit metadata persisted: ${audit.getId()}`); + return audit; } export async function essExperimentationAllAuditRunner(auditUrl, context, site) { @@ -38,9 +40,12 @@ export async function essExperimentationAllAuditRunner(auditUrl, context, site) const startTime = process.hrtime(); const latestAudit = await LatestAudit.findBySiteIdAndAuditType(siteId, 'experimentation-ess-all'); + log.info(`ESS Experimentation All Audit latest audit: ${latestAudit ? latestAudit.getId() : 'null'}`); const experiments = await Experiment.allBySiteId(siteId); + log.info(`ESS Experimentation All Audit experiments: ${JSON.stringify(experiments, null, 2)}`); const activeExperiments = experiments.filter((experiment) => ( experiment.getStatus() && experiment.getStatus().toLowerCase() === 'active' && experiment.getStartDate() !== null)); + log.info(`ESS Experimentation All Audit active experiments: ${JSON.stringify(activeExperiments, null, 2)}`); let days; if (latestAudit === null) { // experiment-ess-all audit has never been run before @@ -65,7 +70,7 @@ export async function essExperimentationAllAuditRunner(auditUrl, context, site) days, ); - log.info(`ESS Experimentation All Audit data size: ${JSON.stringify(auditData).length}`); + log.info(`ESS Experimentation All Audit data : ${JSON.stringify(auditData, null, 2)}`); const endTime = process.hrtime(startTime); const elapsedSeconds = endTime[0] + endTime[1] / 1e9; diff --git a/src/experimentation-ess/common.js b/src/experimentation-ess/common.js index ab00f90df..6e77dc7fb 100755 --- a/src/experimentation-ess/common.js +++ b/src/experimentation-ess/common.js @@ -12,10 +12,8 @@ /* c8 ignore start */ import RUMAPIClient from '@adobe/spacecat-shared-rum-api-client'; -import { tracingFetch as fetch } from '@adobe/spacecat-shared-utils'; +import { tracingFetch as fetch, calculateConfidence } from '@adobe/spacecat-shared-utils'; import { JSDOM } from 'jsdom'; -import { LambdaClient, InvokeCommand } from '@aws-sdk/client-lambda'; -import { defaultProvider } from '@aws-sdk/credential-provider-node'; const DEFAULT_MULTIPAGE_EXPERIMENT_DAILY_PAGE_VIEWS_THRESHOLD = 500; @@ -29,7 +27,6 @@ const EXPERIMENT_PLUGIN_OPTIONS = { const EXCLUDE_UPDATE_PROPERTIES = ['split']; const METRIC_CHECKPOINTS = ['click', 'convert', 'formsubmit']; -const SPACECAT_STATISTICS_SERVICE_ARN = 'arn:aws:lambda:us-east-1:282898975672:function:spacecat-services--statistics-service'; let log = console; @@ -355,67 +352,33 @@ function mergeData(experiment, experimentMetadata, url) { return experiment; } -async function invokeLambdaFunction(payload) { - const lambdaClient = new LambdaClient({ - region: 'us-east-1', - credentials: defaultProvider(), - }); - const invokeParams = { - FunctionName: SPACECAT_STATISTICS_SERVICE_ARN, - InvocationType: 'RequestResponse', - Payload: JSON.stringify(payload), - }; - const response = await lambdaClient.send(new InvokeCommand(invokeParams)); - return JSON.parse(new TextDecoder().decode(response.Payload)); -} - -async function addPValues(experimentData) { - const lambdaPayload = { - type: 'statsig', - payload: { - rumData: { - }, - }, - }; +async function addConfidence(experimentData) { for (const experiment of experimentData) { - const id = `${experiment.id}#${experiment.url}`; - lambdaPayload.payload.rumData[id] = {}; const metric = experiment.conversionEventName || 'click'; - for (const variant of experiment.variants) { - lambdaPayload.payload.rumData[id][variant.name] = { - views: variant.samples || 0, - metrics: variant.metrics?.find((m) => (m.type === metric && m.selector === '*'))?.samples || 0, - }; + const controlVariant = experiment.variants.find((v) => v.name === 'control'); + const controlVariantMetrics = controlVariant.metrics?.find((m) => (m.type === metric && m.selector === '*'))?.samples || 0; + if (!controlVariantMetrics) { + log.error(`Error calculating confidence: No control variant found for experiment ${experiment.id}#${experiment.url}`); + // eslint-disable-next-line no-continue + continue; } - } - log.info('Lambda Payload: ', JSON.stringify(lambdaPayload, null, 2)); - let lambdaResult; - try { - const lambdaResponse = await invokeLambdaFunction(lambdaPayload); - log.info('Lambda Response: ', JSON.stringify(lambdaResponse, null, 2)); - const lambdaResponseBody = typeof (lambdaResponse.body) === 'string' ? JSON.parse(lambdaResponse.body) : lambdaResponse.body; - lambdaResult = lambdaResponseBody.result; - } catch (error) { - log.error('Error invoking lambda function: ', error); - } - if (!lambdaResult) { - log.error('Error calculating p-values: No result from lambda function'); - return; - } - for (const experiment of experimentData) { - const id = `${experiment.id}#${experiment.url}`; - const stats = lambdaResult[id]; - if (stats && !stats.error) { - for (const variant of experiment.variants) { - const variantStats = stats[variant.name]; - if (variantStats && !variantStats.error && !Number.isNaN(variantStats.p_value)) { - variant.p_value = variantStats.p_value; - variant.power = variantStats.power; - variant.statsig = (variantStats.statsig).toLowerCase() === 'true'; - } + for (const variant of experiment.variants) { + if (variant.name === 'control') { + // eslint-disable-next-line no-continue + continue; } - } else { - log.error(`Error calculating p-values: ${stats} for experiment ${experiment.id}`); + const variantMetrics = variant.metrics?.find((m) => (m.type === metric && m.selector === '*'))?.samples; + if (!variantMetrics) { + log.error(`Error calculating confidence: No variant metrics found for experiment ${experiment.id}#${experiment.url}`); + // eslint-disable-next-line no-continue + continue; + } + variant.confidence = calculateConfidence( + controlVariantMetrics, + controlVariant.samples || 0, + variantMetrics, + variant.samples || 0, + ); } } } @@ -644,7 +607,7 @@ async function processExperimentRUMData(experimentInsights, context, days) { dailyPageViewsThreshold, ); const experimentData = await convertToExperimentsSchema(experimentInsights); - await addPValues(experimentData); + await addConfidence(experimentData); return experimentData; } @@ -665,6 +628,7 @@ export async function postProcessor(auditUrl, auditData, context) { const { dataAccess } = context; const { Experiment } = dataAccess; log = context.log; + log.info('Post Processor of ESS Experimentation audit: ', JSON.stringify(auditData, null, 2)); // iterate array auditData.auditResult for (const experiment of auditData.auditResult) { const experimentData = { @@ -672,8 +636,8 @@ export async function postProcessor(auditUrl, auditData, context) { expId: experiment.id, name: experiment.label, url: experiment.url, - startDate: experiment.startDate, - endDate: experiment.endDate, + startDate: experiment.startDate ? new Date(experiment.startDate).toISOString() : null, + endDate: experiment.endDate ? new Date(experiment.endDate).toISOString() : null, status: experiment.status, type: experiment.type, variants: experiment.variants, @@ -720,6 +684,7 @@ export async function postProcessor(auditUrl, auditData, context) { } } } + log.info(`Experiment data for site ${auditData.siteId} has been upserted: ${JSON.stringify(experimentData, null, 2)}`); // eslint-disable-next-line no-await-in-loop await Experiment.create(experimentData); }