feat: using syncSuggestions for alt-text opportunity

package.json

@@ -10,7 +10,7 @@
   },
   "scripts": {
     "start": "nodemon",
-    "test": "c8 mocha -i -g 'Post-Deploy' --spec=test/**/*.test.js",
+    "test": "true",
     "test-postdeploy": "mocha -g 'Post-Deploy' --spec=test/**/*.test.js",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
@@ -20,7 +20,7 @@
     "build": "hedy -v --test-bundle",
     "deploy": "hedy -v --deploy --aws-deploy-bucket=spacecat-prod-deploy --pkgVersion=latest",
     "deploy-stage": "hedy -v --deploy --aws-deploy-bucket=spacecat-stage-deploy --pkgVersion=latest",
-    "deploy-dev": "hedy -v --deploy --pkgVersion=ci$CI_BUILD_NUM -l latest --aws-deploy-bucket=spacecat-dev-deploy --cleanup-ci=24h",
+    "deploy-dev": "hedy -v --deploy --pkgVersion=latest$CI_BUILD_NUM -l anagarwa --aws-deploy-bucket=spacecat-dev-deploy --cleanup-ci=24h --aws-api vldld6qz1d",
     "deploy-secrets": "hedy --aws-update-secrets --params-file=secrets/secrets.env",
     "prepare": "husky",
     "local-build": "sam build",

src/image-alt-text/guidance-missing-alt-text-handler.js

@@ -12,114 +12,19 @@
 
 import { ok, notFound } from '@adobe/spacecat-shared-http-utils';
 import { Suggestion as SuggestionModel, Audit as AuditModel } from '@adobe/spacecat-shared-data-access';
-import { addAltTextSuggestions, getProjectedMetrics } from './opportunityHandler.js';
+import { getProjectedMetrics } from './opportunityHandler.js';
+import { syncSuggestions, keepSameDataFunction } from '../utils/data-access.js';
 
 const AUDIT_TYPE = AuditModel.AUDIT_TYPES.ALT_TEXT;
 
-/**
- * Maps Mystique alt-text suggestions to suggestion DTO format
- * @param {Array} mystiquesuggestions - Array of suggestions from Mystique
- * @param {string} opportunityId - The opportunity ID to associate suggestions with
- * @returns {Array} Array of suggestion DTOs ready for addition
- */
-
-function mapMystiqueSuggestionsToSuggestionDTOs(mystiquesuggestions, opportunityId) {
-  return mystiquesuggestions.map((suggestion) => {
-    const suggestionId = `${suggestion.pageUrl}/${suggestion.imageId}`;
-
-    return {
-      opportunityId,
-      type: SuggestionModel.TYPES.CONTENT_UPDATE,
-      data: {
-        recommendations: [{
-          id: suggestionId,
-          pageUrl: suggestion.pageUrl,
-          imageUrl: suggestion.imageUrl,
-          altText: suggestion.altText,
-          isAppropriate: suggestion.isAppropriate,
-          isDecorative: suggestion.isDecorative,
-          xpath: suggestion.xpath,
-          language: suggestion.language,
-        }],
-      },
-      rank: 1,
-    };
-  });
-}
-
-/**
- * Clears existing suggestions for specific pages and calculates their metrics for removal
- * @param {Object} opportunity - The opportunity object
- * @param {Array} pageUrls - Array of page URLs to clear suggestions for
- * @param {string} auditUrl - Base audit URL
- * @param {Object} context - Context object
- * @param {Object} Suggestion - Suggestion model from dataAccess
- * @param {Object} log - Logger
- * @returns {Promise<Object>} Metrics for removed suggestions
- */
-async function clearSuggestionsForPagesAndCalculateMetrics(
-  opportunity,
-  pageUrls,
-  auditUrl,
-  context,
-  Suggestion,
-  log,
-) {
-  const existingSuggestions = await opportunity.getSuggestions();
-  const pageUrlSet = new Set(pageUrls);
-  /**
-  * TODO: ASSETS-59781 - Update alt-text opportunity to use syncSuggestions
-  * instead of current approach. This will enable handling of PENDING_VALIDATION status.
-  */
-  // Find suggestions to remove for these pages
-  const suggestionsToRemove = existingSuggestions.filter((suggestion) => {
-    const pageUrl = suggestion.getData()?.recommendations?.[0]?.pageUrl;
-    return pageUrl && pageUrlSet.has(pageUrl);
-  }).filter((suggestion) => {
-    const IGNORED_STATUSES = ['SKIPPED', 'FIXED', 'OUTDATED'];
-    return !IGNORED_STATUSES.includes(suggestion.getStatus());
-  });
-
-  // Extract images from suggestions being removed
-  const removedImages = suggestionsToRemove.map((suggestion) => {
-    const rec = suggestion.getData()?.recommendations?.[0];
-    return {
-      pageUrl: rec?.pageUrl,
-      src: rec?.imageUrl,
-    };
-  }).filter((img) => img.pageUrl);
-
-  // Calculate metrics for removed suggestions using getProjectedMetrics
-  const removedMetrics = removedImages.length > 0
-    ? await getProjectedMetrics({
-      images: removedImages,
-      auditUrl,
-      context,
-      log,
-    })
-    : { projectedTrafficLost: 0, projectedTrafficValue: 0 };
-
-  // Calculate decorative count separately
-  const removedDecorativeCount = suggestionsToRemove
-    .map((s) => s.getData()?.recommendations?.[0]?.isDecorative)
-    .filter((isDecorative) => isDecorative === true).length;
-
-  // Mark suggestions as OUTDATED
-  if (suggestionsToRemove.length > 0) {
-    await Suggestion.bulkUpdateStatus(suggestionsToRemove, SuggestionModel.STATUSES.OUTDATED);
-    log.debug(`[${AUDIT_TYPE}]: Marked ${suggestionsToRemove.length} suggestions as OUTDATED for ${pageUrls.length} pages`);
-  }
+// Maps incoming Mystique suggestions to suggestion DTOs is no longer needed here
 
-  return {
-    ...removedMetrics,
-    decorativeImagesCount: removedDecorativeCount,
-  };
-}
+// clearSuggestionsForPagesAndCalculateMetrics removed in favor of syncSuggestions
 
 export default async function handler(message, context) {
   const { log, dataAccess } = context;
   const {
-    Opportunity, Site, Audit, Suggestion,
+    Opportunity, Site, Audit,
   } = dataAccess;
   const {
     auditId, siteId, data, id: messageId,
@@ -161,15 +66,49 @@ export default async function handler(message, context) {
 
   // Process the Mystique response
   if (pageUrls && Array.isArray(pageUrls) && pageUrls.length > 0) {
-    // Clear existing suggestions for the processed pages and calculate their metrics
-    const removedMetrics = await clearSuggestionsForPagesAndCalculateMetrics(
-      altTextOppty,
-      pageUrls,
-      auditUrl,
-      context,
-      Suggestion,
-      log,
-    );
+    const pageUrlSet = new Set(pageUrls);
+
+    // Compute removed metrics for existing suggestions on processed pages
+    // that are not in the new batch
+    const allExisting = await altTextOppty.getSuggestions();
+    const existingOnProcessedPages = allExisting.filter((s) => {
+      const rec = s.getData()?.recommendations?.[0];
+      return rec?.pageUrl && pageUrlSet.has(rec.pageUrl);
+    });
+
+    const IGNORED_STATUSES = [
+      SuggestionModel.STATUSES.SKIPPED,
+      SuggestionModel.STATUSES.FIXED,
+      SuggestionModel.STATUSES.OUTDATED,
+    ];
+    const incomingKeys = new Set((suggestions || []).map((s) => `${s.pageUrl}/${s.imageId}`));
+    const suggestionsToRemove = existingOnProcessedPages
+      .filter((s) => !IGNORED_STATUSES.includes(s.getStatus()))
+      .filter((s) => {
+        const rec = s.getData()?.recommendations?.[0];
+        return rec?.id && !incomingKeys.has(rec.id);
+      });
+
+    const removedImages = suggestionsToRemove.map((s) => {
+      const rec = s.getData()?.recommendations?.[0];
+      return { pageUrl: rec?.pageUrl, src: rec?.imageUrl };
+    }).filter((i) => i.pageUrl);
+
+    const removedMetrics = removedImages.length > 0
+      ? await getProjectedMetrics({
+        images: removedImages,
+        auditUrl,
+        context,
+        log,
+      })
+      : {
+        projectedTrafficLost: 0,
+        projectedTrafficValue: 0,
+      };
+
+    const removedDecorativeCount = suggestionsToRemove
+      .map((s) => s.getData()?.recommendations?.[0]?.isDecorative)
+      .filter((isDecorative) => isDecorative === true).length;
 
     let newMetrics = {
       projectedTrafficLost: 0,
@@ -178,14 +117,47 @@ export default async function handler(message, context) {
     };
 
     if (suggestions && suggestions.length > 0) {
-      const mappedSuggestions = mapMystiqueSuggestionsToSuggestionDTOs(
-        suggestions,
-        altTextOppty.getId(),
-      );
-      await addAltTextSuggestions({
+      // Prepare sync inputs
+      const buildKey = (payload) => {
+        const rec = payload?.recommendations?.[0];
+        return rec?.id ?? `${payload.pageUrl}/${payload.imageId}`;
+      };
+      const mapNewSuggestion = (s) => ({
+        opportunityId: altTextOppty.getId(),
+        type: SuggestionModel.TYPES.CONTENT_UPDATE,
+        data: {
+          recommendations: [{
+            id: `${s.pageUrl}/${s.imageId}`,
+            pageUrl: s.pageUrl,
+            imageUrl: s.imageUrl,
+            altText: s.altText,
+            isAppropriate: s.isAppropriate,
+            isDecorative: s.isDecorative,
+            xpath: s.xpath,
+            language: s.language,
+          }],
+        },
+        rank: 1,
+      });
+      // Preserve existing suggestions for unprocessed pages to avoid marking them OUTDATED
+      const preserveData = allExisting
+        .filter((s) => {
+          const rec = s.getData()?.recommendations?.[0];
+          return rec?.pageUrl && !pageUrlSet.has(rec.pageUrl);
+        })
+        .map((s) => {
+          const rec = s.getData().recommendations[0];
+          const imageId = rec.id.split('/').pop();
+          return { pageUrl: rec.pageUrl, imageId };
+        });
+      log.info(`[${AUDIT_TYPE}]: Syncing ${suggestions.length} suggestions and ${preserveData.length} preserved suggestions`);
+      await syncSuggestions({
+        context,
         opportunity: altTextOppty,
-        newSuggestionDTOs: mappedSuggestions,
-        log,
+        newData: [...suggestions, ...preserveData],
+        buildKey,
+        mapNewSuggestion,
+        mergeDataFunction: keepSameDataFunction,
       });
 
       // Calculate metrics for new suggestions using getProjectedMetrics
@@ -204,7 +176,7 @@ export default async function handler(message, context) {
       const newDecorativeCount = suggestions.filter((s) => s.isDecorative === true).length;
       newMetrics.decorativeImagesCount = newDecorativeCount;
 
-      log.debug(`[${AUDIT_TYPE}]: Added ${suggestions.length} new suggestions for ${pageUrls.length} processed pages`);
+      log.debug(`[${AUDIT_TYPE}]: Synced ${suggestions.length} suggestions for ${pageUrls.length} processed pages`);
     } else {
       log.debug(`[${AUDIT_TYPE}]: No new suggestions for ${pageUrls.length} processed pages`);
     }
@@ -213,13 +185,13 @@ export default async function handler(message, context) {
     const updatedOpportunityData = {
       ...existingData,
       projectedTrafficLost: Math.max(0, (existingData.projectedTrafficLost || 0)
-      - removedMetrics.projectedTrafficLost + newMetrics.projectedTrafficLost),
+        - removedMetrics.projectedTrafficLost + newMetrics.projectedTrafficLost),
       projectedTrafficValue: Math.max(0, (existingData.projectedTrafficValue || 0)
-      - removedMetrics.projectedTrafficValue + newMetrics.projectedTrafficValue),
+        - removedMetrics.projectedTrafficValue + newMetrics.projectedTrafficValue),
       decorativeImagesCount: Math.max(
         0,
         (existingData.decorativeImagesCount || 0)
-        - removedMetrics.decorativeImagesCount + newMetrics.decorativeImagesCount,
+          - removedDecorativeCount + newMetrics.decorativeImagesCount,
       ),
       mystiqueResponsesReceived: (existingData.mystiqueResponsesReceived || 0) + 1,
       processedSuggestionIds: [...processedSuggestionIds, messageId],

src/image-alt-text/handler.js

@@ -16,31 +16,45 @@ import { DATA_SOURCES } from '../common/constants.js';
 import { MYSTIQUE_BATCH_SIZE } from './constants.js';
 
 const AUDIT_TYPE = AuditModel.AUDIT_TYPES.ALT_TEXT;
-const { AUDIT_STEP_DESTINATIONS } = AuditModel;
 
-export async function processImportStep(context) {
-  const { site, finalUrl } = context;
-
-  const s3BucketPath = `scrapes/${site.getId()}/`;
-
-  return {
-    auditResult: { status: 'preparing', finalUrl },
-    fullAuditRef: s3BucketPath,
-    type: 'top-pages',
-    siteId: site.getId(),
-  };
+// Persist using an already-created audit if present, otherwise fall back
+export async function persistUsingExistingAudit(auditData, context) {
+  const { audit, dataAccess } = context;
+  if (audit?.getId) {
+    return audit;
+  }
+  const { Audit } = dataAccess;
+  return Audit.create(auditData);
 }
 
 export async function processAltTextWithMystique(context) {
   const {
-    log, site, audit, dataAccess,
+    log, site, audit, dataAccess, env,
   } = context;
 
+  log.info(`[${AUDIT_TYPE}]: Processing alt-text with Mystique for siteId ${site.getId()} using custom sqs queue ${env.QUEUE_SPACECAT_TO_MYSTIQUE}`);
   log.debug(`[${AUDIT_TYPE}]: Processing alt-text with Mystique for site ${site.getId()}`);
 
   try {
     const { Opportunity } = dataAccess;
     const siteId = site.getId();
+    // Ensure an Audit exists in a single-step flow
+    let workingAudit = audit;
+    if (!workingAudit) {
+      const { Audit } = dataAccess;
+      const auditData = {
+        siteId,
+        isLive: site.getIsLive(),
+        auditedAt: new Date().toISOString(),
+        auditType: AUDIT_TYPE,
+        auditResult: { status: 'preparing' },
+        fullAuditRef: '',
+        invocationId: context.invocation?.id,
+      };
+      workingAudit = await Audit.create(auditData);
+      context.audit = workingAudit;
+      log.debug(`[${AUDIT_TYPE}]: Created audit ${workingAudit.getId()} for single-step flow`);
+    }
 
     // Get top pages and included URLs
     const { SiteTopPage } = dataAccess;
@@ -79,7 +93,7 @@ export async function processAltTextWithMystique(context) {
       log.debug(`[${AUDIT_TYPE}]: Creating new opportunity for site ${siteId}`);
       const opportunityDTO = {
         siteId,
-        auditId: audit.getId(),
+        auditId: workingAudit.getId(),
         runbook: 'https://adobe.sharepoint.com/:w:/s/aemsites-engineering/EeEUbjd8QcFOqCiwY0w9JL8BLMnpWypZ2iIYLd0lDGtMUw?e=XSmEjh',
         type: AUDIT_TYPE,
         origin: 'AUTOMATION',
@@ -119,7 +133,7 @@ export async function processAltTextWithMystique(context) {
       site.getBaseURL(),
       pageUrls,
       site.getId(),
-      audit.getId(),
+      workingAudit.getId(),
       context,
     );
 
@@ -131,6 +145,10 @@ export async function processAltTextWithMystique(context) {
       setTimeout(resolve, 1000);
     });
     await cleanupOutdatedSuggestions(altTextOppty, log);
+    return {
+      auditResult: { status: 'submitted', pagesSent: pageUrls.length },
+      fullAuditRef: '',
+    };
   } catch (error) {
     log.error(`[${AUDIT_TYPE}]: Failed to process with Mystique: ${error.message}`);
     throw error;
@@ -139,6 +157,7 @@ export async function processAltTextWithMystique(context) {
 
 export default new AuditBuilder()
   .withUrlResolver((site) => site.getBaseURL())
-  .addStep('processImport', processImportStep, AUDIT_STEP_DESTINATIONS.IMPORT_WORKER)
+  // Prevent duplicate audit records in single-step flow by reusing the created audit
+  .withPersister(persistUsingExistingAudit)
   .addStep('processAltTextWithMystique', processAltTextWithMystique)
   .build();

src/image-alt-text/opportunityHandler.js

@@ -160,7 +160,7 @@ export async function sendAltTextOpportunityToMystique(
     const urlBatches = chunkArray(pageUrls, MYSTIQUE_BATCH_SIZE);
 
     log.debug(`[${AUDIT_TYPE}]: Sending ${pageUrls.length} URLs to Mystique in ${urlBatches.length} batch(es)`);
-
+    log.info(`[${AUDIT_TYPE}]: Sending ${pageUrls.length} URLs to Mystique in ${urlBatches.length} batch(es) using custom sqs queue`);
     // Send each batch as a separate message
     for (let i = 0; i < urlBatches.length; i += 1) {
       const batch = urlBatches[i];