fix: [broken-internal-links] Prompt completed but no output was found (#1603)

https://jira.corp.adobe.com/browse/SITES-36310

Author: BaDos

src/internal-links/suggestions-generator.js

@@ -13,7 +13,7 @@
 import { getPrompt, isNonEmptyArray } from '@adobe/spacecat-shared-utils';
 import { AzureOpenAIClient } from '@adobe/spacecat-shared-gpt-client';
 import { Audit, Suggestion as SuggestionDataAccess } from '@adobe/spacecat-shared-data-access';
-import { getScrapedDataForSiteId } from '../support/utils.js';
+import { getScrapedDataForSiteId, limitConcurrency } from '../support/utils.js';
 import { syncSuggestions } from '../utils/data-access.js';
 import { filterByAuditScope, extractPathPrefix } from './subpath-filter.js';
 
@@ -25,6 +25,7 @@ export const generateSuggestionData = async (finalUrl, brokenInternalLinks, cont
   const azureOpenAIClient = AzureOpenAIClient.createFrom(context);
   const azureOpenAIOptions = { responseFormat: 'json_object' };
   const BATCH_SIZE = 300;
+  const MAX_CONCURRENT_AI_CALLS = 5;
 
   // Ensure brokenInternalLinks is an array
   if (!Array.isArray(brokenInternalLinks)) {
@@ -205,19 +206,20 @@ export const generateSuggestionData = async (finalUrl, brokenInternalLinks, cont
     }
   }
 
-  const updatedInternalLinks = [];
-  for (let index = 0; index < brokenLinksWithFilteredData.length; index += 1) {
-    const link = brokenLinksWithFilteredData[index];
-    const headerSuggestions = headerSuggestionsResults[index];
-    // eslint-disable-next-line no-await-in-loop
-    const updatedLink = await processLink(link, headerSuggestions);
-    // Remove filtered data before returning (not needed in final result)
-    const cleanLink = { ...updatedLink };
-    delete cleanLink.filteredSiteData;
-    delete cleanLink.filteredHeaderLinks;
-    updatedInternalLinks.push(cleanLink);
-  }
-  return updatedInternalLinks;
+  return limitConcurrency(
+    brokenLinksWithFilteredData.map(
+      (link, index) => async () => {
+        const headerSuggestions = headerSuggestionsResults[index];
+        const updatedLink = await processLink(link, headerSuggestions);
+        // Remove filtered data before returning (not needed in final result)
+        const cleanLink = { ...updatedLink };
+        delete cleanLink.filteredSiteData;
+        delete cleanLink.filteredHeaderLinks;
+        return cleanLink;
+      },
+    ),
+    MAX_CONCURRENT_AI_CALLS,
+  );
 };
 
 export async function syncBrokenInternalLinksSuggestions({

src/paid-traffic-analysis/cache-warmer.js

@@ -19,6 +19,7 @@ import {
 } from '@adobe/spacecat-shared-athena-client';
 import crypto from 'crypto';
 import { fileExists, addResultJsonToCache } from './caching-helper.js';
+import { limitConcurrency } from '../support/utils.js';
 
 const QUERIES = [
   { dimensions: ['utm_campaign', 'path', 'device'], mapper: TrafficDataWithCWVDto },
@@ -80,28 +81,6 @@ function getConfig(env) {
   };
 }
 
-async function limitConcurrency(tasks, maxConcurrent) {
-  const results = [];
-  const executing = [];
-
-  for (const task of tasks) {
-    const promise = task().then((result) => {
-      executing.splice(executing.indexOf(promise), 1);
-      return result;
-    });
-
-    results.push(promise);
-    executing.push(promise);
-
-    if (executing.length >= maxConcurrent) {
-      // eslint-disable-next-line no-await-in-loop
-      await Promise.race(executing);
-    }
-  }
-
-  return Promise.all(results);
-}
-
 async function checkCacheExists(
   context,
   log,

src/preflight/links.js

@@ -17,6 +17,31 @@ import { generateSuggestionData } from '../internal-links/suggestions-generator.
 
 export const PREFLIGHT_LINKS = 'links';
 
+/**
+ * Create an issue object for a broken internal link with AI suggestions
+ * @param {string} urlTo - The URL that is broken
+ * @param {number} status - HTTP status code
+ * @param {string} baseURLOrigin - Base URL origin to replace preview origin
+ * @param {Array} urlsSuggested - Optional array of suggested alternative URLs from AI
+ * @param {string} aiRationale - Optional AI rationale for suggestions
+ * @returns {Object} Issue object with all fields including aiSuggestion
+ */
+export function createBrokenLinkIssue(urlTo, status, baseURLOrigin, urlsSuggested, aiRationale) {
+  const aiUrls = (urlsSuggested && urlsSuggested.length > 0)
+    ? urlsSuggested.map((url) => stripTrailingSlash(
+      url.replace(new URL(url).origin, baseURLOrigin),
+    )) : [];
+
+  return {
+    url: stripTrailingSlash(urlTo.replace(new URL(urlTo).origin, baseURLOrigin)),
+    issue: `Status ${status}`,
+    seoImpact: 'High',
+    seoRecommendation: 'Fix or remove broken links to improve user experience and SEO',
+    aiSuggestion: aiUrls.length > 0 ? aiUrls[0] : undefined,
+    aiRationale,
+  };
+}
+
 export default async function links(context, auditContext) {
   const {
     site, job, log,
@@ -83,17 +108,14 @@ export default async function links(context, auditContext) {
         if (!brokenInternalLinksByPage.has(href)) {
           brokenInternalLinksByPage.set(href, []);
         }
-        const aiUrls = urlsSuggested?.map((url) => stripTrailingSlash(
-          url.replace(new URL(url).origin, baseURLOrigin),
-        ));
-        brokenInternalLinksByPage.get(href).push({
-          url: stripTrailingSlash(urlTo.replace(new URL(urlTo).origin, baseURLOrigin)),
-          issue: `Status ${status}`,
-          seoImpact: 'High',
-          seoRecommendation: 'Fix or remove broken links to improve user experience and SEO',
-          aiSuggestion: aiUrls[0],
+        const issue = createBrokenLinkIssue(
+          urlTo,
+          status,
+          baseURLOrigin,
+          urlsSuggested,
           aiRationale,
-        });
+        );
+        brokenInternalLinksByPage.get(href).push(issue);
       });
     } else {
       auditResult.brokenInternalLinks.forEach(({ urlTo, href, status }) => {

src/support/utils.js

@@ -196,6 +196,34 @@ export function getUrlWithoutPath(url) {
   return `${urlObj.protocol}//${urlObj.host}`;
 }
 
+/**
+ * Limits the concurrency of async tasks
+ * @param {Array<Function>} tasks - Array of async functions to execute
+ * @param {number} maxConcurrent - Maximum number of concurrent tasks
+ * @returns {Promise<Array>} - Array of results from all tasks
+ */
+export async function limitConcurrency(tasks, maxConcurrent) {
+  const results = [];
+  const executing = [];
+
+  for (const task of tasks) {
+    const promise = task().then((result) => {
+      executing.splice(executing.indexOf(promise), 1);
+      return result;
+    });
+
+    results.push(promise);
+    executing.push(promise);
+
+    if (executing.length >= maxConcurrent) {
+      // eslint-disable-next-line no-await-in-loop
+      await Promise.race(executing);
+    }
+  }
+
+  return Promise.all(results);
+}
+
 const extractScrapedMetadataFromJson = (data, log) => {
   try {
     log.debug(`Extracting data from JSON (${data.finalUrl}:`, JSON.stringify(data.scrapeResult.tags));
@@ -296,6 +324,7 @@ export async function calculateCPCValue(context, siteId) {
 export const getScrapedDataForSiteId = async (site, context) => {
   const { s3Client, env, log } = context;
   const siteId = site.getId();
+  const MAX_CONCURRENT_S3_READS = 10;
 
   let allFiles = [];
   let isTruncated = true;
@@ -323,16 +352,16 @@ export const getScrapedDataForSiteId = async (site, context) => {
   }
 
   async function fetchContentOfFiles(files) {
-    return Promise.all(
-      files.map(async (file) => {
-        const fileContent = await getObjectFromKey(
+    return limitConcurrency(
+      files.map(
+        (file) => async () => getObjectFromKey(
           s3Client,
           env.S3_SCRAPER_BUCKET_NAME,
           file.Key,
           log,
-        );
-        return fileContent;
-      }),
+        ),
+      ),
+      MAX_CONCURRENT_S3_READS,
     );
   }
 
@@ -347,16 +376,19 @@ export const getScrapedDataForSiteId = async (site, context) => {
     };
   }
 
-  const extractedData = await Promise.all(
-    allFiles.map(async (file) => {
-      const fileContent = await getObjectFromKey(
-        s3Client,
-        env.S3_SCRAPER_BUCKET_NAME,
-        file.Key,
-        log,
-      );
-      return extractScrapedMetadataFromJson(fileContent, log);
-    }),
+  const extractedData = await limitConcurrency(
+    allFiles.map(
+      (file) => async () => {
+        const fileContent = await getObjectFromKey(
+          s3Client,
+          env.S3_SCRAPER_BUCKET_NAME,
+          file.Key,
+          log,
+        );
+        return extractScrapedMetadataFromJson(fileContent, log);
+      },
+    ),
+    MAX_CONCURRENT_S3_READS,
   );
 
   const indexFile = allFiles