feat: add audit for broken content fragment requests

src/cdn-content-fragment-404/handler.js

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { getStaticContent } from '@adobe/spacecat-shared-utils';
+import { AWSAthenaClient } from '@adobe/spacecat-shared-athena-client';
+import { AuditBuilder } from '../common/audit-builder.js';
+import { wwwUrlResolver } from '../common/base-audit.js';
+import { getImsOrgId } from '../utils/data-access.js';
+import { extractCustomerDomain } from '../utils/cdn-utils.js';
+
+const ONE_HOUR_MS = 60 * 60 * 1000;
+
+function getHourParts() {
+  const previousHour = new Date(Date.now() - ONE_HOUR_MS);
+
+  const year = previousHour.getUTCFullYear().toString();
+  const month = String(previousHour.getUTCMonth() + 1).padStart(2, '0');
+  const day = String(previousHour.getUTCDate()).padStart(2, '0');
+  const hour = String(previousHour.getUTCHours()).padStart(2, '0');
+
+  return {
+    year, month, day, hour,
+  };
+}
+
+async function loadSql(filename, variables) {
+  return getStaticContent(variables, `./src/cdn-content-fragment-404/sql/${filename}.sql`);
+}
+
+export async function cdnContentFragment404Runner(context) {
+  const {
+    site, rawBucket, dataAccess, log,
+  } = context;
+  const sanitizedHostname = extractCustomerDomain(site);
+  const {
+    year, month, day, hour,
+  } = getHourParts();
+
+  if (!rawBucket) {
+    throw new Error('Raw bucket is required');
+  }
+
+  const imsOrg = await getImsOrgId(site, dataAccess, log);
+  if (!imsOrg) {
+    throw new Error('Unable to retrieve IMS organization ID');
+  }
+
+  const database = `cdn_logs_${sanitizedHostname}`;
+  const rawTable = `raw_logs_status_${sanitizedHostname}`;
+  const tempLocation = `s3://${rawBucket}/temp/athena-results/`;
+  const athenaClient = AWSAthenaClient.fromContext(context, tempLocation);
+
+  // Create database
+  const sqlDb = await loadSql('create-database', { database });
+  const sqlDbDescription = `[Athena Query] Create database ${database}`;
+  await athenaClient.execute(sqlDb, database, sqlDbDescription);
+
+  // Each tenant has its own folder mapped via IMS org within the raw bucket
+  const bucket = `${rawBucket}/${imsOrg}`;
+  // Subfolder aem-cs-fastly is used for raw logs currently
+  const rawLocation = `s3://${bucket}/raw/aem-cs-fastly`;
+
+  // Create table
+  const sqlTable = await loadSql('create-raw-table', {
+    database,
+    rawTable,
+    rawLocation,
+  });
+  const sqlTableDescription = `[Athena Query] Create raw logs table ${database}.${rawTable} from ${rawLocation}`;
+  await athenaClient.execute(sqlTable, database, sqlTableDescription);
+
+  const output = `s3://${bucket}/aggregated-404/${year}/${month}/${day}/${hour}/`;
+  const sqlUnload = await loadSql('unload-404-content', {
+    database,
+    rawTable,
+    year,
+    month,
+    day,
+    hour,
+    output,
+  });
+  const sqlUnloadDescription = `[Athena Query] Unload 404 content data to ${output}`;
+  await athenaClient.execute(sqlUnload, database, sqlUnloadDescription);
+
+  return {
+    auditResult: {
+      database,
+      rawTable,
+      completedAt: new Date().toISOString(),
+    },
+    fullAuditRef: output,
+  };
+}
+
+export default new AuditBuilder()
+  .withUrlResolver(wwwUrlResolver)
+  .withRunner(cdnContentFragment404Runner)
+  .build();

src/cdn-content-fragment-404/sql/create-database.sql

@@ -0,0 +1 @@
+CREATE DATABASE IF NOT EXISTS {{database}};

src/cdn-content-fragment-404/sql/create-raw-table.sql

@@ -0,0 +1,29 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS {{database}}.{{rawTable}} (
+  url                   string,
+  request_user_agent    string,
+  response_status       int
+)
+PARTITIONED BY (
+  year  string,
+  month string,
+  day   string,
+  hour  string
+)
+ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
+LOCATION '{{rawLocation}}'
+TBLPROPERTIES (
+  'projection.enabled'        = 'true',
+  'storage.location.template' = '{{rawLocation}}/${year}/${month}/${day}/${hour}/',
+  'projection.year.type'      = 'integer',
+  'projection.year.range'     = '2024,2030',
+  'projection.month.type'     = 'integer',
+  'projection.month.range'    = '1,12',
+  'projection.month.digits'   = '2',
+  'projection.day.type'       = 'integer',
+  'projection.day.range'      = '1,31',
+  'projection.day.digits'     = '2',
+  'projection.hour.type'      = 'integer',
+  'projection.hour.range'     = '0,23',
+  'projection.hour.digits'    = '2',
+  'has_encrypted_data'        = 'false'
+);

src/cdn-content-fragment-404/sql/unload-404-content.sql

@@ -0,0 +1,19 @@
+UNLOAD (
+  SELECT
+    url,
+    request_user_agent,
+    COALESCE(REGEXP_EXTRACT(url, '/content/dam/([^/]+)', 1), 'unknown') AS tenant,
+    count(*) AS request_count
+  FROM {{database}}.{{rawTable}}
+  WHERE year  = '{{year}}'
+    AND month = '{{month}}'
+    AND day   = '{{day}}'
+    AND hour  = '{{hour}}'
+
+    AND response_status = 404
+    -- Only include content fragment requests
+    AND url LIKE '/content/dam/%'
+
+  GROUP BY url, request_user_agent, COALESCE(REGEXP_EXTRACT(url, '/content/dam/([^/]+)', 1), 'unknown')
+) TO '{{output}}'
+WITH (format = 'PARQUET');

src/content-fragment-404/analysis/analysis-strategy.js

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { PublishRule } from '../rules/publish-rule.js';
+import { LocaleFallbackRule } from '../rules/locale-fallback-rule.js';
+import { SimilarPathRule } from '../rules/similar-path-rule.js';
+import { Suggestion, SuggestionType } from '../domain/suggestion/suggestion.js';
+
+export class AnalysisStrategy {
+  constructor(context, aemClient, pathIndex) {
+    this.context = context;
+    this.aemClient = aemClient;
+    this.pathIndex = pathIndex;
+    this.rules = [
+      new PublishRule(context, this.aemClient),
+      new LocaleFallbackRule(context, this.aemClient),
+      new SimilarPathRule(context, this.aemClient, pathIndex),
+    ].sort((a, b) => a.getPriority() - b.getPriority());
+  }
+
+  async analyze(contentFragment404s) {
+    const suggestions = [];
+
+    for (const path of contentFragment404s) {
+      // eslint-disable-next-line no-await-in-loop
+      const suggestion = await this.analyzePath(path);
+      if (suggestion) {
+        suggestions.push(suggestion);
+      }
+    }
+
+    // Post-process suggestions to check content status
+    return this.processSuggestions(suggestions);
+  }
+
+  async analyzePath(brokenPath) {
+    const { log } = this.context;
+    log.info(`Analyzing broken path: ${brokenPath}`);
+
+    for (const rule of this.rules) {
+      try {
+        // eslint-disable-next-line no-await-in-loop
+        const suggestion = await rule.apply(brokenPath);
+
+        if (suggestion) {
+          log.info(`Rule ${rule.constructor.name} applied to ${brokenPath}`);
+          return suggestion;
+        }
+      } catch (error) {
+        log.error(`Error applying rule ${rule.constructor.name} to ${brokenPath}: ${error.message}`);
+        // Continue to next rule
+      }
+    }
+
+    log.warn(`No rules applied to ${brokenPath}`);
+    return Suggestion.notFound(brokenPath);
+  }
+
+  async processSuggestions(suggestions) {
+    const { log } = this.context;
+    log.info(`Post-processing ${suggestions.length} suggestions`);
+
+    const processedSuggestions = [];
+
+    for (const suggestion of suggestions) {
+      if (suggestion.type !== SuggestionType.LOCALE && suggestion.type !== SuggestionType.SIMILAR) {
+        processedSuggestions.push(suggestion);
+        // eslint-disable-next-line no-continue
+        continue;
+      }
+
+      const { suggestedPath } = suggestion;
+      log.debug(`Checking content status for suggestion: ${suggestedPath} with type: ${suggestion.type}`);
+
+      // Path must be available as it was suggested
+      const contentPath = this.pathIndex.find(suggestedPath);
+      const { status } = contentPath;
+
+      if (contentPath.isPublished()) {
+        processedSuggestions.push(suggestion);
+        log.debug(`Kept original suggestion type for ${suggestedPath} with status: ${status}`);
+        // eslint-disable-next-line no-continue
+        continue;
+      }
+
+      suggestion.reason = `Content is in ${status} state. Suggest publishing.`;
+      processedSuggestions.push(suggestion);
+    }
+
+    return processedSuggestions;
+  }
+}

src/content-fragment-404/cache/cache-strategy.js

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+export class CacheStrategy {
+  /**
+   * Find direct children of a parent path
+   * @param {string} parentPath - The parent path
+   * @returns {Array<ContentPath>} Array of child ContentPath objects
+   */
+  // eslint-disable-next-line no-unused-vars, class-methods-use-this
+  findChildren(parentPath) {
+    throw new Error('findChildren() must be implemented by subclass');
+  }
+
+  /**
+   * Cache content items.
+   * @param {Array} items - Array of content items
+   * @param {Function} statusParser - Function to parse content status
+   * @returns {void}
+   */
+  // eslint-disable-next-line no-unused-vars, class-methods-use-this
+  cacheItems(items, statusParser) {
+    throw new Error('cacheItems() must be implemented by subclass');
+  }
+
+  /**
+   * Check if this cache strategy is available.
+   * @returns {boolean}
+   */
+  // eslint-disable-next-line class-methods-use-this
+  isAvailable() {
+    throw new Error('isAvailable() must be implemented by subclass');
+  }
+}

src/content-fragment-404/cache/noop-cache.js

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { CacheStrategy } from './cache-strategy.js';
+
+/**
+ * No-operation cache that doesn't store anything.
+ */
+export class NoOpCache extends CacheStrategy {
+  // eslint-disable-next-line class-methods-use-this
+  findChildren() {
+    return [];
+  }
+
+  // eslint-disable-next-line no-unused-vars, class-methods-use-this
+  cacheItems(items, statusParser) {
+    // Do nothing
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  isAvailable() {
+    return false;
+  }
+}

src/content-fragment-404/cache/path-index-cache.js

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { ContentPath } from '../domain/content/content-path.js';
+import { Locale } from '../domain/language/locale.js';
+import { CacheStrategy } from './cache-strategy.js';
+
+/**
+ * Cache implementation that uses PathIndex for storage.
+ * Provides hierarchical path caching and lookup capabilities.
+ */
+export class PathIndexCache extends CacheStrategy {
+  constructor(pathIndex) {
+    super();
+    this.pathIndex = pathIndex;
+  }
+
+  findChildren(parentPath) {
+    return this.pathIndex.findChildren(parentPath);
+  }
+
+  cacheItems(items, statusParser) {
+    if (!items || items.length === 0) {
+      return;
+    }
+
+    for (const item of items) {
+      const contentPath = new ContentPath(
+        item.path,
+        statusParser(item.status),
+        Locale.fromPath(item.path),
+      );
+      this.pathIndex.insertContentPath(contentPath);
+    }
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  isAvailable() {
+    return true;
+  }
+}