Merge pull request #931 from Adeyemi-cmd/Add_data_retention

feat(data-retention): add automated pruning for audit logs and analyt…

Author: RUKAYAT-CODER

.env.example

@@ -474,6 +474,12 @@ IDEMPOTENCY_TTL_SECONDS=86400
 # Segment write key (for analytics, optional)
 SEGMENT_WRITE_KEY=
 
+# Audit log retention period in days (default: 730 = 2 years)
+AUDIT_LOG_RETENTION_DAYS=730
+
+# Analytics event retention period in days (default: 365 = 1 year)
+ANALYTICS_RETENTION_DAYS=365
+
 # ─────────────────────────────────────────────────────────────────────────────
 # 22. CDN CONFIGURATION
 # ─────────────────────────────────────────────────────────────────────────────

src/analytics/analytics.module.ts

@@ -11,6 +11,7 @@ import { AnalyticsEvent } from './entities/event.entity';
 import { EventBatchingService } from './services/event-batching.service';
 import { EventValidationService } from './services/event-validation.service';
 import { EventTrackingSDK } from './sdk/event-tracking.sdk';
+import { AnalyticsRetentionTask } from './tasks/analytics-retention.task';
 
 @Module({
   imports: [
@@ -24,6 +25,7 @@ import { EventTrackingSDK } from './sdk/event-tracking.sdk';
     EventBatchingService,
     EventValidationService,
     EventTrackingSDK,
+    AnalyticsRetentionTask,
     { provide: APP_INTERCEPTOR, useClass: FingerprintInterceptor },
   ],
   controllers: [AnalyticsController],

src/analytics/tasks/analytics-retention.task.ts

@@ -0,0 +1,75 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { Cron } from '@nestjs/schedule';
+import { InjectRepository } from '@nestjs/typeorm';
+import { Repository, LessThan } from 'typeorm';
+import { Counter } from 'prom-client';
+import { AnalyticsEvent } from '../entities/event.entity';
+import { MetricsCollectionService } from '../../monitoring/metrics/metrics-collection.service';
+import { ConfigService } from '@nestjs/config';
+
+@Injectable()
+export class AnalyticsRetentionTask {
+  private readonly logger = new Logger(AnalyticsRetentionTask.name);
+  private readonly retentionDays: number;
+  private readonly batchSize = 1000;
+  private deletedCounter: Counter<'table'>;
+
+  constructor(
+    @InjectRepository(AnalyticsEvent)
+    private readonly eventRepository: Repository<AnalyticsEvent>,
+    private readonly configService: ConfigService,
+    private readonly metrics: MetricsCollectionService,
+  ) {
+    this.retentionDays = this.configService.get<number>('ANALYTICS_RETENTION_DAYS', 365);
+    const registry = this.metrics.getRegistry();
+    this.deletedCounter =
+      (registry.getSingleMetric('deleted_count') as Counter<'table'>) ??
+      new Counter({
+        name: 'deleted_count',
+        help: 'Number of rows deleted by data retention policies',
+        labelNames: ['table'] as const,
+        registers: [registry],
+      });
+  }
+
+  @Cron('30 2 * * *')
+  async handleDailyRetention(): Promise<void> {
+    this.logger.log('Starting daily analytics event retention policy...');
+    let totalDeleted = 0;
+    try {
+      const cutoff = new Date();
+      cutoff.setDate(cutoff.getDate() - this.retentionDays);
+
+      let deleted = 0;
+      do {
+        const eventsToDelete = await this.eventRepository.find({
+          select: ['id'],
+          where: { timestamp: LessThan(cutoff) },
+          take: this.batchSize,
+        });
+
+        if (eventsToDelete.length === 0) {
+          deleted = 0;
+          break;
+        }
+
+        const idValues = eventsToDelete.map((e) => e.id);
+        const result = await this.eventRepository
+          .createQueryBuilder()
+          .delete()
+          .from(AnalyticsEvent)
+          .whereInIds(idValues)
+          .execute();
+        deleted = result.affected || 0;
+        totalDeleted += deleted;
+      } while (deleted >= this.batchSize);
+
+      this.deletedCounter.inc({ table: 'analytics_events' }, totalDeleted);
+      this.logger.log(
+        `Daily analytics retention policy completed. Deleted ${totalDeleted} old events.`,
+      );
+    } catch (error) {
+      this.logger.error('Failed to apply analytics retention policy:', error);
+    }
+  }
+}

src/audit-log/audit-log.module.ts

@@ -7,6 +7,7 @@ import { AuditQueryService } from './services/audit-query.service';
 import { AuditReportingService } from './services/audit-reporting.service';
 import { AuditExportService } from './services/audit-export.service';
 import { AuditRetentionTask } from './tasks/audit-retention.task';
+import { MetricsCollectionService } from '../monitoring/metrics/metrics-collection.service';
 
 /**
  * Audit Log Module
@@ -26,6 +27,7 @@ import { AuditRetentionTask } from './tasks/audit-retention.task';
     AuditExportService,
     AuditRetentionTask,
     AuditLogService,
+    MetricsCollectionService,
   ],
   exports: [AuditLogService],
 })

src/audit-log/tasks/audit-retention.task.ts

@@ -1,31 +1,79 @@
 import { Injectable, Logger } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
+import { InjectRepository } from '@nestjs/typeorm';
+import { Repository, LessThan } from 'typeorm';
+import { Counter } from 'prom-client';
+import { AuditLog } from '../audit-log.entity';
 import { AuditLogService } from '../audit-log.service';
+import { MetricsCollectionService } from '../../monitoring/metrics/metrics-collection.service';
+import { ConfigService } from '@nestjs/config';
 
-/**
- * Provides audit Retention Task behavior.
- */
 @Injectable()
 export class AuditRetentionTask {
   private readonly logger = new Logger(AuditRetentionTask.name);
-  constructor(private readonly auditLogService: AuditLogService) {}
-  /**
-   * Run retention policy daily at 2 AM
-   */
+  private readonly retentionDays: number;
+  private readonly batchSize = 1000;
+  private deletedCounter: Counter<'table'>;
+
+  constructor(
+    @InjectRepository(AuditLog)
+    private readonly auditLogRepo: Repository<AuditLog>,
+    private readonly auditLogService: AuditLogService,
+    private readonly configService: ConfigService,
+    private readonly metrics: MetricsCollectionService,
+  ) {
+    this.retentionDays = this.configService.get<number>('AUDIT_LOG_RETENTION_DAYS', 730);
+    const registry = this.metrics.getRegistry();
+    this.deletedCounter =
+      (registry.getSingleMetric('deleted_count') as Counter<'table'>) ??
+      new Counter({
+        name: 'deleted_count',
+        help: 'Number of rows deleted by data retention policies',
+        labelNames: ['table'] as const,
+        registers: [registry],
+      });
+  }
+
   @Cron(CronExpression.EVERY_DAY_AT_2AM)
   async handleDailyRetention(): Promise<void> {
     this.logger.log('Starting daily audit log retention policy...');
+    let totalDeleted = 0;
     try {
-      const deletedCount = await this.auditLogService.applyRetentionPolicy();
-      this.logger.log(`Daily retention policy completed. Deleted ${deletedCount} old audit logs.`);
+      const cutoff = new Date();
+      cutoff.setDate(cutoff.getDate() - this.retentionDays);
+
+      let deleted = 0;
+      do {
+        const logsToDelete = await this.auditLogRepo.find({
+          select: ['id'],
+          where: { timestamp: LessThan(cutoff) },
+          take: this.batchSize,
+        });
+
+        if (logsToDelete.length === 0) {
+          deleted = 0;
+          break;
+        }
+
+        const idValues = logsToDelete.map((l) => l.id);
+        const result = await this.auditLogRepo
+          .createQueryBuilder()
+          .delete()
+          .from(AuditLog)
+          .whereInIds(idValues)
+          .execute();
+        deleted = result.affected || 0;
+        totalDeleted += deleted;
+      } while (deleted >= this.batchSize);
+
+      this.deletedCounter.inc({ table: 'audit_logs' }, totalDeleted);
+      this.logger.log(`Daily retention policy completed. Deleted ${totalDeleted} old audit logs.`);
     } catch (error) {
       this.logger.error('Failed to apply retention policy:', error);
     }
   }
-  /**
-   * Generate weekly report every Monday at 3 AM
-   */
-  @Cron('0 3 * * 1') // Every Monday at 3 AM
+
+  @Cron('0 3 * * 1')
   async handleWeeklyReport(): Promise<void> {
     this.logger.log('Generating weekly audit report...');
     try {
@@ -38,8 +86,6 @@ export class AuditRetentionTask {
         criticalEvents: report.eventsBySeverity['CRITICAL'] || 0,
         errorEvents: report.eventsBySeverity['ERROR'] || 0,
       });
-      // In a real implementation, you might send this report via email
-      // or store it for compliance purposes
     } catch (error) {
       this.logger.error('Failed to generate weekly report:', error);
     }

src/config/env.validation.ts

@@ -158,6 +158,10 @@ export const envValidationSchema = Joi.object({
   // Segment Analytics
   SEGMENT_WRITE_KEY: Joi.string().optional(),
 
+  // Data Retention
+  AUDIT_LOG_RETENTION_DAYS: Joi.number().integer().min(1).default(730),
+  ANALYTICS_RETENTION_DAYS: Joi.number().integer().min(1).default(365),
+
   // Circuit Breaker Configuration
   CIRCUIT_BREAKER_TIMEOUT_MS: Joi.number().integer().min(100).default(3000),
   CIRCUIT_BREAKER_ERROR_THRESHOLD: Joi.number().integer().min(1).max(100).default(50),

src/config/retention.config.ts

@@ -17,6 +17,11 @@ export const retentionConfig = registerAs('retention', () => ({
    */
   notificationRetentionDays: parseInt(process.env.RETENTION_NOTIFICATION_DAYS || '30', 10),
 
+  /**
+   * Retention period for analytics events in days.
+   */
+  analyticsRetentionDays: parseInt(process.env.ANALYTICS_RETENTION_DAYS || '365', 10),
+
   /**
    * Whether to archive data before purging.
    */