From 8391dd184688a176c37eaa0ba99cb8c5f2cfbc63 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Wed, 12 Nov 2025 14:37:52 -0700 Subject: [PATCH 01/28] add traffic calculator --- .../optout/vertx/OptOutTrafficCalculator.java | 458 ++++++++++++++++++ 1 file changed, 458 insertions(+) create mode 100644 src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java new file mode 100644 index 00000000..46ef8943 --- /dev/null +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -0,0 +1,458 @@ +package com.uid2.optout.vertx; + +import com.uid2.shared.cloud.ICloudStorage; +import com.uid2.shared.optout.OptOutCollection; +import com.uid2.shared.optout.OptOutEntry; +import com.uid2.shared.optout.OptOutUtils; +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.sqs.model.Message; +import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; + +import java.nio.charset.StandardCharsets; + +import java.io.InputStream; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * Calculates opt-out traffic patterns to determine DEFAULT or DELAYED_PROCESSING status. + * + * Compares recent ~24h traffic (sumCurrent) against previous ~24h baseline (sumPast). + * Both sums exclude records in whitelist ranges (surge windows determined by engineers). + * + * Returns DELAYED_PROCESSING if sumCurrent >= 5 × sumPast, indicating abnormal traffic spike. + */ +public class OptOutTrafficCalculator { + private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficCalculator.class); + + private static final int HOURS_24 = 24 * 3600; // 24 hours in seconds + private static final int DEFAULT_THRESHOLD_MULTIPLIER = 5; + + private final Map deltaFileCache = new ConcurrentHashMap<>(); + private final int thresholdMultiplier; + private final ICloudStorage cloudStorage; + private final String s3DeltaPrefix; // (e.g. "optout-v2/delta/") + private final String whitelistS3Path; // (e.g. "optout-breaker/traffic-filter-config.json") + private List> whitelistRanges; + + public enum TrafficStatus { + DELAYED_PROCESSING, + DEFAULT + } + + /** + * Cache entry for a delta file containing all record timestamps. + * + * Memory usage: ~8 bytes per timestamp (long) + * 1GB of memory can store ~130 million timestamps (1024^3)/8 + */ + private static class FileRecordCache { + final List timestamps; // All non-sentinel record timestamps + final long newestTimestamp; // evict delta from cache based on oldest record timestamp + + FileRecordCache(List timestamps) { + this.timestamps = timestamps; + this.newestTimestamp = timestamps.isEmpty() ? 0 : Collections.max(timestamps); + } + } + + /** + * Constructor for OptOutTrafficCalculator + * + * @param config JsonObject containing configuration + * @param cloudStorage Cloud storage for reading delta files and whitelist from S3 + * @param cloudSync Cloud sync for path conversion + */ + public OptOutTrafficCalculator(JsonObject config, ICloudStorage cloudStorage, String s3DeltaPrefix, String whitelistS3Path) { + this.cloudStorage = cloudStorage; + this.thresholdMultiplier = config.getInteger("traffic_calc_threshold_multiplier", DEFAULT_THRESHOLD_MULTIPLIER); + this.s3DeltaPrefix = s3DeltaPrefix; + this.whitelistS3Path = whitelistS3Path; + + // Initial whitelist load + this.whitelistRanges = Collections.emptyList(); // Start empty + reloadWhitelist(); // Load from S3 + + LOGGER.info("OptOutTrafficCalculator initialized: s3DeltaPrefix={}, whitelistPath={}, threshold={}x", + s3DeltaPrefix, whitelistS3Path, thresholdMultiplier); + } + + /** + * Reload whitelist ranges from S3. + * Expected format: + * { + * "traffic_calc_whitelist_ranges": [ + * [startTimestamp1, endTimestamp1], + * [startTimestamp2, endTimestamp2] + * ] + * } + * + * Can be called periodically to pick up config changes without restarting. + */ + public void reloadWhitelist() { + LOGGER.info("Reloading whitelist from S3: {}", whitelistS3Path); + try (InputStream is = cloudStorage.download(whitelistS3Path)) { + String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); + JsonObject whitelistConfig = new JsonObject(content); + + List> ranges = parseWhitelistRanges(whitelistConfig); + this.whitelistRanges = ranges; + + LOGGER.info("Successfully loaded {} whitelist ranges from S3", ranges.size()); + + } catch (Exception e) { + LOGGER.warn("No whitelist found at: {}", whitelistS3Path, e); + this.whitelistRanges = Collections.emptyList(); + } + } + + /** + * Parse whitelist ranges from JSON config + */ + private List> parseWhitelistRanges(JsonObject config) { + List> ranges = new ArrayList<>(); + + try { + if (config.containsKey("traffic_calc_whitelist_ranges")) { + var rangesArray = config.getJsonArray("traffic_calc_whitelist_ranges"); + if (rangesArray != null) { + for (int i = 0; i < rangesArray.size(); i++) { + var rangeArray = rangesArray.getJsonArray(i); + if (rangeArray != null && rangeArray.size() >= 2) { + long val1 = rangeArray.getLong(0); + long val2 = rangeArray.getLong(1); + + // Ensure start <= end (correct misordered ranges) + long start = Math.min(val1, val2); + long end = Math.max(val1, val2); + + List range = Arrays.asList(start, end); + ranges.add(range); + LOGGER.info("Loaded whitelist range: [{}, {}]", start, end); + } + } + } + } + + // Sort ranges by start time for efficient lookups + ranges.sort(Comparator.comparing(range -> range.get(0))); + + } catch (Exception e) { + LOGGER.error("Failed to parse whitelist ranges", e); + } + + return ranges; + } + + /** + * Calculate traffic status based on delta files and SQS queue messages. + * + * @param sqsMessages List of SQS messages + * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) + */ + public TrafficStatus calculateStatus(List sqsMessages) { + + try { + // Get list of delta files from S3 (sorted newest to oldest) + List deltaS3Paths = listDeltaFiles(); + + if (deltaS3Paths.isEmpty()) { + LOGGER.warn("No delta files found in S3 with prefix: {}", s3DeltaPrefix); + return TrafficStatus.DEFAULT; + } + + // Find t = newest optout record timestamp from newest delta file + long t = findNewestTimestamp(deltaS3Paths); + LOGGER.info("Traffic calculation starting with t={} (newest optout)", t); + + // Define time windows + long currentWindowStart = t - (HOURS_24-300) - getTotalWhitelistDuration(); // for range [t-23h55m, t+5m] + long pastWindowStart = currentWindowStart - HOURS_24 - getTotalWhitelistDuration(); // for range [t-47h55m, t-23h55m] + + // Evict old cache entries (older than past window start) + evictOldCacheEntries(pastWindowStart); + + // Process delta files and count + int sumCurrent = 0; + int sumPast = 0; + + for (String s3Path : deltaS3Paths) { + List timestamps = getTimestampsFromFile(s3Path); + + boolean shouldStop = false; + for (long ts : timestamps) { + // Stop condition: record is older than our 48h window + if (ts < pastWindowStart) { + LOGGER.debug("Stopping delta file processing at timestamp {} (older than t-48h)", ts); + break; + } + + // skip records in whitelisted ranges + if (isInWhitelist(ts, this.whitelistRanges)) { + continue; + } + + // Count for sumCurrent: [t-24h, t] + if (ts >= currentWindowStart && ts <= t) { + sumCurrent++; + } + + // Count for sumPast: [t-48h, t-24h] + if (ts >= pastWindowStart && ts < currentWindowStart) { + sumPast++; + } + } + + if (shouldStop) { + break; + } + } + + // Count SQS messages in [t, t+5m] + if (sqsMessages != null && !sqsMessages.isEmpty()) { + int sqsCount = countSqsMessages( + sqsMessages, t); + sumCurrent += sqsCount; + } + + // Determine status + TrafficStatus status = determineStatus(sumCurrent, sumPast); + + LOGGER.info("Traffic calculation complete: sumCurrent={}, sumPast={}, status={}", + sumCurrent, sumPast, status); + + return status; + + } catch (Exception e) { + LOGGER.error("Error calculating traffic status", e); + return TrafficStatus.DEFAULT; + } + } + + /** + * List all delta files from S3, sorted newest to oldest + */ + private List listDeltaFiles() { + try { + // List all objects with the delta prefix + List allFiles = cloudStorage.list(s3DeltaPrefix); + + // Filter to only .dat delta files and sort newest to oldest + return allFiles.stream() + .filter(OptOutUtils::isDeltaFile) + .sorted(OptOutUtils.DeltaFilenameComparatorDescending) + .collect(Collectors.toList()); + + } catch (Exception e) { + LOGGER.error("Failed to list delta files from S3 with prefix: {}", s3DeltaPrefix, e); + return Collections.emptyList(); + } + } + + /** + * Get timestamps from a delta file (S3 path), using cache if available + */ + private List getTimestampsFromFile(String s3Path) throws IOException { + // Extract filename from S3 path for cache key + String filename = s3Path.substring(s3Path.lastIndexOf('/') + 1); + + // Check cache first + FileRecordCache cached = deltaFileCache.get(filename); + if (cached != null) { + LOGGER.debug("Using cached timestamps for file: {}", filename); + return cached.timestamps; + } + + // Cache miss - download from S3 + LOGGER.debug("Downloading and reading timestamps from S3: {}", s3Path); + List timestamps = readTimestampsFromS3(s3Path); + + // Store in cache + deltaFileCache.put(filename, new FileRecordCache(timestamps)); + + return timestamps; + } + + /** + * Read all non-sentinel record timestamps from a delta file in S3 + */ + private List readTimestampsFromS3(String s3Path) throws IOException { + try (InputStream is = cloudStorage.download(s3Path)) { + byte[] data = is.readAllBytes(); + OptOutCollection collection = new OptOutCollection(data); + + List timestamps = new ArrayList<>(); + for (int i = 0; i < collection.size(); i++) { + OptOutEntry entry = collection.get(i); + + // Skip sentinel entries + if (entry.isSpecialHash()) { + continue; + } + + timestamps.add(entry.timestamp); + } + + return timestamps; + } catch (Exception e) { + LOGGER.error("Failed to read delta file from S3: {}", s3Path, e); + throw new IOException("Failed to read delta file from S3: " + s3Path, e); + } + } + + private long getTotalWhitelistDuration() { + long totalDuration = 0; + for (List range : this.whitelistRanges) { + totalDuration += range.get(1) - range.get(0); + } + return totalDuration; + } + + /** + * Get the newest optout record timestamp from newest delta file + */ + private long findNewestTimestamp(List deltaS3Paths) throws IOException { + long newest = 0; + + if (!deltaS3Paths.isEmpty()) { + List timestamps = getTimestampsFromFile(deltaS3Paths.get(0)); + if (!timestamps.isEmpty()) { + newest = Collections.max(timestamps); + } + } + return newest; + } + + /** + * Extract timestamp from SQS message (from SentTimestamp attribute) + */ + private Long extractTimestampFromMessage(Message msg) { + // Get SentTimestamp attribute (milliseconds) + String sentTimestamp = msg.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); + if (sentTimestamp != null) { + try { + return Long.parseLong(sentTimestamp) / 1000; // Convert ms to seconds + } catch (NumberFormatException e) { + LOGGER.debug("Invalid SentTimestamp: {}", sentTimestamp); + } + } + + // Fallback: use current time + return System.currentTimeMillis() / 1000; + } + + /** + * Count SQS messages from t to t+5 minutes + */ + private int countSqsMessages(List sqsMessages, long t) { + + int count = 0; + + for (Message msg : sqsMessages) { + Long ts = extractTimestampFromMessage(msg); + + if (ts < t || ts > t + 5 * 60) { + continue; + } + + if (isInWhitelist(ts, this.whitelistRanges)) { + continue; + } + count++; + + } + + LOGGER.info("SQS messages: {} in window [t={}, t+5(minutes)={}]", count, t, t + 5 * 60); + return count; + } + + /** + * Check if a timestamp falls within any whitelist range + */ + private boolean isInWhitelist(long timestamp, List> whitelistRanges) { + if (whitelistRanges == null || whitelistRanges.isEmpty()) { + return false; + } + + for (List range : whitelistRanges) { + if (range.size() < 2) { + continue; + } + + long start = range.get(0); + long end = range.get(1); + + if (timestamp >= start && timestamp <= end) { + return true; + } + } + + return false; + } + + /** + * Evict cache entries with data older than the cutoff timestamp + */ + private void evictOldCacheEntries(long cutoffTimestamp) { + int beforeSize = deltaFileCache.size(); + + deltaFileCache.entrySet().removeIf(entry -> + entry.getValue().newestTimestamp < cutoffTimestamp + ); + + int afterSize = deltaFileCache.size(); + if (beforeSize != afterSize) { + LOGGER.info("Evicted {} old cache entries (before={}, after={})", + beforeSize - afterSize, beforeSize, afterSize); + } + } + + /** + * Determine traffic status based on current vs past counts + */ + private TrafficStatus determineStatus(int sumCurrent, int sumPast) { + if (sumPast == 0) { + // Avoid division by zero - if no baseline traffic, return DEFAULT status + LOGGER.warn("sumPast is 0, cannot detect thresholdcrossing. Returning DEFAULT status."); + return TrafficStatus.DEFAULT; + } + + if (sumCurrent >= thresholdMultiplier * sumPast) { + LOGGER.warn("DELAYED_PROCESSING threshold breached: sumCurrent={} >= {}×sumPast={}", + sumCurrent, thresholdMultiplier, sumPast); + return TrafficStatus.DELAYED_PROCESSING; + } + + LOGGER.info("Traffic within normal range: sumCurrent={} < {}×sumPast={}", + sumCurrent, thresholdMultiplier, sumPast); + return TrafficStatus.DEFAULT; + } + + /** + * Get cache statistics for monitoring + */ + public Map getCacheStats() { + Map stats = new HashMap<>(); + stats.put("cached_files", deltaFileCache.size()); + + int totalTimestamps = deltaFileCache.values().stream() + .mapToInt(cache -> cache.timestamps.size()) + .sum(); + stats.put("total_cached_timestamps", totalTimestamps); + + return stats; + } + + /** + * Clear the cache (for testing or manual reset) + */ + public void clearCache() { + int size = deltaFileCache.size(); + deltaFileCache.clear(); + LOGGER.info("Cleared cache ({} entries)", size); + } +} From 2f426ad687a34582dd554c9f9ed807b0fdc55fee Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Thu, 13 Nov 2025 15:56:04 -0700 Subject: [PATCH 02/28] update from review --- .../optout/vertx/OptOutTrafficCalculator.java | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 46ef8943..3cbc060d 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -165,13 +165,13 @@ public TrafficStatus calculateStatus(List sqsMessages) { return TrafficStatus.DEFAULT; } - // Find t = newest optout record timestamp from newest delta file - long t = findNewestTimestamp(deltaS3Paths); - LOGGER.info("Traffic calculation starting with t={} (newest optout)", t); + // Find t = oldest SQS queue message timestamp + long t = findOldestQueueTimestamp(sqsMessages); + LOGGER.info("Traffic calculation starting with t={} (oldest SQS message)", t); // Define time windows - long currentWindowStart = t - (HOURS_24-300) - getTotalWhitelistDuration(); // for range [t-23h55m, t+5m] - long pastWindowStart = currentWindowStart - HOURS_24 - getTotalWhitelistDuration(); // for range [t-47h55m, t-23h55m] + long currentWindowStart = t - (HOURS_24-300) - getWhitelistDuration(t, t - (HOURS_24-300)); // for range [t-23h55m, t+5m] + long pastWindowStart = currentWindowStart - HOURS_24 - getWhitelistDuration(currentWindowStart, currentWindowStart - HOURS_24); // for range [t-47h55m, t-23h55m] // Evict old cache entries (older than past window start) evictOldCacheEntries(pastWindowStart); @@ -304,27 +304,38 @@ private List readTimestampsFromS3(String s3Path) throws IOException { } } - private long getTotalWhitelistDuration() { + private long getWhitelistDuration(long t, long windowStart) { long totalDuration = 0; for (List range : this.whitelistRanges) { - totalDuration += range.get(1) - range.get(0); + long start = range.get(0); + long end = range.get(1); + if (start < windowStart) { + start = windowStart; + } + if (end > t) { + end = t; + } + totalDuration += end - start; } return totalDuration; } /** - * Get the newest optout record timestamp from newest delta file + * Find the oldest SQS queue message timestamp */ - private long findNewestTimestamp(List deltaS3Paths) throws IOException { - long newest = 0; + private long findOldestQueueTimestamp(List sqsMessages) throws IOException { + long oldest = System.currentTimeMillis() / 1000; - if (!deltaS3Paths.isEmpty()) { - List timestamps = getTimestampsFromFile(deltaS3Paths.get(0)); - if (!timestamps.isEmpty()) { - newest = Collections.max(timestamps); + if (sqsMessages != null && !sqsMessages.isEmpty()) { + for (Message msg : sqsMessages) { + Long ts = extractTimestampFromMessage(msg); + if (ts != null && ts < oldest) { + oldest = ts; + } } } - return newest; + + return oldest; } /** From 7d18fbd7b0c9ccb67f86cc55de20210abf21ad0e Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Thu, 13 Nov 2025 17:27:47 -0700 Subject: [PATCH 03/28] add unit tests --- .../optout/vertx/OptOutTrafficCalculator.java | 23 +- .../vertx/OptOutTrafficCalculatorTest.java | 1273 +++++++++++++++++ 2 files changed, 1289 insertions(+), 7 deletions(-) create mode 100644 src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 3cbc060d..a7142b97 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -113,7 +113,7 @@ public void reloadWhitelist() { /** * Parse whitelist ranges from JSON config */ - private List> parseWhitelistRanges(JsonObject config) { + List> parseWhitelistRanges(JsonObject config) { List> ranges = new ArrayList<>(); try { @@ -192,7 +192,7 @@ public TrafficStatus calculateStatus(List sqsMessages) { } // skip records in whitelisted ranges - if (isInWhitelist(ts, this.whitelistRanges)) { + if (isInWhitelist(ts)) { continue; } @@ -304,18 +304,27 @@ private List readTimestampsFromS3(String s3Path) throws IOException { } } - private long getWhitelistDuration(long t, long windowStart) { + /** + * Calculate total duration of whitelist ranges that overlap with the given time window. + */ + long getWhitelistDuration(long t, long windowStart) { long totalDuration = 0; for (List range : this.whitelistRanges) { long start = range.get(0); long end = range.get(1); + + // Clip range to window boundaries if (start < windowStart) { start = windowStart; } if (end > t) { end = t; } - totalDuration += end - start; + + // Only add duration if there's actual overlap (start < end) + if (start < end) { + totalDuration += end - start; + } } return totalDuration; } @@ -370,7 +379,7 @@ private int countSqsMessages(List sqsMessages, long t) { continue; } - if (isInWhitelist(ts, this.whitelistRanges)) { + if (isInWhitelist(ts)) { continue; } count++; @@ -384,7 +393,7 @@ private int countSqsMessages(List sqsMessages, long t) { /** * Check if a timestamp falls within any whitelist range */ - private boolean isInWhitelist(long timestamp, List> whitelistRanges) { + boolean isInWhitelist(long timestamp) { if (whitelistRanges == null || whitelistRanges.isEmpty()) { return false; } @@ -425,7 +434,7 @@ private void evictOldCacheEntries(long cutoffTimestamp) { /** * Determine traffic status based on current vs past counts */ - private TrafficStatus determineStatus(int sumCurrent, int sumPast) { + TrafficStatus determineStatus(int sumCurrent, int sumPast) { if (sumPast == 0) { // Avoid division by zero - if no baseline traffic, return DEFAULT status LOGGER.warn("sumPast is 0, cannot detect thresholdcrossing. Returning DEFAULT status."); diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java new file mode 100644 index 00000000..1078cd23 --- /dev/null +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -0,0 +1,1273 @@ +package com.uid2.optout.vertx; + +import com.uid2.shared.cloud.CloudStorageException; +import com.uid2.shared.cloud.ICloudStorage; +import com.uid2.shared.optout.OptOutCollection; +import com.uid2.shared.optout.OptOutEntry; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import software.amazon.awssdk.services.sqs.model.Message; +import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; + +import java.io.ByteArrayInputStream; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +public class OptOutTrafficCalculatorTest { + + @Mock + private ICloudStorage cloudStorage; + + private JsonObject config; + private static final String S3_DELTA_PREFIX = "optout-v2/delta/"; + private static final String WHITELIST_S3_PATH = "optout-breaker/traffic-filter-config.json"; + private static final int DEFAULT_THRESHOLD = 5; + + @BeforeEach + void setUp() { + config = new JsonObject(); + config.put("traffic_calc_threshold_multiplier", DEFAULT_THRESHOLD); + } + + // ============================================================================ + // SECTION 1: Constructor & Initialization Tests + // ============================================================================ + + @Test + void testConstructor_defaultThreshold() throws Exception { + // Setup - default threshold of 5 + JsonObject configWithoutThreshold = new JsonObject(); + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + configWithoutThreshold, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Assert - DEFAULT when below threshold, DELAYED_PROCESSING when above threshold + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); // 10 < 5*3 + + status = calculator.determineStatus(15, 3); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); // 15 >= 5*3 + } + + @Test + void testConstructor_customThreshold() throws Exception { + // Setup - custom threshold of 10 + config.put("traffic_calc_threshold_multiplier", 10); + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Assert - DEFAULT when below threshold, DELAYED_PROCESSING when above threshold + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(49, 5); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); // 49 < 10*5 + status = calculator.determineStatus(50, 5); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); // 50 >= 10*5 + } + + @Test + void testConstructor_whitelistLoadFailure() throws Exception { + // Setup - whitelist load failure + when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + calculator.reloadWhitelist(); + + // Assert - whitelist should be empty + assertFalse(calculator.isInWhitelist(1000L)); + } + + // ============================================================================ + // SECTION 2: parseWhitelistRanges() + // ============================================================================ + + @Test + void testParseWhitelistRanges_emptyConfig() throws Exception { + // Setup - no config + when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + JsonObject emptyConfig = new JsonObject(); + + // Act + List> ranges = calculator.parseWhitelistRanges(emptyConfig); + + // Assert - empty ranges + assertTrue(ranges.isEmpty()); + } + + @Test + void testParseWhitelistRanges_singleRange() throws Exception { + // Setup - single range + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L).add(2000L)); + configWithRanges.put("traffic_calc_whitelist_ranges", ranges); + + // Act + List> result = calculator.parseWhitelistRanges(configWithRanges); + + // Assert - single range + assertEquals(1, result.size()); + assertEquals(1000L, result.get(0).get(0)); + assertEquals(2000L, result.get(0).get(1)); + } + + @Test + void testParseWhitelistRanges_multipleRanges() throws Exception { + // Setup - multiple ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L).add(2000L)) + .add(new JsonArray().add(3000L).add(4000L)) + .add(new JsonArray().add(5000L).add(6000L)); + configWithRanges.put("traffic_calc_whitelist_ranges", ranges); + + // Act + List> result = calculator.parseWhitelistRanges(configWithRanges); + + // Assert - multiple ranges + assertEquals(3, result.size()); + assertEquals(1000L, result.get(0).get(0)); + assertEquals(3000L, result.get(1).get(0)); + assertEquals(5000L, result.get(2).get(0)); + } + + @Test + void testParseWhitelistRanges_misorderedRange() throws Exception { + // Setup - range with end < start should be corrected + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(2000L).add(1000L)); // End before start + configWithRanges.put("traffic_calc_whitelist_ranges", ranges); + + // Act + List> result = calculator.parseWhitelistRanges(configWithRanges); + + // Assert - should auto-correct to [1000, 2000] + assertEquals(1, result.size()); + assertEquals(1000L, result.get(0).get(0)); + assertEquals(2000L, result.get(0).get(1)); + } + + @Test + void testParseWhitelistRanges_sortsByStartTime() throws Exception { + // Setup - ranges added out of order + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(5000L).add(6000L)) + .add(new JsonArray().add(1000L).add(2000L)) + .add(new JsonArray().add(3000L).add(4000L)); + configWithRanges.put("traffic_calc_whitelist_ranges", ranges); + + // Act + List> result = calculator.parseWhitelistRanges(configWithRanges); + + // Assert - should be sorted by start time + assertEquals(3, result.size()); + assertEquals(1000L, result.get(0).get(0)); + assertEquals(3000L, result.get(1).get(0)); + assertEquals(5000L, result.get(2).get(0)); + } + + @Test + void testParseWhitelistRanges_invalidRangeTooFewElements() throws Exception { + // Setup - invalid range with only 1 element; + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L)) // Only 1 element + .add(new JsonArray().add(2000L).add(3000L)); // Valid + configWithRanges.put("traffic_calc_whitelist_ranges", ranges); + + // Act + List> result = calculator.parseWhitelistRanges(configWithRanges); + + // Assert - should skip invalid range + assertEquals(1, result.size()); + assertEquals(2000L, result.get(0).get(0)); + } + + @Test + void testParseWhitelistRanges_nullArray() throws Exception { + // Setup - null array + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + JsonObject configWithRanges = new JsonObject(); + configWithRanges.put("traffic_calc_whitelist_ranges", (JsonArray) null); + + // Act + List> result = calculator.parseWhitelistRanges(configWithRanges); + + // Assert - empty ranges + assertTrue(result.isEmpty()); + } + + // ============================================================================ + // SECTION 3: isInWhitelist() + // ============================================================================ + + @Test + void testIsInWhitelist_withinSingleRange() throws Exception { + // Setup - load whitelist with single range [1000, 2000] + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [1000, 2000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Assert - true when within range + assertTrue(calculator.isInWhitelist(1500L)); + } + + @Test + void testIsInWhitelist_exactlyAtStart() throws Exception { + // Setup - load whitelist with single range [1000, 2000] + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [1000, 2000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Assert - true when exactly at start of range + assertTrue(calculator.isInWhitelist(1000L)); + } + + @Test + void testIsInWhitelist_exactlyAtEnd() throws Exception { + // Setup - load whitelist with single range [1000, 2000] + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [1000, 2000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Assert - true when exactly at end of range + assertTrue(calculator.isInWhitelist(2000L)); + } + + @Test + void testIsInWhitelist_beforeRange() throws Exception { + // Setup - load whitelist with single range [1000, 2000] + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [1000, 2000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Assert - false when before range + assertFalse(calculator.isInWhitelist(999L)); + } + + @Test + void testIsInWhitelist_afterRange() throws Exception { + // Setup - load whitelist with single range [1000, 2000] + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [1000, 2000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Assert - false when after range + assertFalse(calculator.isInWhitelist(2001L)); + } + + @Test + void testIsInWhitelist_betweenRanges() throws Exception { + // Setup - load whitelist with two ranges [1000, 2000] and [3000, 4000] + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [1000, 2000], + [3000, 4000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Assert - false when between ranges + assertFalse(calculator.isInWhitelist(2500L)); + } + + @Test + void testIsInWhitelist_emptyRanges() throws Exception { + // Setup - no whitelist loaded (will fail and set empty) + when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Assert - false when empty ranges + assertFalse(calculator.isInWhitelist(1500L)); + } + + @Test + void testIsInWhitelist_nullRanges() throws Exception { + // Setup - no whitelist loaded (will fail and set empty) + when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Assert - false when null/empty ranges + assertFalse(calculator.isInWhitelist(1500L)); + } + + @Test + void testIsInWhitelist_invalidRangeSize() throws Exception { + // Setup - load whitelist with invalid range (only 1 element) and valid range + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [1000], + [2000, 3000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Assert + assertFalse(calculator.isInWhitelist(1500L)); // Should not match invalid range + assertTrue(calculator.isInWhitelist(2500L)); // Should match valid range + } + + @Test + void testIsInWhitelist_multipleRanges() throws Exception { + // Setup - load whitelist with multiple ranges + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [1000, 2000], + [3000, 4000], + [5000, 6000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Assert + assertTrue(calculator.isInWhitelist(1500L)); // In first range + assertTrue(calculator.isInWhitelist(3500L)); // In second range + assertTrue(calculator.isInWhitelist(5500L)); // In third range + assertFalse(calculator.isInWhitelist(2500L)); // Between first and second + } + + // ============================================================================ + // SECTION 4: getWhitelistDuration() + // ============================================================================ + + @Test + void testGetWhitelistDuration_noRanges() throws Exception { + // Setup - no ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Assert + assertEquals(0L, calculator.getWhitelistDuration(10000L, 5000L)); // 0 duration when no ranges + } + + @Test + void testGetWhitelistDuration_rangeFullyWithinWindow() throws Exception { + // Setup - range fully within window + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [6000, 7000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - window [5000, 10000], range [6000, 7000] + long duration = calculator.getWhitelistDuration(10000L, 5000L); + + // Assert - full range duration + assertEquals(1000L, duration); + } + + @Test + void testGetWhitelistDuration_rangePartiallyOverlapsStart() throws Exception { + // Setup - range partially overlaps start of window + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [3000, 7000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - window [5000, 10000], range [3000, 7000] + long duration = calculator.getWhitelistDuration(10000L, 5000L); + + // Assert - should clip to [5000, 7000] = 2000 + assertEquals(2000L, duration); + } + + @Test + void testGetWhitelistDuration_rangePartiallyOverlapsEnd() throws Exception { + // Setup - range partially overlaps end of window + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [8000, 12000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - window [5000, 10000], range [8000, 12000] + long duration = calculator.getWhitelistDuration(10000L, 5000L); + + // Assert - should clip to [8000, 10000] = 2000 + assertEquals(2000L, duration); + } + + @Test + void testGetWhitelistDuration_rangeCompletelyOutsideWindow() throws Exception { + // Setup - range completely outside window + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [1000, 2000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - window [5000, 10000], range [1000, 2000] + long duration = calculator.getWhitelistDuration(10000L, 5000L); + + // Assert - 0 duration when range completely outside window + assertEquals(0L, duration); + } + + @Test + void testGetWhitelistDuration_multipleRanges() throws Exception { + // Setup - multiple ranges + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [6000, 7000], + [8000, 9000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - window [5000, 10000], ranges [6000, 7000] and [8000, 9000] + long duration = calculator.getWhitelistDuration(10000L, 5000L); + + // Assert - 1000 + 1000 = 2000 + assertEquals(2000L, duration); + } + + @Test + void testGetWhitelistDuration_rangeSpansEntireWindow() throws Exception { + // Setup - range spans entire window + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [3000, 12000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - window [5000, 10000], range [3000, 12000] + long duration = calculator.getWhitelistDuration(10000L, 5000L); + + // Assert - entire window is whitelisted = 5000 + assertEquals(5000L, duration); + } + + // ============================================================================ + // SECTION 5: determineStatus() + // ============================================================================ + + @Test + void testDetermineStatus_belowThreshold() throws Exception { + // Setup - below threshold + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - 10 < 5 * 3 + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); + + // Assert - DEFAULT when below threshold + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testDetermineStatus_atThreshold() throws Exception { + // Setup - at threshold + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - 15 == 5 * 3 + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(15, 3); + + // Assert - DELAYED_PROCESSING when at threshold + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testDetermineStatus_aboveThreshold() throws Exception { + // Setup - above threshold + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - 20 > 5 * 3 + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(20, 3); + + // Assert - DELAYED_PROCESSING when above threshold + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testDetermineStatus_sumPastZero() throws Exception { + // Setup - sumPast is 0 + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - should return DEFAULT to avoid crash + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(100, 0); + + // Assert + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testDetermineStatus_bothZero() throws Exception { + // Setup - both sumCurrent and sumPast are 0; + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - should return DEFAULT to avoid crash + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(0, 0); + + // Assert + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testDetermineStatus_sumCurrentZero() throws Exception { + // Setup - sumCurrent is 0 + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - 0 < 5 * 10 + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(0, 10); + + // Assert - DEFAULT when sumCurrent is 0 + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @ParameterizedTest + @CsvSource({ + "1, 1, 1, DELAYED_PROCESSING", // threshold=1: 1 >= 1*1 + "2, 4, 2, DELAYED_PROCESSING", // threshold=2: 4 >= 2*2 + "5, 10, 2, DELAYED_PROCESSING", // threshold=5: 10 >= 5*2 + "10, 100, 10, DELAYED_PROCESSING", // threshold=10: 100 >= 10*10 + "5, 24, 5, DEFAULT", // threshold=5: 24 < 5*5 + "100, 1000, 11, DEFAULT" // threshold=100: 1000 < 100*11 + }) + void testDetermineStatus_variousThresholds(int threshold, int sumCurrent, int sumPast, String expectedStatus) throws Exception { + // Setup - various thresholds + config.put("traffic_calc_threshold_multiplier", threshold); + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(sumCurrent, sumPast); + + // Assert + assertEquals(OptOutTrafficCalculator.TrafficStatus.valueOf(expectedStatus), status); + } + + @Test + void testDetermineStatus_largeNumbers() throws Exception { + // Setup - test with large numbers + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act + OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(1_000_000, 200_000); + + // Assert - 1M >= 5 * 200K = 1M + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + // ============================================================================ + // SECTION 6: Whitelist Reload Tests + // ============================================================================ + + @Test + void testReloadWhitelist_success() throws Exception { + // Setup - initial whitelist + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [1000, 2000], + [3000, 4000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Change the whitelist to a new range + String newWhitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [5000, 6000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(newWhitelistJson.getBytes())); + + // Act - reload the whitelist + calculator.reloadWhitelist(); + + // Assert - verify new whitelist is loaded + assertTrue(calculator.isInWhitelist(5500L)); + } + + @Test + void testReloadWhitelist_failure() throws Exception { + // Setup - initial whitelist + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": [ + [1000, 2000] + ] + } + """; + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Now make it fail + when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Network error")); + + // Act - should not throw exception + calculator.reloadWhitelist(); + + // Assert + assertFalse(calculator.isInWhitelist(1500L)); + } + + // ============================================================================ + // SECTION 7: Cache Management Tests + // ============================================================================ + + @Test + void testGetCacheStats_emptyCache() throws Exception { + // Setup + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act + Map stats = calculator.getCacheStats(); + + // Assert - should return empty stats + assertEquals(0, stats.get("cached_files")); + assertEquals(0, stats.get("total_cached_timestamps")); + } + + @Test + void testClearCache() throws Exception { + // Setup + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act + calculator.clearCache(); + + // Assert - should return empty stats + Map stats = calculator.getCacheStats(); + assertEquals(0, stats.get("cached_files")); + } + + // ============================================================================ + // SECTION 8: Helper Methods for Test Data Creation + // ============================================================================ + + /** + * Create a mock SQS message with specified timestamp + */ + private Message createSqsMessage(long timestampSeconds) { + Map attributes = new HashMap<>(); + attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(timestampSeconds * 1000)); + + return Message.builder() + .messageId("test-msg-" + timestampSeconds) + .body("{\"test\": \"data\"}") + .attributes(attributes) + .build(); + } + + /** + * Create a mock SQS message without timestamp + */ + private Message createSqsMessageWithoutTimestamp() { + return Message.builder() + .messageId("test-msg-no-timestamp") + .body("{\"test\": \"data\"}") + .attributes(new HashMap<>()) + .build(); + } + + /** + * Create delta file bytes with specified timestamps + */ + private byte[] createDeltaFileBytes(List timestamps) throws Exception { + // Create OptOutEntry objects using newTestEntry + List entries = new ArrayList<>(); + + long idCounter = 1000; // Use incrementing IDs for test entries + for (long timestamp : timestamps) { + entries.add(OptOutEntry.newTestEntry(idCounter++, timestamp)); + } + + // Create OptOutCollection + OptOutCollection collection = new OptOutCollection(entries.toArray(new OptOutEntry[0])); + return collection.getStore(); + } + + + // ============================================================================ + // SECTION 9: Tests for calculateStatus() + // ============================================================================ + + @Test + void testCalculateStatus_noDeltaFiles() throws Exception { + // Setup - no delta files + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Collections.emptyList()); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); + + // Assert - should return DEFAULT when no delta files + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_normalTraffic() throws Exception { + // Setup - setup time: current time + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // Create delta files with timestamps distributed over 48 hours + List timestamps = new ArrayList<>(); + + // Past window: t-47h to t-25h (add 50 entries) + for (int i = 0; i < 50; i++) { + timestamps.add(t - 47*3600 + i * 1000); + } + + // Current window: t-23h to t-1h (add 100 entries - 2x past) + for (int i = 0; i < 100; i++) { + timestamps.add(t - 23*3600 + i * 1000); + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + + // Assert - 100+1 < 5 * 50 = 250, so should be DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_delayedProcessing() throws Exception { + // Setup - create delta files with spike in current window + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // Create delta files with spike in current window + List timestamps = new ArrayList<>(); + + // Past window: t-47h to t-25h (add 10 entries) + for (int i = 0; i < 10; i++) { + timestamps.add(t - 47*3600 + i * 1000); + } + + // Current window: t-23h to t-1h (add 100 entries - 10x past!) + for (int i = 0; i < 100; i++) { + timestamps.add(t - 23*3600 + i * 1000); + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + + // Assert - 100+1 >= 5 * 10 = 50, DELAYED_PROCESSING + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testCalculateStatus_noSqsMessages() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600, t - 7200); // Some entries + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - null SQS messages + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null); + + // Assert - should still calculate based on delta files, DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_emptySqsMessages() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - empty SQS messages + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); + + // Assert - should still calculate based on delta files, DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_multipleSqsMessages() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = new ArrayList<>(); + // Past window - 10 entries + for (int i = 0; i < 10; i++) { + timestamps.add(t - 48*3600 + i * 1000); + } + // Current window - 20 entries + for (int i = 0; i < 20; i++) { + timestamps.add(t - 12*3600 + i * 1000); + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - multiple SQS messages, oldest determines t + // Add enough SQS messages to push total count over DELAYED_PROCESSING threshold + List sqsMessages = new ArrayList<>(); + for (int i = 0; i < 101; i++) { + sqsMessages.add(createSqsMessage(t - i * 10)); + } + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + + // Assert - DELAYED_PROCESSING + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testCalculateStatus_withWhitelist() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // Whitelist that covers part of current window + String whitelistJson = String.format(""" + { + "traffic_calc_whitelist_ranges": [ + [%d, %d] + ] + } + """, t - 12*3600, t - 6*3600); + + List timestamps = new ArrayList<>(); + // Past window - 20 entries + for (int i = 0; i < 20; i++) { + timestamps.add(t - 48*3600 + i * 100); + } + + // Current window - 100 entries (50 in whitelist range, 50 outside) + for (int i = 0; i < 50; i++) { + timestamps.add(t - 12*3600 + i * 100); // In whitelist + } + for (int i = 0; i < 50; i++) { + timestamps.add(t - 3600 + i * 100); // Outside whitelist + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/delta-001.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + + // Assert - should filter out whitelisted entries + // Only ~50 from current window count (not whitelisted) + 1 SQS = 51 + // 51 < 5 * 20 = 100, so DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_cacheUtilization() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600, t - 7200); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - first call should populate cache + List sqsMessages = Arrays.asList(createSqsMessage(t)); + calculator.calculateStatus(sqsMessages); + + Map stats = calculator.getCacheStats(); + int cachedFiles = (Integer) stats.get("cached_files"); + + // Second call should use cache (no additional S3 download) + calculator.calculateStatus(sqsMessages); + + Map stats2 = calculator.getCacheStats(); + int cachedFiles2 = (Integer) stats2.get("cached_files"); + + // Assert - cache should be populated and remain consistent + assertEquals(1, cachedFiles); + assertEquals(cachedFiles, cachedFiles2); + + // Verify S3 download was called only once per file + verify(cloudStorage, times(1)).download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat"); + } + + @Test + void testCalculateStatus_s3Exception() throws Exception { + // Setup - S3 list error + when(cloudStorage.list(S3_DELTA_PREFIX)).thenThrow(new RuntimeException("S3 error")); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - should not throw exception + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); + + // Assert - DEFAULT on error + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_deltaFileReadException() throws Exception { + // Setup - S3 download error + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenThrow(new CloudStorageException("Failed to download")); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - empty SQS messages + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); + + // Assert - DEFAULT on error + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_invalidSqsMessageTimestamp() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act - SQS message without timestamp (should use current time) + List sqsMessages = Arrays.asList(createSqsMessageWithoutTimestamp()); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + + // Assert - DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_multipleDeltaFiles() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // File 1 - recent entries + List timestamps1 = new ArrayList<>(); + for (int i = 0; i < 50; i++) { + timestamps1.add(t - 12*3600 + i * 1000); + } + byte[] deltaFileBytes1 = createDeltaFileBytes(timestamps1); + + // File 2 - older entries + List timestamps2 = new ArrayList<>(); + for (int i = 0; i < 30; i++) { + timestamps2.add(t - 36*3600 + i * 1000); + } + byte[] deltaFileBytes2 = createDeltaFileBytes(timestamps2); + + when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList( + "optout-v2/delta/optout-delta--01_2025-11-13T02.00.00Z_bbbbbbbb.dat", + "optout-v2/delta/optout-delta--01_2025-11-13T01.00.00Z_aaaaaaaa.dat" + )); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T02.00.00Z_bbbbbbbb.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes1)); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T01.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes2)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + + // Assert - DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + + // Verify cache has both files + Map stats = calculator.getCacheStats(); + assertEquals(2, stats.get("cached_files")); + } + + @Test + void testCalculateStatus_windowBoundaryTimestamps() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + long currentWindowStart = t - 24*3600 + 300; // t-23h55m + long pastWindowStart = currentWindowStart - 24*3600; // t-47h55m + + List timestamps = Arrays.asList( + t, + currentWindowStart, + pastWindowStart, + t - 24*3600, + t - 48*3600 + ); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + + // Assert - DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_imestampsCached() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600, t - 7200); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + + // Assert + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + + // Cache should contain the timestamps + Map stats = calculator.getCacheStats(); + assertEquals(2, stats.get("total_cached_timestamps")); + } +} + From 726feda72323237a54810973f1166a0f883cf27e Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Fri, 14 Nov 2025 11:20:00 -0700 Subject: [PATCH 04/28] allow custom eval window --- .../optout/vertx/OptOutTrafficCalculator.java | 37 +++-- .../vertx/OptOutTrafficCalculatorTest.java | 132 +++++++++++++++++- 2 files changed, 149 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index a7142b97..c018ef56 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -36,7 +36,9 @@ public class OptOutTrafficCalculator { private final int thresholdMultiplier; private final ICloudStorage cloudStorage; private final String s3DeltaPrefix; // (e.g. "optout-v2/delta/") - private final String whitelistS3Path; // (e.g. "optout-breaker/traffic-filter-config.json") + private final String trafficCalcConfigS3Path; // (e.g. "optout-breaker/traffic-filter-config.json") + private int currentEvaluationWindowSeconds; + private int previousEvaluationWindowSeconds; private List> whitelistRanges; public enum TrafficStatus { @@ -67,24 +69,27 @@ private static class FileRecordCache { * @param cloudStorage Cloud storage for reading delta files and whitelist from S3 * @param cloudSync Cloud sync for path conversion */ - public OptOutTrafficCalculator(JsonObject config, ICloudStorage cloudStorage, String s3DeltaPrefix, String whitelistS3Path) { + public OptOutTrafficCalculator(JsonObject config, ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficCalcConfigS3Path) { this.cloudStorage = cloudStorage; this.thresholdMultiplier = config.getInteger("traffic_calc_threshold_multiplier", DEFAULT_THRESHOLD_MULTIPLIER); this.s3DeltaPrefix = s3DeltaPrefix; - this.whitelistS3Path = whitelistS3Path; - + this.trafficCalcConfigS3Path = trafficCalcConfigS3Path; + this.currentEvaluationWindowSeconds = HOURS_24 - 300; //23h55m (includes 5m queue window) + this.previousEvaluationWindowSeconds = HOURS_24; //24h // Initial whitelist load this.whitelistRanges = Collections.emptyList(); // Start empty - reloadWhitelist(); // Load from S3 + reloadTrafficCalcConfig(); // Load from S3 LOGGER.info("OptOutTrafficCalculator initialized: s3DeltaPrefix={}, whitelistPath={}, threshold={}x", - s3DeltaPrefix, whitelistS3Path, thresholdMultiplier); + s3DeltaPrefix, trafficCalcConfigS3Path, thresholdMultiplier); } /** - * Reload whitelist ranges from S3. + * Reload traffic calc config from S3. * Expected format: * { + * "traffic_calc_current_evaluation_window_seconds": 86400, + * "traffic_calc_previous_evaluation_window_seconds": 86400, * "traffic_calc_whitelist_ranges": [ * [startTimestamp1, endTimestamp1], * [startTimestamp2, endTimestamp2] @@ -93,19 +98,23 @@ public OptOutTrafficCalculator(JsonObject config, ICloudStorage cloudStorage, St * * Can be called periodically to pick up config changes without restarting. */ - public void reloadWhitelist() { - LOGGER.info("Reloading whitelist from S3: {}", whitelistS3Path); - try (InputStream is = cloudStorage.download(whitelistS3Path)) { + public void reloadTrafficCalcConfig() { + LOGGER.info("Reloading traffic calc config from S3: {}", trafficCalcConfigS3Path); + try (InputStream is = cloudStorage.download(trafficCalcConfigS3Path)) { String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); JsonObject whitelistConfig = new JsonObject(content); + + this.currentEvaluationWindowSeconds = whitelistConfig.getInteger("traffic_calc_current_evaluation_window_seconds", HOURS_24 - 300); + this.previousEvaluationWindowSeconds = whitelistConfig.getInteger("traffic_calc_previous_evaluation_window_seconds", HOURS_24); List> ranges = parseWhitelistRanges(whitelistConfig); this.whitelistRanges = ranges; - LOGGER.info("Successfully loaded {} whitelist ranges from S3", ranges.size()); + LOGGER.info("Successfully loaded traffic calc config from S3: currentEvaluationWindowSeconds={}, previousEvaluationWindowSeconds={}, whitelistRanges={}", + this.currentEvaluationWindowSeconds, this.previousEvaluationWindowSeconds, ranges.size()); } catch (Exception e) { - LOGGER.warn("No whitelist found at: {}", whitelistS3Path, e); + LOGGER.warn("No traffic calc config found at: {}", trafficCalcConfigS3Path, e); this.whitelistRanges = Collections.emptyList(); } } @@ -170,8 +179,8 @@ public TrafficStatus calculateStatus(List sqsMessages) { LOGGER.info("Traffic calculation starting with t={} (oldest SQS message)", t); // Define time windows - long currentWindowStart = t - (HOURS_24-300) - getWhitelistDuration(t, t - (HOURS_24-300)); // for range [t-23h55m, t+5m] - long pastWindowStart = currentWindowStart - HOURS_24 - getWhitelistDuration(currentWindowStart, currentWindowStart - HOURS_24); // for range [t-47h55m, t-23h55m] + long currentWindowStart = t - this.currentEvaluationWindowSeconds - getWhitelistDuration(t, t - this.currentEvaluationWindowSeconds); // for range [t-23h55m, t+5m] + long pastWindowStart = currentWindowStart - this.previousEvaluationWindowSeconds - getWhitelistDuration(currentWindowStart, currentWindowStart - this.previousEvaluationWindowSeconds); // for range [t-47h55m, t-23h55m] // Evict old cache entries (older than past window start) evictOldCacheEntries(pastWindowStart); diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java index 1078cd23..2f17fb97 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -82,7 +82,7 @@ void testConstructor_whitelistLoadFailure() throws Exception { when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); - calculator.reloadWhitelist(); + calculator.reloadTrafficCalcConfig(); // Assert - whitelist should be empty assertFalse(calculator.isInWhitelist(1000L)); @@ -696,7 +696,7 @@ void testDetermineStatus_largeNumbers() throws Exception { } // ============================================================================ - // SECTION 6: Whitelist Reload Tests + // SECTION 6: S3 Config Reload Tests // ============================================================================ @Test @@ -728,7 +728,7 @@ void testReloadWhitelist_success() throws Exception { .thenReturn(new ByteArrayInputStream(newWhitelistJson.getBytes())); // Act - reload the whitelist - calculator.reloadWhitelist(); + calculator.reloadTrafficCalcConfig(); // Assert - verify new whitelist is loaded assertTrue(calculator.isInWhitelist(5500L)); @@ -754,7 +754,7 @@ void testReloadWhitelist_failure() throws Exception { when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Network error")); // Act - should not throw exception - calculator.reloadWhitelist(); + calculator.reloadTrafficCalcConfig(); // Assert assertFalse(calculator.isInWhitelist(1500L)); @@ -1242,7 +1242,7 @@ void testCalculateStatus_windowBoundaryTimestamps() throws Exception { } @Test - void testCalculateStatus_imestampsCached() throws Exception { + void testCalculateStatus_timestampsCached() throws Exception { // Setup - create delta files with some entries long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; @@ -1269,5 +1269,125 @@ void testCalculateStatus_imestampsCached() throws Exception { Map stats = calculator.getCacheStats(); assertEquals(2, stats.get("total_cached_timestamps")); } -} + @Test + void testCalculateStatus_whitelistReducesPreviousWindowBaseline_customWindows() throws Exception { + // Setup - test with custom 3-hour evaluation windows + // Whitelist in previous window reduces baseline, causing DELAYED_PROCESSING + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + long threeHours = 3 * 3600; // 10800 seconds + + // Whitelist covering most of the PREVIOUS window (t-6h to t-4h) + // This reduces the baseline count in the previous window + String whitelistJson = String.format(""" + { + "traffic_calc_current_evaluation_window_seconds": %d, + "traffic_calc_previous_evaluation_window_seconds": %d, + "traffic_calc_whitelist_ranges": [ + [%d, %d] + ] + } + """, threeHours, threeHours, t - 6*3600, t - 4*3600); + + List timestamps = new ArrayList<>(); + + // Previous window (t-6h to t-3h): Add 100 entries + // 80 of these will be whitelisted (between t-6h and t-4h) + // Only 20 will count toward baseline + for (int i = 0; i < 80; i++) { + timestamps.add(t - 6*3600 + i); // Whitelisted entries in previous window + } + for (int i = 0; i < 20; i++) { + timestamps.add(t - 4*3600 + i); // Non-whitelisted entries in previous window + } + + // Current window (t-3h to t): Add 120 entries (none whitelisted) + // This creates a spike: 120 + 1 SQS >= 5 * 20 = 100 + for (int i = 0; i < 120; i++) { + timestamps.add(t - threeHours + i); // Current window entries + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + + // Assert - DELAYED_PROCESSING + // With 3-hour evaluation windows: + // Previous window (t-6h to t-3h): 100 total entries, 80 whitelisted → 20 counted + // Current window (t-3h to t): 120 entries (none whitelisted) + 1 SQS = 121 + // 121 >= 5 * 20 = 100, so DELAYED_PROCESSING + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testCalculateStatus_whitelistReducesCurrentWindowBaseline_customWindows() throws Exception { + // Setup - test with custom 3-hour evaluation windows + // Whitelist in previous window reduces baseline, causing DELAYED_PROCESSING + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + long threeHours = 3 * 3600; // 10800 seconds + + // Whitelist covering most of the CURRENT window (t-3h to t+5m) + // This reduces the baseline count in the previous window + String whitelistJson = String.format(""" + { + "traffic_calc_current_evaluation_window_seconds": %d, + "traffic_calc_previous_evaluation_window_seconds": %d, + "traffic_calc_whitelist_ranges": [ + [%d, %d] + ] + } + """, threeHours, threeHours, t - 3*3600, t - 1*3600); + + List timestamps = new ArrayList<>(); + + // Current window (t-3h to t+5m): Add 100 entries + // 80 of these will be whitelisted (between t-3h and t-1h) + // Only 20 will count toward baseline + for (int i = 0; i < 80; i++) { + timestamps.add(t - 3*3600 + i); // Whitelisted entries in current window + } + for (int i = 0; i < 20; i++) { + timestamps.add(t - 1*3600 + i); // Non-whitelisted entries in current window + } + + // Previous window (t-6h to t-3h): Add 10 entries (none whitelisted) + for (int i = 0; i < 10; i++) { + timestamps.add(t - 6*3600 + i); // Non-whitelisted entries in previous window + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.download(WHITELIST_S3_PATH)) + .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + + // Assert - DEFAULT + // With 3-hour evaluation windows: + // Previous window (t-6h to t-3h): 10 entries (none whitelisted) + // Current window (t-3h to t): 20 entries (non-whitelisted) + 1 SQS = 21 + // 21 < 5 * 10 = 50, so DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } +} From 1d8f050dbbb2b1c806cf55e7cd3de87e11816b01 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Fri, 14 Nov 2025 12:28:18 -0700 Subject: [PATCH 05/28] update comment --- .../java/com/uid2/optout/vertx/OptOutTrafficCalculator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index c018ef56..62d2df38 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -67,7 +67,8 @@ private static class FileRecordCache { * * @param config JsonObject containing configuration * @param cloudStorage Cloud storage for reading delta files and whitelist from S3 - * @param cloudSync Cloud sync for path conversion + * @param s3DeltaPrefix S3 prefix for delta files + * @param trafficCalcConfigS3Path S3 path for traffic calc config */ public OptOutTrafficCalculator(JsonObject config, ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficCalcConfigS3Path) { this.cloudStorage = cloudStorage; From 4bc5e45195df0ef917edcba436e79efb1a35fcbb Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 18 Nov 2025 17:32:19 -0700 Subject: [PATCH 06/28] switch to configmap for traffic config --- .../optout/vertx/OptOutTrafficCalculator.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 62d2df38..b92ca9a3 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -12,6 +12,8 @@ import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; import java.io.InputStream; import java.io.IOException; import java.util.*; @@ -31,12 +33,12 @@ public class OptOutTrafficCalculator { private static final int HOURS_24 = 24 * 3600; // 24 hours in seconds private static final int DEFAULT_THRESHOLD_MULTIPLIER = 5; + private static final String TRAFFIC_CONFIG_PATH = "/app/conf/traffic-config.json"; private final Map deltaFileCache = new ConcurrentHashMap<>(); private final int thresholdMultiplier; private final ICloudStorage cloudStorage; private final String s3DeltaPrefix; // (e.g. "optout-v2/delta/") - private final String trafficCalcConfigS3Path; // (e.g. "optout-breaker/traffic-filter-config.json") private int currentEvaluationWindowSeconds; private int previousEvaluationWindowSeconds; private List> whitelistRanges; @@ -70,23 +72,22 @@ private static class FileRecordCache { * @param s3DeltaPrefix S3 prefix for delta files * @param trafficCalcConfigS3Path S3 path for traffic calc config */ - public OptOutTrafficCalculator(JsonObject config, ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficCalcConfigS3Path) { + public OptOutTrafficCalculator(JsonObject config, ICloudStorage cloudStorage, String s3DeltaPrefix) { this.cloudStorage = cloudStorage; this.thresholdMultiplier = config.getInteger("traffic_calc_threshold_multiplier", DEFAULT_THRESHOLD_MULTIPLIER); this.s3DeltaPrefix = s3DeltaPrefix; - this.trafficCalcConfigS3Path = trafficCalcConfigS3Path; this.currentEvaluationWindowSeconds = HOURS_24 - 300; //23h55m (includes 5m queue window) this.previousEvaluationWindowSeconds = HOURS_24; //24h // Initial whitelist load this.whitelistRanges = Collections.emptyList(); // Start empty - reloadTrafficCalcConfig(); // Load from S3 + reloadTrafficCalcConfig(); // Load ConfigMap - LOGGER.info("OptOutTrafficCalculator initialized: s3DeltaPrefix={}, whitelistPath={}, threshold={}x", - s3DeltaPrefix, trafficCalcConfigS3Path, thresholdMultiplier); + LOGGER.info("OptOutTrafficCalculator initialized: s3DeltaPrefix={}, threshold={}x", + s3DeltaPrefix, thresholdMultiplier); } /** - * Reload traffic calc config from S3. + * Reload traffic calc config from ConfigMap. * Expected format: * { * "traffic_calc_current_evaluation_window_seconds": 86400, @@ -100,8 +101,8 @@ public OptOutTrafficCalculator(JsonObject config, ICloudStorage cloudStorage, St * Can be called periodically to pick up config changes without restarting. */ public void reloadTrafficCalcConfig() { - LOGGER.info("Reloading traffic calc config from S3: {}", trafficCalcConfigS3Path); - try (InputStream is = cloudStorage.download(trafficCalcConfigS3Path)) { + LOGGER.info("Loading traffic calc config from ConfigMap"); + try (InputStream is = Files.newInputStream(Paths.get(TRAFFIC_CONFIG_PATH))) { String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); JsonObject whitelistConfig = new JsonObject(content); @@ -111,11 +112,11 @@ public void reloadTrafficCalcConfig() { List> ranges = parseWhitelistRanges(whitelistConfig); this.whitelistRanges = ranges; - LOGGER.info("Successfully loaded traffic calc config from S3: currentEvaluationWindowSeconds={}, previousEvaluationWindowSeconds={}, whitelistRanges={}", + LOGGER.info("Successfully loaded traffic calc config from ConfigMap: currentEvaluationWindowSeconds={}, previousEvaluationWindowSeconds={}, whitelistRanges={}", this.currentEvaluationWindowSeconds, this.previousEvaluationWindowSeconds, ranges.size()); } catch (Exception e) { - LOGGER.warn("No traffic calc config found at: {}", trafficCalcConfigS3Path, e); + LOGGER.warn("No traffic calc config found at: {}", TRAFFIC_CONFIG_PATH, e); this.whitelistRanges = Collections.emptyList(); } } From 72a99d0357bb9ca0c558092ab75466e2079cdf72 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 18 Nov 2025 22:31:13 -0700 Subject: [PATCH 07/28] update to all k8s --- .../optout/vertx/OptOutTrafficCalculator.java | 20 +- .../vertx/OptOutTrafficCalculatorTest.java | 234 +++++++++--------- 2 files changed, 127 insertions(+), 127 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index b92ca9a3..b2db85f3 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -33,12 +33,12 @@ public class OptOutTrafficCalculator { private static final int HOURS_24 = 24 * 3600; // 24 hours in seconds private static final int DEFAULT_THRESHOLD_MULTIPLIER = 5; - private static final String TRAFFIC_CONFIG_PATH = "/app/conf/traffic-config.json"; private final Map deltaFileCache = new ConcurrentHashMap<>(); - private final int thresholdMultiplier; private final ICloudStorage cloudStorage; private final String s3DeltaPrefix; // (e.g. "optout-v2/delta/") + private final String trafficConfigPath; + private int thresholdMultiplier; private int currentEvaluationWindowSeconds; private int previousEvaluationWindowSeconds; private List> whitelistRanges; @@ -67,16 +67,16 @@ private static class FileRecordCache { /** * Constructor for OptOutTrafficCalculator * - * @param config JsonObject containing configuration * @param cloudStorage Cloud storage for reading delta files and whitelist from S3 * @param s3DeltaPrefix S3 prefix for delta files * @param trafficCalcConfigS3Path S3 path for traffic calc config */ - public OptOutTrafficCalculator(JsonObject config, ICloudStorage cloudStorage, String s3DeltaPrefix) { + public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficConfigPath) { this.cloudStorage = cloudStorage; - this.thresholdMultiplier = config.getInteger("traffic_calc_threshold_multiplier", DEFAULT_THRESHOLD_MULTIPLIER); + this.thresholdMultiplier = DEFAULT_THRESHOLD_MULTIPLIER; this.s3DeltaPrefix = s3DeltaPrefix; - this.currentEvaluationWindowSeconds = HOURS_24 - 300; //23h55m (includes 5m queue window) + this.trafficConfigPath = trafficConfigPath; + this.currentEvaluationWindowSeconds = HOURS_24 - 300; //23h55m (5m queue window) this.previousEvaluationWindowSeconds = HOURS_24; //24h // Initial whitelist load this.whitelistRanges = Collections.emptyList(); // Start empty @@ -95,17 +95,19 @@ public OptOutTrafficCalculator(JsonObject config, ICloudStorage cloudStorage, St * "traffic_calc_whitelist_ranges": [ * [startTimestamp1, endTimestamp1], * [startTimestamp2, endTimestamp2] - * ] + * ], + * "traffic_calc_threshold_multiplier": 5 * } * * Can be called periodically to pick up config changes without restarting. */ public void reloadTrafficCalcConfig() { LOGGER.info("Loading traffic calc config from ConfigMap"); - try (InputStream is = Files.newInputStream(Paths.get(TRAFFIC_CONFIG_PATH))) { + try (InputStream is = Files.newInputStream(Paths.get(trafficConfigPath))) { String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); JsonObject whitelistConfig = new JsonObject(content); + this.thresholdMultiplier = whitelistConfig.getInteger("traffic_calc_threshold_multiplier", DEFAULT_THRESHOLD_MULTIPLIER); this.currentEvaluationWindowSeconds = whitelistConfig.getInteger("traffic_calc_current_evaluation_window_seconds", HOURS_24 - 300); this.previousEvaluationWindowSeconds = whitelistConfig.getInteger("traffic_calc_previous_evaluation_window_seconds", HOURS_24); @@ -116,7 +118,7 @@ public void reloadTrafficCalcConfig() { this.currentEvaluationWindowSeconds, this.previousEvaluationWindowSeconds, ranges.size()); } catch (Exception e) { - LOGGER.warn("No traffic calc config found at: {}", TRAFFIC_CONFIG_PATH, e); + LOGGER.warn("No traffic calc config found at: {}", trafficConfigPath, e); this.whitelistRanges = Collections.emptyList(); } } diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java index 2f17fb97..d8ef71ad 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -6,6 +6,10 @@ import com.uid2.shared.optout.OptOutEntry; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -32,15 +36,36 @@ public class OptOutTrafficCalculatorTest { @Mock private ICloudStorage cloudStorage; - private JsonObject config; private static final String S3_DELTA_PREFIX = "optout-v2/delta/"; - private static final String WHITELIST_S3_PATH = "optout-breaker/traffic-filter-config.json"; - private static final int DEFAULT_THRESHOLD = 5; + private static final String TRAFFIC_CONFIG_PATH = "./traffic-config.json"; @BeforeEach void setUp() { - config = new JsonObject(); - config.put("traffic_calc_threshold_multiplier", DEFAULT_THRESHOLD); + try { + createTrafficConfigFile("{}"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @AfterEach + void tearDown() { + if (Files.exists(Path.of(TRAFFIC_CONFIG_PATH))) { + try { + Files.delete(Path.of(TRAFFIC_CONFIG_PATH)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + private void createTrafficConfigFile(String content) { + try { + Path configPath = Path.of(TRAFFIC_CONFIG_PATH); + Files.writeString(configPath, content); + } catch (Exception e) { + throw new RuntimeException(e); + } } // ============================================================================ @@ -50,9 +75,8 @@ void setUp() { @Test void testConstructor_defaultThreshold() throws Exception { // Setup - default threshold of 5 - JsonObject configWithoutThreshold = new JsonObject(); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - configWithoutThreshold, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - DEFAULT when below threshold, DELAYED_PROCESSING when above threshold OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); @@ -65,9 +89,9 @@ void testConstructor_defaultThreshold() throws Exception { @Test void testConstructor_customThreshold() throws Exception { // Setup - custom threshold of 10 - config.put("traffic_calc_threshold_multiplier", 10); + createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 10}"); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - DEFAULT when below threshold, DELAYED_PROCESSING when above threshold OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(49, 5); @@ -79,9 +103,9 @@ void testConstructor_customThreshold() throws Exception { @Test void testConstructor_whitelistLoadFailure() throws Exception { // Setup - whitelist load failure - when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + createTrafficConfigFile("Invalid JSON"); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); calculator.reloadTrafficCalcConfig(); // Assert - whitelist should be empty @@ -95,9 +119,8 @@ void testConstructor_whitelistLoadFailure() throws Exception { @Test void testParseWhitelistRanges_emptyConfig() throws Exception { // Setup - no config - when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject emptyConfig = new JsonObject(); // Act @@ -111,7 +134,7 @@ void testParseWhitelistRanges_emptyConfig() throws Exception { void testParseWhitelistRanges_singleRange() throws Exception { // Setup - single range OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); JsonArray ranges = new JsonArray() @@ -131,7 +154,7 @@ void testParseWhitelistRanges_singleRange() throws Exception { void testParseWhitelistRanges_multipleRanges() throws Exception { // Setup - multiple ranges OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); JsonArray ranges = new JsonArray() @@ -154,7 +177,7 @@ void testParseWhitelistRanges_multipleRanges() throws Exception { void testParseWhitelistRanges_misorderedRange() throws Exception { // Setup - range with end < start should be corrected OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); JsonArray ranges = new JsonArray() @@ -174,7 +197,7 @@ void testParseWhitelistRanges_misorderedRange() throws Exception { void testParseWhitelistRanges_sortsByStartTime() throws Exception { // Setup - ranges added out of order OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); JsonArray ranges = new JsonArray() @@ -197,7 +220,7 @@ void testParseWhitelistRanges_sortsByStartTime() throws Exception { void testParseWhitelistRanges_invalidRangeTooFewElements() throws Exception { // Setup - invalid range with only 1 element; OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); JsonArray ranges = new JsonArray() @@ -217,7 +240,7 @@ void testParseWhitelistRanges_invalidRangeTooFewElements() throws Exception { void testParseWhitelistRanges_nullArray() throws Exception { // Setup - null array OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); configWithRanges.put("traffic_calc_whitelist_ranges", (JsonArray) null); @@ -243,11 +266,10 @@ void testIsInWhitelist_withinSingleRange() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - true when within range assertTrue(calculator.isInWhitelist(1500L)); @@ -263,11 +285,10 @@ void testIsInWhitelist_exactlyAtStart() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - true when exactly at start of range assertTrue(calculator.isInWhitelist(1000L)); @@ -283,11 +304,10 @@ void testIsInWhitelist_exactlyAtEnd() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - true when exactly at end of range assertTrue(calculator.isInWhitelist(2000L)); @@ -303,11 +323,10 @@ void testIsInWhitelist_beforeRange() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - false when before range assertFalse(calculator.isInWhitelist(999L)); @@ -323,11 +342,10 @@ void testIsInWhitelist_afterRange() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - false when after range assertFalse(calculator.isInWhitelist(2001L)); @@ -344,11 +362,10 @@ void testIsInWhitelist_betweenRanges() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - false when between ranges assertFalse(calculator.isInWhitelist(2500L)); @@ -357,10 +374,10 @@ void testIsInWhitelist_betweenRanges() throws Exception { @Test void testIsInWhitelist_emptyRanges() throws Exception { // Setup - no whitelist loaded (will fail and set empty) - when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + createTrafficConfigFile("Invalid JSON"); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - false when empty ranges assertFalse(calculator.isInWhitelist(1500L)); @@ -369,10 +386,15 @@ void testIsInWhitelist_emptyRanges() throws Exception { @Test void testIsInWhitelist_nullRanges() throws Exception { // Setup - no whitelist loaded (will fail and set empty) - when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); + String whitelistJson = """ + { + "traffic_calc_whitelist_ranges": null + } + """; + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - false when null/empty ranges assertFalse(calculator.isInWhitelist(1500L)); @@ -389,11 +411,10 @@ void testIsInWhitelist_invalidRangeSize() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert assertFalse(calculator.isInWhitelist(1500L)); // Should not match invalid range @@ -412,11 +433,10 @@ void testIsInWhitelist_multipleRanges() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert assertTrue(calculator.isInWhitelist(1500L)); // In first range @@ -433,7 +453,7 @@ void testIsInWhitelist_multipleRanges() throws Exception { void testGetWhitelistDuration_noRanges() throws Exception { // Setup - no ranges OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert assertEquals(0L, calculator.getWhitelistDuration(10000L, 5000L)); // 0 duration when no ranges @@ -449,11 +469,10 @@ void testGetWhitelistDuration_rangeFullyWithinWindow() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], range [6000, 7000] long duration = calculator.getWhitelistDuration(10000L, 5000L); @@ -472,11 +491,10 @@ void testGetWhitelistDuration_rangePartiallyOverlapsStart() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], range [3000, 7000] long duration = calculator.getWhitelistDuration(10000L, 5000L); @@ -495,11 +513,10 @@ void testGetWhitelistDuration_rangePartiallyOverlapsEnd() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], range [8000, 12000] long duration = calculator.getWhitelistDuration(10000L, 5000L); @@ -518,11 +535,10 @@ void testGetWhitelistDuration_rangeCompletelyOutsideWindow() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], range [1000, 2000] long duration = calculator.getWhitelistDuration(10000L, 5000L); @@ -542,11 +558,10 @@ void testGetWhitelistDuration_multipleRanges() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], ranges [6000, 7000] and [8000, 9000] long duration = calculator.getWhitelistDuration(10000L, 5000L); @@ -565,11 +580,10 @@ void testGetWhitelistDuration_rangeSpansEntireWindow() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], range [3000, 12000] long duration = calculator.getWhitelistDuration(10000L, 5000L); @@ -586,7 +600,7 @@ void testGetWhitelistDuration_rangeSpansEntireWindow() throws Exception { void testDetermineStatus_belowThreshold() throws Exception { // Setup - below threshold OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - 10 < 5 * 3 OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(10, 3); @@ -599,7 +613,7 @@ void testDetermineStatus_belowThreshold() throws Exception { void testDetermineStatus_atThreshold() throws Exception { // Setup - at threshold OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - 15 == 5 * 3 OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(15, 3); @@ -612,7 +626,7 @@ void testDetermineStatus_atThreshold() throws Exception { void testDetermineStatus_aboveThreshold() throws Exception { // Setup - above threshold OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - 20 > 5 * 3 OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(20, 3); @@ -625,7 +639,7 @@ void testDetermineStatus_aboveThreshold() throws Exception { void testDetermineStatus_sumPastZero() throws Exception { // Setup - sumPast is 0 OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - should return DEFAULT to avoid crash OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(100, 0); @@ -638,7 +652,7 @@ void testDetermineStatus_sumPastZero() throws Exception { void testDetermineStatus_bothZero() throws Exception { // Setup - both sumCurrent and sumPast are 0; OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - should return DEFAULT to avoid crash OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(0, 0); @@ -651,7 +665,7 @@ void testDetermineStatus_bothZero() throws Exception { void testDetermineStatus_sumCurrentZero() throws Exception { // Setup - sumCurrent is 0 OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - 0 < 5 * 10 OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(0, 10); @@ -671,9 +685,9 @@ void testDetermineStatus_sumCurrentZero() throws Exception { }) void testDetermineStatus_variousThresholds(int threshold, int sumCurrent, int sumPast, String expectedStatus) throws Exception { // Setup - various thresholds - config.put("traffic_calc_threshold_multiplier", threshold); + createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": " + threshold + "}"); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(sumCurrent, sumPast); @@ -686,7 +700,7 @@ void testDetermineStatus_variousThresholds(int threshold, int sumCurrent, int su void testDetermineStatus_largeNumbers() throws Exception { // Setup - test with large numbers OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(1_000_000, 200_000); @@ -710,11 +724,10 @@ void testReloadWhitelist_success() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Change the whitelist to a new range String newWhitelistJson = """ @@ -724,8 +737,7 @@ void testReloadWhitelist_success() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(newWhitelistJson.getBytes())); + createTrafficConfigFile(newWhitelistJson); // Act - reload the whitelist calculator.reloadTrafficCalcConfig(); @@ -744,14 +756,13 @@ void testReloadWhitelist_failure() throws Exception { ] } """; - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Now make it fail - when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Network error")); + createTrafficConfigFile("Invalid JSON"); // Act - should not throw exception calculator.reloadTrafficCalcConfig(); @@ -768,7 +779,7 @@ void testReloadWhitelist_failure() throws Exception { void testGetCacheStats_emptyCache() throws Exception { // Setup OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act Map stats = calculator.getCacheStats(); @@ -782,7 +793,7 @@ void testGetCacheStats_emptyCache() throws Exception { void testClearCache() throws Exception { // Setup OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act calculator.clearCache(); @@ -849,7 +860,7 @@ void testCalculateStatus_noDeltaFiles() throws Exception { when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Collections.emptyList()); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); @@ -879,13 +890,12 @@ void testCalculateStatus_normalTraffic() throws Exception { byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); @@ -916,13 +926,12 @@ void testCalculateStatus_delayedProcessing() throws Exception { byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); @@ -941,13 +950,12 @@ void testCalculateStatus_noSqsMessages() throws Exception { List timestamps = Arrays.asList(t - 3600, t - 7200); // Some entries byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - null SQS messages OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null); @@ -965,13 +973,12 @@ void testCalculateStatus_emptySqsMessages() throws Exception { List timestamps = Arrays.asList(t - 3600); byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - empty SQS messages OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); @@ -998,13 +1005,12 @@ void testCalculateStatus_multipleSqsMessages() throws Exception { byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - multiple SQS messages, oldest determines t // Add enough SQS messages to push total count over DELAYED_PROCESSING threshold @@ -1049,14 +1055,13 @@ void testCalculateStatus_withWhitelist() throws Exception { byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/delta-001.dat")); when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); @@ -1077,13 +1082,12 @@ void testCalculateStatus_cacheUtilization() throws Exception { List timestamps = Arrays.asList(t - 3600, t - 7200); byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - first call should populate cache List sqsMessages = Arrays.asList(createSqsMessage(t)); @@ -1112,7 +1116,7 @@ void testCalculateStatus_s3Exception() throws Exception { when(cloudStorage.list(S3_DELTA_PREFIX)).thenThrow(new RuntimeException("S3 error")); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - should not throw exception OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); @@ -1129,7 +1133,7 @@ void testCalculateStatus_deltaFileReadException() throws Exception { .thenThrow(new CloudStorageException("Failed to download")); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - empty SQS messages OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); @@ -1147,13 +1151,12 @@ void testCalculateStatus_invalidSqsMessageTimestamp() throws Exception { List timestamps = Arrays.asList(t - 3600); byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - SQS message without timestamp (should use current time) List sqsMessages = Arrays.asList(createSqsMessageWithoutTimestamp()); @@ -1183,7 +1186,6 @@ void testCalculateStatus_multipleDeltaFiles() throws Exception { } byte[] deltaFileBytes2 = createDeltaFileBytes(timestamps2); - when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList( "optout-v2/delta/optout-delta--01_2025-11-13T02.00.00Z_bbbbbbbb.dat", "optout-v2/delta/optout-delta--01_2025-11-13T01.00.00Z_aaaaaaaa.dat" @@ -1194,7 +1196,7 @@ void testCalculateStatus_multipleDeltaFiles() throws Exception { .thenReturn(new ByteArrayInputStream(deltaFileBytes2)); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); @@ -1225,13 +1227,12 @@ void testCalculateStatus_windowBoundaryTimestamps() throws Exception { ); byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); @@ -1250,13 +1251,12 @@ void testCalculateStatus_timestampsCached() throws Exception { List timestamps = Arrays.asList(t - 3600, t - 7200); byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - when(cloudStorage.download(WHITELIST_S3_PATH)).thenThrow(new CloudStorageException("Not found")); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); @@ -1310,14 +1310,13 @@ void testCalculateStatus_whitelistReducesPreviousWindowBaseline_customWindows() byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); @@ -1370,14 +1369,13 @@ void testCalculateStatus_whitelistReducesCurrentWindowBaseline_customWindows() t byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - when(cloudStorage.download(WHITELIST_S3_PATH)) - .thenReturn(new ByteArrayInputStream(whitelistJson.getBytes())); + createTrafficConfigFile(whitelistJson); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - config, cloudStorage, S3_DELTA_PREFIX, WHITELIST_S3_PATH); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); From 027f5766854baa002036c680bca010ea4bdd68f5 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Thu, 20 Nov 2025 01:45:05 -0700 Subject: [PATCH 08/28] update config validations --- .../optout/vertx/OptOutTrafficCalculator.java | 35 +++++++++---- .../vertx/OptOutTrafficCalculatorTest.java | 52 +++++++++++++------ 2 files changed, 62 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index b2db85f3..ab0f7563 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -63,7 +63,15 @@ private static class FileRecordCache { this.newestTimestamp = timestamps.isEmpty() ? 0 : Collections.max(timestamps); } } - + + /** + * Exception thrown by malformed traffic calculator config + */ + public static class MalformedTrafficCalcConfigException extends Exception { + public MalformedTrafficCalcConfigException(String message) { + super(message); + } + } /** * Constructor for OptOutTrafficCalculator * @@ -71,7 +79,7 @@ private static class FileRecordCache { * @param s3DeltaPrefix S3 prefix for delta files * @param trafficCalcConfigS3Path S3 path for traffic calc config */ - public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficConfigPath) { + public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficConfigPath) throws MalformedTrafficCalcConfigException { this.cloudStorage = cloudStorage; this.thresholdMultiplier = DEFAULT_THRESHOLD_MULTIPLIER; this.s3DeltaPrefix = s3DeltaPrefix; @@ -101,7 +109,7 @@ public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, * * Can be called periodically to pick up config changes without restarting. */ - public void reloadTrafficCalcConfig() { + public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException { LOGGER.info("Loading traffic calc config from ConfigMap"); try (InputStream is = Files.newInputStream(Paths.get(trafficConfigPath))) { String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); @@ -119,14 +127,14 @@ public void reloadTrafficCalcConfig() { } catch (Exception e) { LOGGER.warn("No traffic calc config found at: {}", trafficConfigPath, e); - this.whitelistRanges = Collections.emptyList(); + throw new MalformedTrafficCalcConfigException("Failed to load traffic calc config: " + e.getMessage()); } } /** * Parse whitelist ranges from JSON config */ - List> parseWhitelistRanges(JsonObject config) { + List> parseWhitelistRanges(JsonObject config) throws MalformedTrafficCalcConfigException { List> ranges = new ArrayList<>(); try { @@ -136,12 +144,18 @@ List> parseWhitelistRanges(JsonObject config) { for (int i = 0; i < rangesArray.size(); i++) { var rangeArray = rangesArray.getJsonArray(i); if (rangeArray != null && rangeArray.size() >= 2) { - long val1 = rangeArray.getLong(0); - long val2 = rangeArray.getLong(1); + long start = rangeArray.getLong(0); + long end = rangeArray.getLong(1); - // Ensure start <= end (correct misordered ranges) - long start = Math.min(val1, val2); - long end = Math.max(val1, val2); + if(start >= end) { + LOGGER.error("Invalid whitelist range: start must be less than end: [{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("Invalid whitelist range at index " + i + ": start must be less than end"); + } + + if (end - start > 86400) { + LOGGER.error("Invalid whitelist range: range must be less than 24 hours: [{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("Invalid whitelist range at index " + i + ": range must be less than 24 hours"); + } List range = Arrays.asList(start, end); ranges.add(range); @@ -156,6 +170,7 @@ List> parseWhitelistRanges(JsonObject config) { } catch (Exception e) { LOGGER.error("Failed to parse whitelist ranges", e); + throw new MalformedTrafficCalcConfigException("Failed to parse whitelist ranges: " + e.getMessage()); } return ranges; diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java index d8ef71ad..acc0b945 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -22,6 +22,7 @@ import software.amazon.awssdk.services.sqs.model.Message; import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; +import com.uid2.optout.vertx.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; import java.io.ByteArrayInputStream; import java.util.*; @@ -104,12 +105,19 @@ void testConstructor_customThreshold() throws Exception { void testConstructor_whitelistLoadFailure() throws Exception { // Setup - whitelist load failure createTrafficConfigFile("Invalid JSON"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + }); + + createTrafficConfigFile("{}"); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - calculator.reloadTrafficCalcConfig(); + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Assert - whitelist should be empty - assertFalse(calculator.isInWhitelist(1000L)); + createTrafficConfigFile("Invalid JSON"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); } // ============================================================================ @@ -175,7 +183,7 @@ void testParseWhitelistRanges_multipleRanges() throws Exception { @Test void testParseWhitelistRanges_misorderedRange() throws Exception { - // Setup - range with end < start should be corrected + // Setup - range with end < start is malformed OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -185,12 +193,26 @@ void testParseWhitelistRanges_misorderedRange() throws Exception { configWithRanges.put("traffic_calc_whitelist_ranges", ranges); // Act - List> result = calculator.parseWhitelistRanges(configWithRanges); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.parseWhitelistRanges(configWithRanges); + }); + } - // Assert - should auto-correct to [1000, 2000] - assertEquals(1, result.size()); - assertEquals(1000L, result.get(0).get(0)); - assertEquals(2000L, result.get(0).get(1)); + @Test + void testParseWhitelistRanges_rangeTooLong() throws Exception { + // Setup - range longer than 24 hours is malformed + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(2000L).add(200000L)); // Longer than 24 hours + configWithRanges.put("traffic_calc_whitelist_ranges", ranges); + + // Act + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.parseWhitelistRanges(configWithRanges); + }); } @Test @@ -373,8 +395,8 @@ void testIsInWhitelist_betweenRanges() throws Exception { @Test void testIsInWhitelist_emptyRanges() throws Exception { - // Setup - no whitelist loaded (will fail and set empty) - createTrafficConfigFile("Invalid JSON"); + // Setup - no whitelist loaded + createTrafficConfigFile("{}"); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -765,10 +787,10 @@ void testReloadWhitelist_failure() throws Exception { createTrafficConfigFile("Invalid JSON"); // Act - should not throw exception - calculator.reloadTrafficCalcConfig(); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); - // Assert - assertFalse(calculator.isInWhitelist(1500L)); } // ============================================================================ From 58bd29876bf36800b5325fc448c7c5a19a9146dc Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Fri, 21 Nov 2025 11:50:28 -0700 Subject: [PATCH 09/28] small rename --- .../com/uid2/optout/vertx/OptOutTrafficCalculator.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index ab0f7563..3ce733e4 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -37,7 +37,7 @@ public class OptOutTrafficCalculator { private final Map deltaFileCache = new ConcurrentHashMap<>(); private final ICloudStorage cloudStorage; private final String s3DeltaPrefix; // (e.g. "optout-v2/delta/") - private final String trafficConfigPath; + private final String trafficCalcConfigPath; private int thresholdMultiplier; private int currentEvaluationWindowSeconds; private int previousEvaluationWindowSeconds; @@ -79,11 +79,11 @@ public MalformedTrafficCalcConfigException(String message) { * @param s3DeltaPrefix S3 prefix for delta files * @param trafficCalcConfigS3Path S3 path for traffic calc config */ - public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficConfigPath) throws MalformedTrafficCalcConfigException { + public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficCalcConfigPath) throws MalformedTrafficCalcConfigException { this.cloudStorage = cloudStorage; this.thresholdMultiplier = DEFAULT_THRESHOLD_MULTIPLIER; this.s3DeltaPrefix = s3DeltaPrefix; - this.trafficConfigPath = trafficConfigPath; + this.trafficCalcConfigPath = trafficCalcConfigPath; this.currentEvaluationWindowSeconds = HOURS_24 - 300; //23h55m (5m queue window) this.previousEvaluationWindowSeconds = HOURS_24; //24h // Initial whitelist load @@ -111,7 +111,7 @@ public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, */ public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException { LOGGER.info("Loading traffic calc config from ConfigMap"); - try (InputStream is = Files.newInputStream(Paths.get(trafficConfigPath))) { + try (InputStream is = Files.newInputStream(Paths.get(trafficCalcConfigPath))) { String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); JsonObject whitelistConfig = new JsonObject(content); @@ -126,7 +126,7 @@ public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException this.currentEvaluationWindowSeconds, this.previousEvaluationWindowSeconds, ranges.size()); } catch (Exception e) { - LOGGER.warn("No traffic calc config found at: {}", trafficConfigPath, e); + LOGGER.warn("No traffic calc config found at: {}", trafficCalcConfigPath, e); throw new MalformedTrafficCalcConfigException("Failed to load traffic calc config: " + e.getMessage()); } } From b3cbdfdf23bfc57bad1df53fcaab1ff9a2d92978 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Fri, 21 Nov 2025 11:57:38 -0700 Subject: [PATCH 10/28] test fix --- .../optout/vertx/OptOutTrafficCalculator.java | 8 ----- .../vertx/OptOutTrafficCalculatorTest.java | 14 --------- .../optout/vertx/SqsMessageParserTest.java | 30 +++++++++---------- 3 files changed, 15 insertions(+), 37 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 3ce733e4..db3591d6 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -495,12 +495,4 @@ public Map getCacheStats() { return stats; } - /** - * Clear the cache (for testing or manual reset) - */ - public void clearCache() { - int size = deltaFileCache.size(); - deltaFileCache.clear(); - LOGGER.info("Cleared cache ({} entries)", size); - } } diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java index acc0b945..f35b7bf7 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -811,20 +811,6 @@ void testGetCacheStats_emptyCache() throws Exception { assertEquals(0, stats.get("total_cached_timestamps")); } - @Test - void testClearCache() throws Exception { - // Setup - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - calculator.clearCache(); - - // Assert - should return empty stats - Map stats = calculator.getCacheStats(); - assertEquals(0, stats.get("cached_files")); - } - // ============================================================================ // SECTION 8: Helper Methods for Test Data Creation // ============================================================================ diff --git a/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java b/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java index 552c1efe..6a2ab873 100644 --- a/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java +++ b/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java @@ -184,8 +184,8 @@ public void testFilterEligibleMessages_allEligible() { // Create messages from 10 minutes ago long oldTimestamp = System.currentTimeMillis() / 1000 - 600; - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], oldTimestamp)); - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], oldTimestamp - 100)); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], oldTimestamp, "dummy@email.com", null, "127.0.0.1", "trace-id")); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], oldTimestamp - 100, "dummy@email.com", null, "127.0.0.1", "trace-id")); long currentTime = System.currentTimeMillis() / 1000; List result = SqsMessageParser.filterEligibleMessages(messages, 300, currentTime); @@ -200,8 +200,8 @@ public void testFilterEligibleMessages_noneEligible() { // Create messages from 1 minute ago (too recent) long recentTimestamp = System.currentTimeMillis() / 1000 - 60; - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], recentTimestamp)); - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], recentTimestamp + 10)); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], recentTimestamp, "dummy@email.com", null, "127.0.0.1", "trace-id")); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], recentTimestamp + 10, "dummy@email.com", null, "127.0.0.1", "trace-id")); long currentTime = System.currentTimeMillis() / 1000; List result = SqsMessageParser.filterEligibleMessages(messages, 300, currentTime); @@ -217,13 +217,13 @@ public void testFilterEligibleMessages_mixedEligibility() { long currentTime = 1000L; // Old enough (600 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - 600)); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - 600, "dummy@email.com", null, "127.0.0.1", "trace-id")); // Exactly at threshold (300 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - 300)); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - 300, "dummy@email.com", null, "127.0.0.1", "trace-id")); // Too recent (100 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - 100)); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - 100, "dummy@email.com", null, "127.0.0.1", "trace-id")); List result = SqsMessageParser.filterEligibleMessages(messages, 300, currentTime); @@ -291,13 +291,13 @@ public void testFilterEligibleMessages_boundaryCases() { int windowSeconds = 300; // One second too new (299 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - windowSeconds + 1)); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - windowSeconds + 1, "dummy@email.com", null, "127.0.0.1", "trace-id")); // Exactly at threshold (300 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - windowSeconds)); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - windowSeconds, "dummy@email.com", null, "127.0.0.1", "trace-id")); // One second past threshold (301 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - windowSeconds - 1)); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - windowSeconds - 1, "dummy@email.com", null, "127.0.0.1", "trace-id")); List result = SqsMessageParser.filterEligibleMessages(messages, windowSeconds, currentTime); @@ -328,10 +328,10 @@ public void testFilterEligibleMessages_preservesOrder() { long currentTime = 1000L; // Add eligible messages in specific order - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 100)); - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 200)); - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 300)); - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 900)); // Too recent + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 100, "dummy@email.com", null, "127.0.0.1", "trace-id")); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 200, "dummy@email.com", null, "127.0.0.1", "trace-id")); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 300, "dummy@email.com", null, "127.0.0.1", "trace-id")); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 900, "dummy@email.com", null, "127.0.0.1", "trace-id")); // Too recent List result = SqsMessageParser.filterEligibleMessages(messages, 300, currentTime); @@ -385,7 +385,7 @@ public void testFilterEligibleMessages_zeroWindowSeconds() { Message mockMsg = createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS); long currentTime = 1000L; - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime)); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime, "dummy@email.com", null, "127.0.0.1", "trace-id")); List result = SqsMessageParser.filterEligibleMessages(messages, 0, currentTime); From a802e31f864099b1ffd675462497645d513311e9 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Fri, 21 Nov 2025 12:03:30 -0700 Subject: [PATCH 11/28] undo accidental change --- .../optout/vertx/SqsMessageParserTest.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java b/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java index 6a2ab873..552c1efe 100644 --- a/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java +++ b/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java @@ -184,8 +184,8 @@ public void testFilterEligibleMessages_allEligible() { // Create messages from 10 minutes ago long oldTimestamp = System.currentTimeMillis() / 1000 - 600; - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], oldTimestamp, "dummy@email.com", null, "127.0.0.1", "trace-id")); - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], oldTimestamp - 100, "dummy@email.com", null, "127.0.0.1", "trace-id")); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], oldTimestamp)); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], oldTimestamp - 100)); long currentTime = System.currentTimeMillis() / 1000; List result = SqsMessageParser.filterEligibleMessages(messages, 300, currentTime); @@ -200,8 +200,8 @@ public void testFilterEligibleMessages_noneEligible() { // Create messages from 1 minute ago (too recent) long recentTimestamp = System.currentTimeMillis() / 1000 - 60; - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], recentTimestamp, "dummy@email.com", null, "127.0.0.1", "trace-id")); - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], recentTimestamp + 10, "dummy@email.com", null, "127.0.0.1", "trace-id")); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], recentTimestamp)); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], recentTimestamp + 10)); long currentTime = System.currentTimeMillis() / 1000; List result = SqsMessageParser.filterEligibleMessages(messages, 300, currentTime); @@ -217,13 +217,13 @@ public void testFilterEligibleMessages_mixedEligibility() { long currentTime = 1000L; // Old enough (600 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - 600, "dummy@email.com", null, "127.0.0.1", "trace-id")); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - 600)); // Exactly at threshold (300 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - 300, "dummy@email.com", null, "127.0.0.1", "trace-id")); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - 300)); // Too recent (100 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - 100, "dummy@email.com", null, "127.0.0.1", "trace-id")); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - 100)); List result = SqsMessageParser.filterEligibleMessages(messages, 300, currentTime); @@ -291,13 +291,13 @@ public void testFilterEligibleMessages_boundaryCases() { int windowSeconds = 300; // One second too new (299 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - windowSeconds + 1, "dummy@email.com", null, "127.0.0.1", "trace-id")); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - windowSeconds + 1)); // Exactly at threshold (300 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - windowSeconds, "dummy@email.com", null, "127.0.0.1", "trace-id")); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - windowSeconds)); // One second past threshold (301 seconds ago) - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - windowSeconds - 1, "dummy@email.com", null, "127.0.0.1", "trace-id")); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime - windowSeconds - 1)); List result = SqsMessageParser.filterEligibleMessages(messages, windowSeconds, currentTime); @@ -328,10 +328,10 @@ public void testFilterEligibleMessages_preservesOrder() { long currentTime = 1000L; // Add eligible messages in specific order - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 100, "dummy@email.com", null, "127.0.0.1", "trace-id")); - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 200, "dummy@email.com", null, "127.0.0.1", "trace-id")); - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 300, "dummy@email.com", null, "127.0.0.1", "trace-id")); - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 900, "dummy@email.com", null, "127.0.0.1", "trace-id")); // Too recent + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 100)); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 200)); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 300)); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], 900)); // Too recent List result = SqsMessageParser.filterEligibleMessages(messages, 300, currentTime); @@ -385,7 +385,7 @@ public void testFilterEligibleMessages_zeroWindowSeconds() { Message mockMsg = createValidMessage(VALID_HASH_BASE64, VALID_ID_BASE64, TEST_TIMESTAMP_MS); long currentTime = 1000L; - messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime, "dummy@email.com", null, "127.0.0.1", "trace-id")); + messages.add(new SqsParsedMessage(mockMsg, new byte[32], new byte[32], currentTime)); List result = SqsMessageParser.filterEligibleMessages(messages, 0, currentTime); From e26a64f12a2a930ad16eba4cb55229ced7c037d3 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Fri, 21 Nov 2025 12:05:04 -0700 Subject: [PATCH 12/28] whitespace --- src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index db3591d6..89173761 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -72,6 +72,7 @@ public MalformedTrafficCalcConfigException(String message) { super(message); } } + /** * Constructor for OptOutTrafficCalculator * From fd13b6918095986d5f5bf2393a4b4e9f6fa15533 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Fri, 21 Nov 2025 12:09:30 -0700 Subject: [PATCH 13/28] whitespace --- .../java/com/uid2/optout/vertx/OptOutTrafficCalculator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 89173761..ff32b9fe 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -72,7 +72,7 @@ public MalformedTrafficCalcConfigException(String message) { super(message); } } - + /** * Constructor for OptOutTrafficCalculator * @@ -127,7 +127,7 @@ public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException this.currentEvaluationWindowSeconds, this.previousEvaluationWindowSeconds, ranges.size()); } catch (Exception e) { - LOGGER.warn("No traffic calc config found at: {}", trafficCalcConfigPath, e); + LOGGER.warn("Failed to load traffic calc config. Config is malformed or missing: {}", trafficCalcConfigPath, e); throw new MalformedTrafficCalcConfigException("Failed to load traffic calc config: " + e.getMessage()); } } From 14a6c1ff9638a09d05d9478cf112447115ed5376 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 25 Nov 2025 16:03:31 -0700 Subject: [PATCH 14/28] update traffic baseline to hardcoded --- src/main/java/com/uid2/optout/Const.java | 4 + .../optout/vertx/OptOutTrafficCalculator.java | 116 +++++++++--------- 2 files changed, 64 insertions(+), 56 deletions(-) diff --git a/src/main/java/com/uid2/optout/Const.java b/src/main/java/com/uid2/optout/Const.java index fc39c2f1..6dacd9a1 100644 --- a/src/main/java/com/uid2/optout/Const.java +++ b/src/main/java/com/uid2/optout/Const.java @@ -22,6 +22,10 @@ public static class Config extends com.uid2.shared.Const.Config { public static final String OptOutSqsS3FolderProp = "optout_sqs_s3_folder"; // Default: "sqs-delta" - folder within same S3 bucket as regular optout public static final String OptOutSqsMaxMessagesPerPollProp = "optout_sqs_max_messages_per_poll"; public static final String OptOutSqsVisibilityTimeoutProp = "optout_sqs_visibility_timeout"; + public static final String OptOutTrafficCalcBaselineTrafficProp = "traffic_calc_baseline_traffic"; + public static final String OptOutTrafficCalcThresholdMultiplierProp = "traffic_calc_threshold_multiplier"; + public static final String OptOutTrafficCalcEvaluationWindowSecondsProp = "traffic_calc_evaluation_window_seconds"; + public static final String OptOutTrafficCalcWhitelistRangesProp = "traffic_calc_whitelist_ranges"; } public static class Event { diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index ff32b9fe..9452ec25 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -23,24 +23,24 @@ /** * Calculates opt-out traffic patterns to determine DEFAULT or DELAYED_PROCESSING status. * - * Compares recent ~24h traffic (sumCurrent) against previous ~24h baseline (sumPast). - * Both sums exclude records in whitelist ranges (surge windows determined by engineers). + * Compares recent ~24h traffic (sumCurrent) against a configurable baseline (baselineTraffic) of expected traffic in 24 hours. + * The baseline is calculated as baselineTraffic*thresholdMultiplier. + * sumCurrent excludes records in whitelist ranges (surge windows determined by engineers). * - * Returns DELAYED_PROCESSING if sumCurrent >= 5 × sumPast, indicating abnormal traffic spike. + * Returns DELAYED_PROCESSING if sumCurrent >= thresholdMultiplier * baselineTraffic, indicating abnormal traffic spike. */ public class OptOutTrafficCalculator { private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficCalculator.class); private static final int HOURS_24 = 24 * 3600; // 24 hours in seconds - private static final int DEFAULT_THRESHOLD_MULTIPLIER = 5; private final Map deltaFileCache = new ConcurrentHashMap<>(); private final ICloudStorage cloudStorage; private final String s3DeltaPrefix; // (e.g. "optout-v2/delta/") private final String trafficCalcConfigPath; + private int baselineTraffic; private int thresholdMultiplier; - private int currentEvaluationWindowSeconds; - private int previousEvaluationWindowSeconds; + private int evaluationWindowSeconds; private List> whitelistRanges; public enum TrafficStatus { @@ -82,13 +82,8 @@ public MalformedTrafficCalcConfigException(String message) { */ public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, String trafficCalcConfigPath) throws MalformedTrafficCalcConfigException { this.cloudStorage = cloudStorage; - this.thresholdMultiplier = DEFAULT_THRESHOLD_MULTIPLIER; this.s3DeltaPrefix = s3DeltaPrefix; this.trafficCalcConfigPath = trafficCalcConfigPath; - this.currentEvaluationWindowSeconds = HOURS_24 - 300; //23h55m (5m queue window) - this.previousEvaluationWindowSeconds = HOURS_24; //24h - // Initial whitelist load - this.whitelistRanges = Collections.emptyList(); // Start empty reloadTrafficCalcConfig(); // Load ConfigMap LOGGER.info("OptOutTrafficCalculator initialized: s3DeltaPrefix={}, threshold={}x", @@ -116,16 +111,33 @@ public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); JsonObject whitelistConfig = new JsonObject(content); - this.thresholdMultiplier = whitelistConfig.getInteger("traffic_calc_threshold_multiplier", DEFAULT_THRESHOLD_MULTIPLIER); - this.currentEvaluationWindowSeconds = whitelistConfig.getInteger("traffic_calc_current_evaluation_window_seconds", HOURS_24 - 300); - this.previousEvaluationWindowSeconds = whitelistConfig.getInteger("traffic_calc_previous_evaluation_window_seconds", HOURS_24); + // Validate required fields exist + if (!whitelistConfig.containsKey("traffic_calc_evaluation_window_seconds")) { + throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_evaluation_window_seconds"); + } + if (!whitelistConfig.containsKey("traffic_calc_baseline_traffic")) { + throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_baseline_traffic"); + } + if (!whitelistConfig.containsKey("traffic_calc_threshold_multiplier")) { + throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_threshold_multiplier"); + } + if (!whitelistConfig.containsKey("traffic_calc_whitelist_ranges")) { + throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_whitelist_ranges"); + } + + this.evaluationWindowSeconds = whitelistConfig.getInteger("traffic_calc_evaluation_window_seconds"); + this.baselineTraffic = whitelistConfig.getInteger("traffic_calc_baseline_traffic"); + this.thresholdMultiplier = whitelistConfig.getInteger("traffic_calc_threshold_multiplier"); List> ranges = parseWhitelistRanges(whitelistConfig); this.whitelistRanges = ranges; - LOGGER.info("Successfully loaded traffic calc config from ConfigMap: currentEvaluationWindowSeconds={}, previousEvaluationWindowSeconds={}, whitelistRanges={}", - this.currentEvaluationWindowSeconds, this.previousEvaluationWindowSeconds, ranges.size()); + LOGGER.info("Successfully loaded traffic calc config from ConfigMap: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, whitelistRanges={}", + this.evaluationWindowSeconds, this.baselineTraffic, this.thresholdMultiplier, ranges.size()); + } catch (MalformedTrafficCalcConfigException e) { + LOGGER.warn("Failed to load traffic calc config. Config is malformed: {}", trafficCalcConfigPath, e); + throw e; } catch (Exception e) { LOGGER.warn("Failed to load traffic calc config. Config is malformed or missing: {}", trafficCalcConfigPath, e); throw new MalformedTrafficCalcConfigException("Failed to load traffic calc config: " + e.getMessage()); @@ -139,29 +151,27 @@ List> parseWhitelistRanges(JsonObject config) throws MalformedTraffic List> ranges = new ArrayList<>(); try { - if (config.containsKey("traffic_calc_whitelist_ranges")) { - var rangesArray = config.getJsonArray("traffic_calc_whitelist_ranges"); - if (rangesArray != null) { - for (int i = 0; i < rangesArray.size(); i++) { - var rangeArray = rangesArray.getJsonArray(i); - if (rangeArray != null && rangeArray.size() >= 2) { - long start = rangeArray.getLong(0); - long end = rangeArray.getLong(1); - - if(start >= end) { - LOGGER.error("Invalid whitelist range: start must be less than end: [{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("Invalid whitelist range at index " + i + ": start must be less than end"); - } + var rangesArray = config.getJsonArray("traffic_calc_whitelist_ranges"); + if (rangesArray != null) { + for (int i = 0; i < rangesArray.size(); i++) { + var rangeArray = rangesArray.getJsonArray(i); + if (rangeArray != null && rangeArray.size() >= 2) { + long start = rangeArray.getLong(0); + long end = rangeArray.getLong(1); + + if(start >= end) { + LOGGER.error("Invalid whitelist range: start must be less than end: [{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("Invalid whitelist range at index " + i + ": start must be less than end"); + } - if (end - start > 86400) { - LOGGER.error("Invalid whitelist range: range must be less than 24 hours: [{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("Invalid whitelist range at index " + i + ": range must be less than 24 hours"); - } - - List range = Arrays.asList(start, end); - ranges.add(range); - LOGGER.info("Loaded whitelist range: [{}, {}]", start, end); + if (end - start > 86400) { + LOGGER.error("Invalid whitelist range: range must be less than 24 hours: [{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("Invalid whitelist range at index " + i + ": range must be less than 24 hours"); } + + List range = Arrays.asList(start, end); + ranges.add(range); + LOGGER.info("Loaded whitelist range: [{}, {}]", start, end); } } } @@ -198,16 +208,14 @@ public TrafficStatus calculateStatus(List sqsMessages) { long t = findOldestQueueTimestamp(sqsMessages); LOGGER.info("Traffic calculation starting with t={} (oldest SQS message)", t); - // Define time windows - long currentWindowStart = t - this.currentEvaluationWindowSeconds - getWhitelistDuration(t, t - this.currentEvaluationWindowSeconds); // for range [t-23h55m, t+5m] - long pastWindowStart = currentWindowStart - this.previousEvaluationWindowSeconds - getWhitelistDuration(currentWindowStart, currentWindowStart - this.previousEvaluationWindowSeconds); // for range [t-47h55m, t-23h55m] + // define start time of the evaluation window [t-24h, t+5m] + long windowStart = t - this.evaluationWindowSeconds - getWhitelistDuration(t, t - this.evaluationWindowSeconds); - // Evict old cache entries (older than past window start) - evictOldCacheEntries(pastWindowStart); + // Evict old cache entries (older than window start) + evictOldCacheEntries(windowStart); // Process delta files and count - int sumCurrent = 0; - int sumPast = 0; + int sum = 0; for (String s3Path : deltaS3Paths) { List timestamps = getTimestampsFromFile(s3Path); @@ -215,7 +223,7 @@ public TrafficStatus calculateStatus(List sqsMessages) { boolean shouldStop = false; for (long ts : timestamps) { // Stop condition: record is older than our 48h window - if (ts < pastWindowStart) { + if (ts < windowStart) { LOGGER.debug("Stopping delta file processing at timestamp {} (older than t-48h)", ts); break; } @@ -225,15 +233,11 @@ public TrafficStatus calculateStatus(List sqsMessages) { continue; } - // Count for sumCurrent: [t-24h, t] - if (ts >= currentWindowStart && ts <= t) { - sumCurrent++; + // Count for sumCurrent: [t-24h, t+5m] + if (ts >= windowStart && ts <= t) { + sum++; } - // Count for sumPast: [t-48h, t-24h] - if (ts >= pastWindowStart && ts < currentWindowStart) { - sumPast++; - } } if (shouldStop) { @@ -245,14 +249,14 @@ public TrafficStatus calculateStatus(List sqsMessages) { if (sqsMessages != null && !sqsMessages.isEmpty()) { int sqsCount = countSqsMessages( sqsMessages, t); - sumCurrent += sqsCount; + sum += sqsCount; } // Determine status - TrafficStatus status = determineStatus(sumCurrent, sumPast); + TrafficStatus status = determineStatus(sum, this.baselineTraffic); - LOGGER.info("Traffic calculation complete: sumCurrent={}, sumPast={}, status={}", - sumCurrent, sumPast, status); + LOGGER.info("Traffic calculation complete: sum={}, baselineTraffic={}, thresholdMultiplier={}, status={}", + sum, this.baselineTraffic, this.thresholdMultiplier, status); return status; From 843f41f4a0cb4802f6b5caf1ea5a44e16e93dba2 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 25 Nov 2025 16:11:51 -0700 Subject: [PATCH 15/28] naming improvements --- .../optout/vertx/OptOutTrafficCalculator.java | 31 +- .../vertx/OptOutTrafficCalculatorTest.java | 355 ++++++++---------- 2 files changed, 174 insertions(+), 212 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 9452ec25..568a8c5a 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -4,6 +4,7 @@ import com.uid2.shared.optout.OptOutCollection; import com.uid2.shared.optout.OptOutEntry; import com.uid2.shared.optout.OptOutUtils; +import com.uid2.optout.Const; import io.vertx.core.json.JsonObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,7 +25,7 @@ * Calculates opt-out traffic patterns to determine DEFAULT or DELAYED_PROCESSING status. * * Compares recent ~24h traffic (sumCurrent) against a configurable baseline (baselineTraffic) of expected traffic in 24 hours. - * The baseline is calculated as baselineTraffic*thresholdMultiplier. + * The baseline is multiplied by (thresholdMultiplier) to determine the threshold. * sumCurrent excludes records in whitelist ranges (surge windows determined by engineers). * * Returns DELAYED_PROCESSING if sumCurrent >= thresholdMultiplier * baselineTraffic, indicating abnormal traffic spike. @@ -76,7 +77,7 @@ public MalformedTrafficCalcConfigException(String message) { /** * Constructor for OptOutTrafficCalculator * - * @param cloudStorage Cloud storage for reading delta files and whitelist from S3 + * @param cloudStorage Cloud storage for reading delta files * @param s3DeltaPrefix S3 prefix for delta files * @param trafficCalcConfigS3Path S3 path for traffic calc config */ @@ -94,13 +95,13 @@ public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, * Reload traffic calc config from ConfigMap. * Expected format: * { - * "traffic_calc_current_evaluation_window_seconds": 86400, - * "traffic_calc_previous_evaluation_window_seconds": 86400, + * "traffic_calc_evaluation_window_seconds": 86400, + * "traffic_calc_baseline_traffic": 100, + * "traffic_calc_threshold_multiplier": 5, * "traffic_calc_whitelist_ranges": [ * [startTimestamp1, endTimestamp1], * [startTimestamp2, endTimestamp2] * ], - * "traffic_calc_threshold_multiplier": 5 * } * * Can be called periodically to pick up config changes without restarting. @@ -109,27 +110,27 @@ public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException LOGGER.info("Loading traffic calc config from ConfigMap"); try (InputStream is = Files.newInputStream(Paths.get(trafficCalcConfigPath))) { String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); - JsonObject whitelistConfig = new JsonObject(content); + JsonObject trafficCalcConfig = new JsonObject(content); // Validate required fields exist - if (!whitelistConfig.containsKey("traffic_calc_evaluation_window_seconds")) { + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_evaluation_window_seconds"); } - if (!whitelistConfig.containsKey("traffic_calc_baseline_traffic")) { + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) { throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_baseline_traffic"); } - if (!whitelistConfig.containsKey("traffic_calc_threshold_multiplier")) { + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_threshold_multiplier"); } - if (!whitelistConfig.containsKey("traffic_calc_whitelist_ranges")) { + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcWhitelistRangesProp)) { throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_whitelist_ranges"); } - this.evaluationWindowSeconds = whitelistConfig.getInteger("traffic_calc_evaluation_window_seconds"); - this.baselineTraffic = whitelistConfig.getInteger("traffic_calc_baseline_traffic"); - this.thresholdMultiplier = whitelistConfig.getInteger("traffic_calc_threshold_multiplier"); + this.evaluationWindowSeconds = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp); + this.baselineTraffic = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcBaselineTrafficProp); + this.thresholdMultiplier = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcThresholdMultiplierProp); - List> ranges = parseWhitelistRanges(whitelistConfig); + List> ranges = parseWhitelistRanges(trafficCalcConfig); this.whitelistRanges = ranges; LOGGER.info("Successfully loaded traffic calc config from ConfigMap: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, whitelistRanges={}", @@ -151,7 +152,7 @@ List> parseWhitelistRanges(JsonObject config) throws MalformedTraffic List> ranges = new ArrayList<>(); try { - var rangesArray = config.getJsonArray("traffic_calc_whitelist_ranges"); + var rangesArray = config.getJsonArray(Const.Config.OptOutTrafficCalcWhitelistRangesProp); if (rangesArray != null) { for (int i = 0; i < rangesArray.size(); i++) { var rangeArray = rangesArray.getJsonArray(i); diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java index f35b7bf7..f2fb25c4 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -4,6 +4,7 @@ import com.uid2.shared.cloud.ICloudStorage; import com.uid2.shared.optout.OptOutCollection; import com.uid2.shared.optout.OptOutEntry; +import com.uid2.optout.Const; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import java.nio.file.Files; @@ -39,11 +40,20 @@ public class OptOutTrafficCalculatorTest { private static final String S3_DELTA_PREFIX = "optout-v2/delta/"; private static final String TRAFFIC_CONFIG_PATH = "./traffic-config.json"; + private static final int BASELINE_TRAFFIC = 100; + private static final int THRESHOLD_MULTIPLIER = 5; + private static final int EVALUATION_WINDOW_SECONDS = 24 * 3600; @BeforeEach void setUp() { + // default config + JsonObject config = new JsonObject(); + config.put(Const.Config.OptOutTrafficCalcBaselineTrafficProp, BASELINE_TRAFFIC); + config.put(Const.Config.OptOutTrafficCalcThresholdMultiplierProp, THRESHOLD_MULTIPLIER); + config.put(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp, EVALUATION_WINDOW_SECONDS); + config.put(Const.Config.OptOutTrafficCalcWhitelistRangesProp, new JsonArray()); try { - createTrafficConfigFile("{}"); + createTrafficConfigFile(config.toString()); } catch (Exception e) { throw new RuntimeException(e); } @@ -69,6 +79,40 @@ private void createTrafficConfigFile(String content) { } } + /** + * Helper to create config by merging partial JSON with defaults + */ + private void createConfigFromPartialJson(String partialJson) { + JsonObject partial = new JsonObject(partialJson); + JsonObject config = new JsonObject(); + + // Set defaults + if (!partial.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) { + config.put(Const.Config.OptOutTrafficCalcBaselineTrafficProp, BASELINE_TRAFFIC); + } + if (!partial.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { + config.put(Const.Config.OptOutTrafficCalcThresholdMultiplierProp, THRESHOLD_MULTIPLIER); + } + if (!partial.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { + config.put(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp, EVALUATION_WINDOW_SECONDS); + } + if (!partial.containsKey(Const.Config.OptOutTrafficCalcWhitelistRangesProp)) { + config.put(Const.Config.OptOutTrafficCalcWhitelistRangesProp, new JsonArray()); + } + + // Merge in partial config (overrides defaults) + partial.forEach(entry -> config.put(entry.getKey(), entry.getValue())); + + createTrafficConfigFile(config.toString()); + } + + /** + * Helper to create config with custom threshold + */ + private void createConfigWithThreshold(int threshold) { + createConfigFromPartialJson("{\"" + Const.Config.OptOutTrafficCalcThresholdMultiplierProp + "\": " + threshold + "}"); + } + // ============================================================================ // SECTION 1: Constructor & Initialization Tests // ============================================================================ @@ -90,7 +134,7 @@ void testConstructor_defaultThreshold() throws Exception { @Test void testConstructor_customThreshold() throws Exception { // Setup - custom threshold of 10 - createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 10}"); + createConfigWithThreshold(10); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -106,11 +150,12 @@ void testConstructor_whitelistLoadFailure() throws Exception { // Setup - whitelist load failure createTrafficConfigFile("Invalid JSON"); assertThrows(MalformedTrafficCalcConfigException.class, () -> { - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); }); - createTrafficConfigFile("{}"); + // Create valid config to test reload failure + createConfigFromPartialJson("{}"); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -288,7 +333,7 @@ void testIsInWhitelist_withinSingleRange() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -307,7 +352,7 @@ void testIsInWhitelist_exactlyAtStart() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -326,7 +371,7 @@ void testIsInWhitelist_exactlyAtEnd() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -345,7 +390,7 @@ void testIsInWhitelist_beforeRange() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -364,7 +409,7 @@ void testIsInWhitelist_afterRange() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -384,7 +429,7 @@ void testIsInWhitelist_betweenRanges() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -395,9 +440,7 @@ void testIsInWhitelist_betweenRanges() throws Exception { @Test void testIsInWhitelist_emptyRanges() throws Exception { - // Setup - no whitelist loaded - createTrafficConfigFile("{}"); - + // Setup uses default config from setUp() which has empty whitelist OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -413,7 +456,7 @@ void testIsInWhitelist_nullRanges() throws Exception { "traffic_calc_whitelist_ranges": null } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -433,7 +476,7 @@ void testIsInWhitelist_invalidRangeSize() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -455,7 +498,7 @@ void testIsInWhitelist_multipleRanges() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -491,7 +534,7 @@ void testGetWhitelistDuration_rangeFullyWithinWindow() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -513,7 +556,7 @@ void testGetWhitelistDuration_rangePartiallyOverlapsStart() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -535,7 +578,7 @@ void testGetWhitelistDuration_rangePartiallyOverlapsEnd() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -557,7 +600,7 @@ void testGetWhitelistDuration_rangeCompletelyOutsideWindow() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -580,7 +623,7 @@ void testGetWhitelistDuration_multipleRanges() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -602,7 +645,7 @@ void testGetWhitelistDuration_rangeSpansEntireWindow() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -707,7 +750,7 @@ void testDetermineStatus_sumCurrentZero() throws Exception { }) void testDetermineStatus_variousThresholds(int threshold, int sumCurrent, int sumPast, String expectedStatus) throws Exception { // Setup - various thresholds - createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": " + threshold + "}"); + createConfigWithThreshold(threshold); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -746,7 +789,7 @@ void testReloadWhitelist_success() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -759,7 +802,7 @@ void testReloadWhitelist_success() throws Exception { ] } """; - createTrafficConfigFile(newWhitelistJson); + createConfigFromPartialJson(newWhitelistJson); // Act - reload the whitelist calculator.reloadTrafficCalcConfig(); @@ -778,7 +821,7 @@ void testReloadWhitelist_failure() throws Exception { ] } """; - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -793,8 +836,65 @@ void testReloadWhitelist_failure() throws Exception { } + @Test + public void testReloadWhitelist_failure_missingKeys() throws Exception { + // Setup + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act & Assert missing threshold multiplier + createTrafficConfigFile("{\"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_baseline_traffic\": 100, \"traffic_calc_whitelist_ranges\": [ [1000, 2000] ]}"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + + // Act & Assert missing evaluation window seconds + createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_baseline_traffic\": 100, \"traffic_calc_whitelist_ranges\": [ [1000, 2000] ]}"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + + // Act & Assert missing baseline traffic + createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_whitelist_ranges\": [ [1000, 2000] ]}"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + + // Act & Assert missing whitelist ranges + createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_baseline_traffic\": 100}"); + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + } + + @Test + public void testReloadWhitelist_failure_misorderedRanges() throws Exception { + // Setup - misordered ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + createConfigFromPartialJson("{\"traffic_calc_whitelist_ranges\": [ [2000, 1000] ]}"); + + // Act & Assert + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + } + + @Test + public void testReloadWhitelist_failure_rangeTooLong() throws Exception { + // Setup - range greater than 24 hours + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + createConfigFromPartialJson("{\"traffic_calc_whitelist_ranges\": [ [1000, 200000] ]}"); + + // Act & Assert + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.reloadTrafficCalcConfig(); + }); + } + // ============================================================================ - // SECTION 7: Cache Management Tests + // SECTION 7: Cache Management Tests (also tested in section 9) // ============================================================================ @Test @@ -886,14 +986,9 @@ void testCalculateStatus_normalTraffic() throws Exception { // Create delta files with timestamps distributed over 48 hours List timestamps = new ArrayList<>(); - // Past window: t-47h to t-25h (add 50 entries) - for (int i = 0; i < 50; i++) { - timestamps.add(t - 47*3600 + i * 1000); - } - - // Current window: t-23h to t-1h (add 100 entries - 2x past) - for (int i = 0; i < 100; i++) { - timestamps.add(t - 23*3600 + i * 1000); + // add 499 entries in current window + for (int i = 0; i < 49; i++) { + timestamps.add(t - 23*3600 + i * 60); } byte[] deltaFileBytes = createDeltaFileBytes(timestamps); @@ -922,14 +1017,9 @@ void testCalculateStatus_delayedProcessing() throws Exception { // Create delta files with spike in current window List timestamps = new ArrayList<>(); - // Past window: t-47h to t-25h (add 10 entries) - for (int i = 0; i < 10; i++) { - timestamps.add(t - 47*3600 + i * 1000); - } - - // Current window: t-23h to t-1h (add 100 entries - 10x past!) - for (int i = 0; i < 100; i++) { - timestamps.add(t - 23*3600 + i * 1000); + // add 500 entries in current window + for (int i = 0; i < 500; i++) { + timestamps.add(t - 23*3600 + i * 60); } byte[] deltaFileBytes = createDeltaFileBytes(timestamps); @@ -1002,13 +1092,9 @@ void testCalculateStatus_multipleSqsMessages() throws Exception { long t = currentTime; List timestamps = new ArrayList<>(); - // Past window - 10 entries - for (int i = 0; i < 10; i++) { - timestamps.add(t - 48*3600 + i * 1000); - } - // Current window - 20 entries - for (int i = 0; i < 20; i++) { - timestamps.add(t - 12*3600 + i * 1000); + // add 470 entries in window + for (int i = 0; i < 470; i++) { + timestamps.add(t - 24*3600 + i * 60); } byte[] deltaFileBytes = createDeltaFileBytes(timestamps); @@ -1020,10 +1106,9 @@ void testCalculateStatus_multipleSqsMessages() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - multiple SQS messages, oldest determines t - // Add enough SQS messages to push total count over DELAYED_PROCESSING threshold + // Add 30 SQS entries in [t, t+5min] List sqsMessages = new ArrayList<>(); - for (int i = 0; i < 101; i++) { + for (int i = 0; i < 30; i++) { sqsMessages.add(createSqsMessage(t - i * 10)); } OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); @@ -1038,7 +1123,7 @@ void testCalculateStatus_withWhitelist() throws Exception { long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; - // Whitelist that covers part of current window + // Whitelist that covers part of window String whitelistJson = String.format(""" { "traffic_calc_whitelist_ranges": [ @@ -1048,22 +1133,18 @@ void testCalculateStatus_withWhitelist() throws Exception { """, t - 12*3600, t - 6*3600); List timestamps = new ArrayList<>(); - // Past window - 20 entries - for (int i = 0; i < 20; i++) { - timestamps.add(t - 48*3600 + i * 100); - } - // Current window - 100 entries (50 in whitelist range, 50 outside) - for (int i = 0; i < 50; i++) { - timestamps.add(t - 12*3600 + i * 100); // In whitelist + // window - 600 entries (300 in whitelist range, 300 outside) + for (int i = 0; i < 300; i++) { + timestamps.add(t - 12*3600 + i); } - for (int i = 0; i < 50; i++) { - timestamps.add(t - 3600 + i * 100); // Outside whitelist + for (int i = 0; i < 300; i++) { + timestamps.add(t - 3600 + i); } byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - createTrafficConfigFile(whitelistJson); + createConfigFromPartialJson(whitelistJson); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/delta-001.dat")); when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); @@ -1076,8 +1157,8 @@ void testCalculateStatus_withWhitelist() throws Exception { OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); // Assert - should filter out whitelisted entries - // Only ~50 from current window count (not whitelisted) + 1 SQS = 51 - // 51 < 5 * 20 = 100, so DEFAULT + // Only 300 from window count (not whitelisted) + 1 SQS = 301 + // 301 < 5*100, so DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); } @@ -1219,20 +1300,18 @@ void testCalculateStatus_multipleDeltaFiles() throws Exception { } @Test - void testCalculateStatus_windowBoundaryTimestamps() throws Exception { - // Setup - create delta files with some entries + void testCalculateStatus_windowBoundaryTimestamp() throws Exception { + // Setup - create delta file with timestamps at window boundary long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; - long currentWindowStart = t - 24*3600 + 300; // t-23h55m - long pastWindowStart = currentWindowStart - 24*3600; // t-47h55m - - List timestamps = Arrays.asList( - t, - currentWindowStart, - pastWindowStart, - t - 24*3600, - t - 48*3600 - ); + long currentWindowStart = t - 24*3600; + List timestamps = new ArrayList<>(); + for (int i = 0; i < 250; i++) { + timestamps.add(t); + } + for (int i = 0; i < 250; i++) { + timestamps.add(currentWindowStart); + } byte[] deltaFileBytes = createDeltaFileBytes(timestamps); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); @@ -1247,7 +1326,7 @@ void testCalculateStatus_windowBoundaryTimestamps() throws Exception { OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); // Assert - DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } @Test @@ -1278,122 +1357,4 @@ void testCalculateStatus_timestampsCached() throws Exception { assertEquals(2, stats.get("total_cached_timestamps")); } - @Test - void testCalculateStatus_whitelistReducesPreviousWindowBaseline_customWindows() throws Exception { - // Setup - test with custom 3-hour evaluation windows - // Whitelist in previous window reduces baseline, causing DELAYED_PROCESSING - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - long threeHours = 3 * 3600; // 10800 seconds - - // Whitelist covering most of the PREVIOUS window (t-6h to t-4h) - // This reduces the baseline count in the previous window - String whitelistJson = String.format(""" - { - "traffic_calc_current_evaluation_window_seconds": %d, - "traffic_calc_previous_evaluation_window_seconds": %d, - "traffic_calc_whitelist_ranges": [ - [%d, %d] - ] - } - """, threeHours, threeHours, t - 6*3600, t - 4*3600); - - List timestamps = new ArrayList<>(); - - // Previous window (t-6h to t-3h): Add 100 entries - // 80 of these will be whitelisted (between t-6h and t-4h) - // Only 20 will count toward baseline - for (int i = 0; i < 80; i++) { - timestamps.add(t - 6*3600 + i); // Whitelisted entries in previous window - } - for (int i = 0; i < 20; i++) { - timestamps.add(t - 4*3600 + i); // Non-whitelisted entries in previous window - } - - // Current window (t-3h to t): Add 120 entries (none whitelisted) - // This creates a spike: 120 + 1 SQS >= 5 * 20 = 100 - for (int i = 0; i < 120; i++) { - timestamps.add(t - threeHours + i); // Current window entries - } - - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - createTrafficConfigFile(whitelistJson); - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); - - // Assert - DELAYED_PROCESSING - // With 3-hour evaluation windows: - // Previous window (t-6h to t-3h): 100 total entries, 80 whitelisted → 20 counted - // Current window (t-3h to t): 120 entries (none whitelisted) + 1 SQS = 121 - // 121 >= 5 * 20 = 100, so DELAYED_PROCESSING - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); - } - - @Test - void testCalculateStatus_whitelistReducesCurrentWindowBaseline_customWindows() throws Exception { - // Setup - test with custom 3-hour evaluation windows - // Whitelist in previous window reduces baseline, causing DELAYED_PROCESSING - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - long threeHours = 3 * 3600; // 10800 seconds - - // Whitelist covering most of the CURRENT window (t-3h to t+5m) - // This reduces the baseline count in the previous window - String whitelistJson = String.format(""" - { - "traffic_calc_current_evaluation_window_seconds": %d, - "traffic_calc_previous_evaluation_window_seconds": %d, - "traffic_calc_whitelist_ranges": [ - [%d, %d] - ] - } - """, threeHours, threeHours, t - 3*3600, t - 1*3600); - - List timestamps = new ArrayList<>(); - - // Current window (t-3h to t+5m): Add 100 entries - // 80 of these will be whitelisted (between t-3h and t-1h) - // Only 20 will count toward baseline - for (int i = 0; i < 80; i++) { - timestamps.add(t - 3*3600 + i); // Whitelisted entries in current window - } - for (int i = 0; i < 20; i++) { - timestamps.add(t - 1*3600 + i); // Non-whitelisted entries in current window - } - - // Previous window (t-6h to t-3h): Add 10 entries (none whitelisted) - for (int i = 0; i < 10; i++) { - timestamps.add(t - 6*3600 + i); // Non-whitelisted entries in previous window - } - - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - createTrafficConfigFile(whitelistJson); - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); - - // Assert - DEFAULT - // With 3-hour evaluation windows: - // Previous window (t-6h to t-3h): 10 entries (none whitelisted) - // Current window (t-3h to t): 20 entries (non-whitelisted) + 1 SQS = 21 - // 21 < 5 * 10 = 50, so DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } } From 1477e2f6d004222d31e1a6f4ce92a99493fdc97e Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 25 Nov 2025 16:19:42 -0700 Subject: [PATCH 16/28] naming improvements --- .../vertx/OptOutTrafficCalculatorTest.java | 190 +++++++++--------- 1 file changed, 95 insertions(+), 95 deletions(-) diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java index f2fb25c4..97b8c971 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -146,8 +146,8 @@ void testConstructor_customThreshold() throws Exception { } @Test - void testConstructor_whitelistLoadFailure() throws Exception { - // Setup - whitelist load failure + void testConstructor_trafficCalcConfigLoadFailure() throws Exception { + // Setup - traffic calc config load failure createTrafficConfigFile("Invalid JSON"); assertThrows(MalformedTrafficCalcConfigException.class, () -> { new OptOutTrafficCalculator( @@ -166,11 +166,11 @@ void testConstructor_whitelistLoadFailure() throws Exception { } // ============================================================================ - // SECTION 2: parseWhitelistRanges() + // SECTION 2: parseTrafficCalcConfigRanges() // ============================================================================ @Test - void testParseWhitelistRanges_emptyConfig() throws Exception { + void testParseTrafficCalcConfigRanges_emptyConfig() throws Exception { // Setup - no config OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -184,7 +184,7 @@ void testParseWhitelistRanges_emptyConfig() throws Exception { } @Test - void testParseWhitelistRanges_singleRange() throws Exception { + void testParseTrafficCalcConfigRanges_singleRange() throws Exception { // Setup - single range OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -204,7 +204,7 @@ void testParseWhitelistRanges_singleRange() throws Exception { } @Test - void testParseWhitelistRanges_multipleRanges() throws Exception { + void testParseTrafficCalcConfigRanges_multipleRanges() throws Exception { // Setup - multiple ranges OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -227,7 +227,7 @@ void testParseWhitelistRanges_multipleRanges() throws Exception { } @Test - void testParseWhitelistRanges_misorderedRange() throws Exception { + void testParseTrafficCalcConfigRanges_misorderedRange() throws Exception { // Setup - range with end < start is malformed OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -244,7 +244,7 @@ void testParseWhitelistRanges_misorderedRange() throws Exception { } @Test - void testParseWhitelistRanges_rangeTooLong() throws Exception { + void testParseTrafficCalcConfigRanges_rangeTooLong() throws Exception { // Setup - range longer than 24 hours is malformed OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -261,7 +261,7 @@ void testParseWhitelistRanges_rangeTooLong() throws Exception { } @Test - void testParseWhitelistRanges_sortsByStartTime() throws Exception { + void testParseTrafficCalcConfigRanges_sortsByStartTime() throws Exception { // Setup - ranges added out of order OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -284,7 +284,7 @@ void testParseWhitelistRanges_sortsByStartTime() throws Exception { } @Test - void testParseWhitelistRanges_invalidRangeTooFewElements() throws Exception { + void testParseTrafficCalcConfigRanges_invalidRangeTooFewElements() throws Exception { // Setup - invalid range with only 1 element; OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -304,7 +304,7 @@ void testParseWhitelistRanges_invalidRangeTooFewElements() throws Exception { } @Test - void testParseWhitelistRanges_nullArray() throws Exception { + void testParseTrafficCalcConfigRanges_nullArray() throws Exception { // Setup - null array OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -320,20 +320,20 @@ void testParseWhitelistRanges_nullArray() throws Exception { } // ============================================================================ - // SECTION 3: isInWhitelist() + // SECTION 3: isInTrafficCalcConfig() // ============================================================================ @Test - void testIsInWhitelist_withinSingleRange() throws Exception { - // Setup - load whitelist with single range [1000, 2000] - String whitelistJson = """ + void testIsInTrafficCalcConfig_withinSingleRange() throws Exception { + // Setup - load traffic calc config with single range [1000, 2000] + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [1000, 2000] ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -343,16 +343,16 @@ void testIsInWhitelist_withinSingleRange() throws Exception { } @Test - void testIsInWhitelist_exactlyAtStart() throws Exception { - // Setup - load whitelist with single range [1000, 2000] - String whitelistJson = """ + void testIsInTrafficCalcConfig_exactlyAtStart() throws Exception { + // Setup - load traffic calc config with single range [1000, 2000] + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [1000, 2000] ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -362,16 +362,16 @@ void testIsInWhitelist_exactlyAtStart() throws Exception { } @Test - void testIsInWhitelist_exactlyAtEnd() throws Exception { - // Setup - load whitelist with single range [1000, 2000] - String whitelistJson = """ + void testIsInTrafficCalcConfig_exactlyAtEnd() throws Exception { + // Setup - load traffic calc config with single range [1000, 2000] + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [1000, 2000] ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -381,16 +381,16 @@ void testIsInWhitelist_exactlyAtEnd() throws Exception { } @Test - void testIsInWhitelist_beforeRange() throws Exception { - // Setup - load whitelist with single range [1000, 2000] - String whitelistJson = """ + void testIsInTrafficCalcConfig_beforeRange() throws Exception { + // Setup - load traffic calc config with single range [1000, 2000] + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [1000, 2000] ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -400,16 +400,16 @@ void testIsInWhitelist_beforeRange() throws Exception { } @Test - void testIsInWhitelist_afterRange() throws Exception { - // Setup - load whitelist with single range [1000, 2000] - String whitelistJson = """ + void testIsInTrafficCalcConfig_afterRange() throws Exception { + // Setup - load traffic calc config with single range [1000, 2000] + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [1000, 2000] ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -419,9 +419,9 @@ void testIsInWhitelist_afterRange() throws Exception { } @Test - void testIsInWhitelist_betweenRanges() throws Exception { - // Setup - load whitelist with two ranges [1000, 2000] and [3000, 4000] - String whitelistJson = """ + void testIsInTrafficCalcConfig_betweenRanges() throws Exception { + // Setup - load traffic calc config with two ranges [1000, 2000] and [3000, 4000] + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [1000, 2000], @@ -429,7 +429,7 @@ void testIsInWhitelist_betweenRanges() throws Exception { ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -439,8 +439,8 @@ void testIsInWhitelist_betweenRanges() throws Exception { } @Test - void testIsInWhitelist_emptyRanges() throws Exception { - // Setup uses default config from setUp() which has empty whitelist + void testIsInTrafficCalcConfig_emptyRanges() throws Exception { + // Setup uses default config from setUp() which has empty traffic calc config ranges OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -449,14 +449,14 @@ void testIsInWhitelist_emptyRanges() throws Exception { } @Test - void testIsInWhitelist_nullRanges() throws Exception { - // Setup - no whitelist loaded (will fail and set empty) - String whitelistJson = """ + void testIsInTrafficCalcConfig_nullRanges() throws Exception { + // Setup - no traffic calc config ranges loaded (will fail and set empty) + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": null } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -466,9 +466,9 @@ void testIsInWhitelist_nullRanges() throws Exception { } @Test - void testIsInWhitelist_invalidRangeSize() throws Exception { - // Setup - load whitelist with invalid range (only 1 element) and valid range - String whitelistJson = """ + void testIsInTrafficCalcConfig_invalidRangeSize() throws Exception { + // Setup - load traffic calc config with invalid range (only 1 element) and valid range + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [1000], @@ -476,7 +476,7 @@ void testIsInWhitelist_invalidRangeSize() throws Exception { ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -487,9 +487,9 @@ void testIsInWhitelist_invalidRangeSize() throws Exception { } @Test - void testIsInWhitelist_multipleRanges() throws Exception { - // Setup - load whitelist with multiple ranges - String whitelistJson = """ + void testIsInTrafficCalcConfig_multipleRanges() throws Exception { + // Setup - load traffic calc config with multiple ranges + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [1000, 2000], @@ -498,7 +498,7 @@ void testIsInWhitelist_multipleRanges() throws Exception { ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -511,11 +511,11 @@ void testIsInWhitelist_multipleRanges() throws Exception { } // ============================================================================ - // SECTION 4: getWhitelistDuration() + // SECTION 4: getTrafficCalcConfigDuration() // ============================================================================ @Test - void testGetWhitelistDuration_noRanges() throws Exception { + void testGetTrafficCalcConfigDuration_noRanges() throws Exception { // Setup - no ranges OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -525,16 +525,16 @@ void testGetWhitelistDuration_noRanges() throws Exception { } @Test - void testGetWhitelistDuration_rangeFullyWithinWindow() throws Exception { + void testGetTrafficCalcConfigDuration_rangeFullyWithinWindow() throws Exception { // Setup - range fully within window - String whitelistJson = """ + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [6000, 7000] ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -547,16 +547,16 @@ void testGetWhitelistDuration_rangeFullyWithinWindow() throws Exception { } @Test - void testGetWhitelistDuration_rangePartiallyOverlapsStart() throws Exception { + void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsStart() throws Exception { // Setup - range partially overlaps start of window - String whitelistJson = """ + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [3000, 7000] ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -569,16 +569,16 @@ void testGetWhitelistDuration_rangePartiallyOverlapsStart() throws Exception { } @Test - void testGetWhitelistDuration_rangePartiallyOverlapsEnd() throws Exception { + void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsEnd() throws Exception { // Setup - range partially overlaps end of window - String whitelistJson = """ + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [8000, 12000] ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -591,16 +591,16 @@ void testGetWhitelistDuration_rangePartiallyOverlapsEnd() throws Exception { } @Test - void testGetWhitelistDuration_rangeCompletelyOutsideWindow() throws Exception { + void testGetTrafficCalcConfigDuration_rangeCompletelyOutsideWindow() throws Exception { // Setup - range completely outside window - String whitelistJson = """ + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [1000, 2000] ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -613,9 +613,9 @@ void testGetWhitelistDuration_rangeCompletelyOutsideWindow() throws Exception { } @Test - void testGetWhitelistDuration_multipleRanges() throws Exception { + void testGetTrafficCalcConfigDuration_multipleRanges() throws Exception { // Setup - multiple ranges - String whitelistJson = """ + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [6000, 7000], @@ -623,7 +623,7 @@ void testGetWhitelistDuration_multipleRanges() throws Exception { ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -636,16 +636,16 @@ void testGetWhitelistDuration_multipleRanges() throws Exception { } @Test - void testGetWhitelistDuration_rangeSpansEntireWindow() throws Exception { + void testGetTrafficCalcConfigDuration_rangeSpansEntireWindow() throws Exception { // Setup - range spans entire window - String whitelistJson = """ + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [3000, 12000] ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -653,7 +653,7 @@ void testGetWhitelistDuration_rangeSpansEntireWindow() throws Exception { // Act - window [5000, 10000], range [3000, 12000] long duration = calculator.getWhitelistDuration(10000L, 5000L); - // Assert - entire window is whitelisted = 5000 + // Assert - entire window is in traffic calc config ranges = 5000 assertEquals(5000L, duration); } @@ -779,9 +779,9 @@ void testDetermineStatus_largeNumbers() throws Exception { // ============================================================================ @Test - void testReloadWhitelist_success() throws Exception { - // Setup - initial whitelist - String whitelistJson = """ + void testReloadTrafficCalcConfig_success() throws Exception { + // Setup - initial traffic calc config + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [1000, 2000], @@ -789,39 +789,39 @@ void testReloadWhitelist_success() throws Exception { ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Change the whitelist to a new range - String newWhitelistJson = """ + // Change the traffic calc config to a new range + String newTrafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [5000, 6000] ] } """; - createConfigFromPartialJson(newWhitelistJson); + createConfigFromPartialJson(newTrafficCalcConfigJson); - // Act - reload the whitelist + // Act - reload the traffic calc config calculator.reloadTrafficCalcConfig(); - // Assert - verify new whitelist is loaded + // Assert - verify new traffic calc config is loaded assertTrue(calculator.isInWhitelist(5500L)); } @Test - void testReloadWhitelist_failure() throws Exception { - // Setup - initial whitelist - String whitelistJson = """ + void testReloadTrafficCalcConfig_failure() throws Exception { + // Setup - initial traffic calc config + String trafficCalcConfigJson = """ { "traffic_calc_whitelist_ranges": [ [1000, 2000] ] } """; - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -837,7 +837,7 @@ void testReloadWhitelist_failure() throws Exception { } @Test - public void testReloadWhitelist_failure_missingKeys() throws Exception { + public void testReloadTrafficCalcConfig_failure_missingKeys() throws Exception { // Setup OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -860,7 +860,7 @@ public void testReloadWhitelist_failure_missingKeys() throws Exception { calculator.reloadTrafficCalcConfig(); }); - // Act & Assert missing whitelist ranges + // Act & Assert missing traffic calc config ranges createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_baseline_traffic\": 100}"); assertThrows(MalformedTrafficCalcConfigException.class, () -> { calculator.reloadTrafficCalcConfig(); @@ -868,7 +868,7 @@ public void testReloadWhitelist_failure_missingKeys() throws Exception { } @Test - public void testReloadWhitelist_failure_misorderedRanges() throws Exception { + public void testReloadTrafficCalcConfig_failure_misorderedRanges() throws Exception { // Setup - misordered ranges OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -881,7 +881,7 @@ public void testReloadWhitelist_failure_misorderedRanges() throws Exception { } @Test - public void testReloadWhitelist_failure_rangeTooLong() throws Exception { + public void testReloadTrafficCalcConfig_failure_rangeTooLong() throws Exception { // Setup - range greater than 24 hours OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); @@ -1118,13 +1118,13 @@ void testCalculateStatus_multipleSqsMessages() throws Exception { } @Test - void testCalculateStatus_withWhitelist() throws Exception { + void testCalculateStatus_withTrafficCalcConfig() throws Exception { // Setup - create delta files with some entries long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; - // Whitelist that covers part of window - String whitelistJson = String.format(""" + // Traffic calc config that covers part of window + String trafficCalcConfigJson = String.format(""" { "traffic_calc_whitelist_ranges": [ [%d, %d] @@ -1134,7 +1134,7 @@ void testCalculateStatus_withWhitelist() throws Exception { List timestamps = new ArrayList<>(); - // window - 600 entries (300 in whitelist range, 300 outside) + // window - 600 entries (300 in traffic calc config range, 300 outside) for (int i = 0; i < 300; i++) { timestamps.add(t - 12*3600 + i); } @@ -1144,7 +1144,7 @@ void testCalculateStatus_withWhitelist() throws Exception { byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - createConfigFromPartialJson(whitelistJson); + createConfigFromPartialJson(trafficCalcConfigJson); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/delta-001.dat")); when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); @@ -1156,8 +1156,8 @@ void testCalculateStatus_withWhitelist() throws Exception { List sqsMessages = Arrays.asList(createSqsMessage(t)); OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); - // Assert - should filter out whitelisted entries - // Only 300 from window count (not whitelisted) + 1 SQS = 301 + // Assert - should filter out entries in traffic calc config ranges + // Only 300 from window count (not in traffic calc config ranges) + 1 SQS = 301 // 301 < 5*100, so DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); } From d75bc0d9a82a5bddd783b87fccbf21f6a3610f76 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 25 Nov 2025 17:15:53 -0700 Subject: [PATCH 17/28] naming improvements --- src/main/java/com/uid2/optout/Const.java | 2 +- .../optout/vertx/OptOutTrafficCalculator.java | 58 ++++---- .../vertx/OptOutTrafficCalculatorTest.java | 128 +++++++++--------- 3 files changed, 94 insertions(+), 94 deletions(-) diff --git a/src/main/java/com/uid2/optout/Const.java b/src/main/java/com/uid2/optout/Const.java index 6dacd9a1..114e5464 100644 --- a/src/main/java/com/uid2/optout/Const.java +++ b/src/main/java/com/uid2/optout/Const.java @@ -25,7 +25,7 @@ public static class Config extends com.uid2.shared.Const.Config { public static final String OptOutTrafficCalcBaselineTrafficProp = "traffic_calc_baseline_traffic"; public static final String OptOutTrafficCalcThresholdMultiplierProp = "traffic_calc_threshold_multiplier"; public static final String OptOutTrafficCalcEvaluationWindowSecondsProp = "traffic_calc_evaluation_window_seconds"; - public static final String OptOutTrafficCalcWhitelistRangesProp = "traffic_calc_whitelist_ranges"; + public static final String OptOutTrafficCalcAllowlistRangesProp = "traffic_calc_allowlist_ranges"; } public static class Event { diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 568a8c5a..2cf2792e 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -26,7 +26,7 @@ * * Compares recent ~24h traffic (sumCurrent) against a configurable baseline (baselineTraffic) of expected traffic in 24 hours. * The baseline is multiplied by (thresholdMultiplier) to determine the threshold. - * sumCurrent excludes records in whitelist ranges (surge windows determined by engineers). + * sumCurrent excludes records in allowlist ranges (surge windows determined by engineers). * * Returns DELAYED_PROCESSING if sumCurrent >= thresholdMultiplier * baselineTraffic, indicating abnormal traffic spike. */ @@ -42,7 +42,7 @@ public class OptOutTrafficCalculator { private int baselineTraffic; private int thresholdMultiplier; private int evaluationWindowSeconds; - private List> whitelistRanges; + private List> allowlistRanges; public enum TrafficStatus { DELAYED_PROCESSING, @@ -98,7 +98,7 @@ public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, * "traffic_calc_evaluation_window_seconds": 86400, * "traffic_calc_baseline_traffic": 100, * "traffic_calc_threshold_multiplier": 5, - * "traffic_calc_whitelist_ranges": [ + * "traffic_calc_allowlist_ranges": [ * [startTimestamp1, endTimestamp1], * [startTimestamp2, endTimestamp2] * ], @@ -122,18 +122,18 @@ public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_threshold_multiplier"); } - if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcWhitelistRangesProp)) { - throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_whitelist_ranges"); + if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) { + throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_allowlist_ranges"); } this.evaluationWindowSeconds = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp); this.baselineTraffic = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcBaselineTrafficProp); this.thresholdMultiplier = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcThresholdMultiplierProp); - List> ranges = parseWhitelistRanges(trafficCalcConfig); - this.whitelistRanges = ranges; + List> ranges = parseAllowlistRanges(trafficCalcConfig); + this.allowlistRanges = ranges; - LOGGER.info("Successfully loaded traffic calc config from ConfigMap: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, whitelistRanges={}", + LOGGER.info("Successfully loaded traffic calc config from ConfigMap: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, allowlistRanges={}", this.evaluationWindowSeconds, this.baselineTraffic, this.thresholdMultiplier, ranges.size()); } catch (MalformedTrafficCalcConfigException e) { @@ -146,13 +146,13 @@ public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException } /** - * Parse whitelist ranges from JSON config + * Parse allowlist ranges from JSON config */ - List> parseWhitelistRanges(JsonObject config) throws MalformedTrafficCalcConfigException { + List> parseAllowlistRanges(JsonObject config) throws MalformedTrafficCalcConfigException { List> ranges = new ArrayList<>(); try { - var rangesArray = config.getJsonArray(Const.Config.OptOutTrafficCalcWhitelistRangesProp); + var rangesArray = config.getJsonArray(Const.Config.OptOutTrafficCalcAllowlistRangesProp); if (rangesArray != null) { for (int i = 0; i < rangesArray.size(); i++) { var rangeArray = rangesArray.getJsonArray(i); @@ -161,18 +161,18 @@ List> parseWhitelistRanges(JsonObject config) throws MalformedTraffic long end = rangeArray.getLong(1); if(start >= end) { - LOGGER.error("Invalid whitelist range: start must be less than end: [{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("Invalid whitelist range at index " + i + ": start must be less than end"); + LOGGER.error("Invalid allowlist range: start must be less than end: [{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("Invalid allowlist range at index " + i + ": start must be less than end"); } if (end - start > 86400) { - LOGGER.error("Invalid whitelist range: range must be less than 24 hours: [{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("Invalid whitelist range at index " + i + ": range must be less than 24 hours"); + LOGGER.error("Invalid allowlist range: range must be less than 24 hours: [{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("Invalid allowlist range at index " + i + ": range must be less than 24 hours"); } List range = Arrays.asList(start, end); ranges.add(range); - LOGGER.info("Loaded whitelist range: [{}, {}]", start, end); + LOGGER.info("Loaded allowlist range: [{}, {}]", start, end); } } } @@ -181,8 +181,8 @@ List> parseWhitelistRanges(JsonObject config) throws MalformedTraffic ranges.sort(Comparator.comparing(range -> range.get(0))); } catch (Exception e) { - LOGGER.error("Failed to parse whitelist ranges", e); - throw new MalformedTrafficCalcConfigException("Failed to parse whitelist ranges: " + e.getMessage()); + LOGGER.error("Failed to parse allowlist ranges", e); + throw new MalformedTrafficCalcConfigException("Failed to parse allowlist ranges: " + e.getMessage()); } return ranges; @@ -210,7 +210,7 @@ public TrafficStatus calculateStatus(List sqsMessages) { LOGGER.info("Traffic calculation starting with t={} (oldest SQS message)", t); // define start time of the evaluation window [t-24h, t+5m] - long windowStart = t - this.evaluationWindowSeconds - getWhitelistDuration(t, t - this.evaluationWindowSeconds); + long windowStart = t - this.evaluationWindowSeconds - getAllowlistDuration(t, t - this.evaluationWindowSeconds); // Evict old cache entries (older than window start) evictOldCacheEntries(windowStart); @@ -229,8 +229,8 @@ public TrafficStatus calculateStatus(List sqsMessages) { break; } - // skip records in whitelisted ranges - if (isInWhitelist(ts)) { + // skip records in allowlisted ranges + if (isInAllowlist(ts)) { continue; } @@ -339,11 +339,11 @@ private List readTimestampsFromS3(String s3Path) throws IOException { } /** - * Calculate total duration of whitelist ranges that overlap with the given time window. + * Calculate total duration of allowlist ranges that overlap with the given time window. */ - long getWhitelistDuration(long t, long windowStart) { + long getAllowlistDuration(long t, long windowStart) { long totalDuration = 0; - for (List range : this.whitelistRanges) { + for (List range : this.allowlistRanges) { long start = range.get(0); long end = range.get(1); @@ -413,7 +413,7 @@ private int countSqsMessages(List sqsMessages, long t) { continue; } - if (isInWhitelist(ts)) { + if (isInAllowlist(ts)) { continue; } count++; @@ -425,14 +425,14 @@ private int countSqsMessages(List sqsMessages, long t) { } /** - * Check if a timestamp falls within any whitelist range + * Check if a timestamp falls within any allowlist range */ - boolean isInWhitelist(long timestamp) { - if (whitelistRanges == null || whitelistRanges.isEmpty()) { + boolean isInAllowlist(long timestamp) { + if (allowlistRanges == null || allowlistRanges.isEmpty()) { return false; } - for (List range : whitelistRanges) { + for (List range : allowlistRanges) { if (range.size() < 2) { continue; } diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java index 97b8c971..0824c3b2 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -51,7 +51,7 @@ void setUp() { config.put(Const.Config.OptOutTrafficCalcBaselineTrafficProp, BASELINE_TRAFFIC); config.put(Const.Config.OptOutTrafficCalcThresholdMultiplierProp, THRESHOLD_MULTIPLIER); config.put(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp, EVALUATION_WINDOW_SECONDS); - config.put(Const.Config.OptOutTrafficCalcWhitelistRangesProp, new JsonArray()); + config.put(Const.Config.OptOutTrafficCalcAllowlistRangesProp, new JsonArray()); try { createTrafficConfigFile(config.toString()); } catch (Exception e) { @@ -96,8 +96,8 @@ private void createConfigFromPartialJson(String partialJson) { if (!partial.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { config.put(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp, EVALUATION_WINDOW_SECONDS); } - if (!partial.containsKey(Const.Config.OptOutTrafficCalcWhitelistRangesProp)) { - config.put(Const.Config.OptOutTrafficCalcWhitelistRangesProp, new JsonArray()); + if (!partial.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) { + config.put(Const.Config.OptOutTrafficCalcAllowlistRangesProp, new JsonArray()); } // Merge in partial config (overrides defaults) @@ -177,7 +177,7 @@ void testParseTrafficCalcConfigRanges_emptyConfig() throws Exception { JsonObject emptyConfig = new JsonObject(); // Act - List> ranges = calculator.parseWhitelistRanges(emptyConfig); + List> ranges = calculator.parseAllowlistRanges(emptyConfig); // Assert - empty ranges assertTrue(ranges.isEmpty()); @@ -192,10 +192,10 @@ void testParseTrafficCalcConfigRanges_singleRange() throws Exception { JsonObject configWithRanges = new JsonObject(); JsonArray ranges = new JsonArray() .add(new JsonArray().add(1000L).add(2000L)); - configWithRanges.put("traffic_calc_whitelist_ranges", ranges); + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); // Act - List> result = calculator.parseWhitelistRanges(configWithRanges); + List> result = calculator.parseAllowlistRanges(configWithRanges); // Assert - single range assertEquals(1, result.size()); @@ -214,10 +214,10 @@ void testParseTrafficCalcConfigRanges_multipleRanges() throws Exception { .add(new JsonArray().add(1000L).add(2000L)) .add(new JsonArray().add(3000L).add(4000L)) .add(new JsonArray().add(5000L).add(6000L)); - configWithRanges.put("traffic_calc_whitelist_ranges", ranges); + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); // Act - List> result = calculator.parseWhitelistRanges(configWithRanges); + List> result = calculator.parseAllowlistRanges(configWithRanges); // Assert - multiple ranges assertEquals(3, result.size()); @@ -235,11 +235,11 @@ void testParseTrafficCalcConfigRanges_misorderedRange() throws Exception { JsonObject configWithRanges = new JsonObject(); JsonArray ranges = new JsonArray() .add(new JsonArray().add(2000L).add(1000L)); // End before start - configWithRanges.put("traffic_calc_whitelist_ranges", ranges); + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); // Act assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.parseWhitelistRanges(configWithRanges); + calculator.parseAllowlistRanges(configWithRanges); }); } @@ -252,11 +252,11 @@ void testParseTrafficCalcConfigRanges_rangeTooLong() throws Exception { JsonObject configWithRanges = new JsonObject(); JsonArray ranges = new JsonArray() .add(new JsonArray().add(2000L).add(200000L)); // Longer than 24 hours - configWithRanges.put("traffic_calc_whitelist_ranges", ranges); + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); // Act assertThrows(MalformedTrafficCalcConfigException.class, () -> { - calculator.parseWhitelistRanges(configWithRanges); + calculator.parseAllowlistRanges(configWithRanges); }); } @@ -271,10 +271,10 @@ void testParseTrafficCalcConfigRanges_sortsByStartTime() throws Exception { .add(new JsonArray().add(5000L).add(6000L)) .add(new JsonArray().add(1000L).add(2000L)) .add(new JsonArray().add(3000L).add(4000L)); - configWithRanges.put("traffic_calc_whitelist_ranges", ranges); + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); // Act - List> result = calculator.parseWhitelistRanges(configWithRanges); + List> result = calculator.parseAllowlistRanges(configWithRanges); // Assert - should be sorted by start time assertEquals(3, result.size()); @@ -293,10 +293,10 @@ void testParseTrafficCalcConfigRanges_invalidRangeTooFewElements() throws Except JsonArray ranges = new JsonArray() .add(new JsonArray().add(1000L)) // Only 1 element .add(new JsonArray().add(2000L).add(3000L)); // Valid - configWithRanges.put("traffic_calc_whitelist_ranges", ranges); + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); // Act - List> result = calculator.parseWhitelistRanges(configWithRanges); + List> result = calculator.parseAllowlistRanges(configWithRanges); // Assert - should skip invalid range assertEquals(1, result.size()); @@ -310,10 +310,10 @@ void testParseTrafficCalcConfigRanges_nullArray() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); JsonObject configWithRanges = new JsonObject(); - configWithRanges.put("traffic_calc_whitelist_ranges", (JsonArray) null); + configWithRanges.put("traffic_calc_allowlist_ranges", (JsonArray) null); // Act - List> result = calculator.parseWhitelistRanges(configWithRanges); + List> result = calculator.parseAllowlistRanges(configWithRanges); // Assert - empty ranges assertTrue(result.isEmpty()); @@ -328,7 +328,7 @@ void testIsInTrafficCalcConfig_withinSingleRange() throws Exception { // Setup - load traffic calc config with single range [1000, 2000] String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [1000, 2000] ] } @@ -339,7 +339,7 @@ void testIsInTrafficCalcConfig_withinSingleRange() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - true when within range - assertTrue(calculator.isInWhitelist(1500L)); + assertTrue(calculator.isInAllowlist(1500L)); } @Test @@ -347,7 +347,7 @@ void testIsInTrafficCalcConfig_exactlyAtStart() throws Exception { // Setup - load traffic calc config with single range [1000, 2000] String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [1000, 2000] ] } @@ -358,7 +358,7 @@ void testIsInTrafficCalcConfig_exactlyAtStart() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - true when exactly at start of range - assertTrue(calculator.isInWhitelist(1000L)); + assertTrue(calculator.isInAllowlist(1000L)); } @Test @@ -366,7 +366,7 @@ void testIsInTrafficCalcConfig_exactlyAtEnd() throws Exception { // Setup - load traffic calc config with single range [1000, 2000] String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [1000, 2000] ] } @@ -377,7 +377,7 @@ void testIsInTrafficCalcConfig_exactlyAtEnd() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - true when exactly at end of range - assertTrue(calculator.isInWhitelist(2000L)); + assertTrue(calculator.isInAllowlist(2000L)); } @Test @@ -385,7 +385,7 @@ void testIsInTrafficCalcConfig_beforeRange() throws Exception { // Setup - load traffic calc config with single range [1000, 2000] String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [1000, 2000] ] } @@ -396,7 +396,7 @@ void testIsInTrafficCalcConfig_beforeRange() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - false when before range - assertFalse(calculator.isInWhitelist(999L)); + assertFalse(calculator.isInAllowlist(999L)); } @Test @@ -404,7 +404,7 @@ void testIsInTrafficCalcConfig_afterRange() throws Exception { // Setup - load traffic calc config with single range [1000, 2000] String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [1000, 2000] ] } @@ -415,7 +415,7 @@ void testIsInTrafficCalcConfig_afterRange() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - false when after range - assertFalse(calculator.isInWhitelist(2001L)); + assertFalse(calculator.isInAllowlist(2001L)); } @Test @@ -423,7 +423,7 @@ void testIsInTrafficCalcConfig_betweenRanges() throws Exception { // Setup - load traffic calc config with two ranges [1000, 2000] and [3000, 4000] String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [1000, 2000], [3000, 4000] ] @@ -435,7 +435,7 @@ void testIsInTrafficCalcConfig_betweenRanges() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - false when between ranges - assertFalse(calculator.isInWhitelist(2500L)); + assertFalse(calculator.isInAllowlist(2500L)); } @Test @@ -445,7 +445,7 @@ void testIsInTrafficCalcConfig_emptyRanges() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - false when empty ranges - assertFalse(calculator.isInWhitelist(1500L)); + assertFalse(calculator.isInAllowlist(1500L)); } @Test @@ -453,7 +453,7 @@ void testIsInTrafficCalcConfig_nullRanges() throws Exception { // Setup - no traffic calc config ranges loaded (will fail and set empty) String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": null + "traffic_calc_allowlist_ranges": null } """; createConfigFromPartialJson(trafficCalcConfigJson); @@ -462,7 +462,7 @@ void testIsInTrafficCalcConfig_nullRanges() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - false when null/empty ranges - assertFalse(calculator.isInWhitelist(1500L)); + assertFalse(calculator.isInAllowlist(1500L)); } @Test @@ -470,7 +470,7 @@ void testIsInTrafficCalcConfig_invalidRangeSize() throws Exception { // Setup - load traffic calc config with invalid range (only 1 element) and valid range String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [1000], [2000, 3000] ] @@ -482,8 +482,8 @@ void testIsInTrafficCalcConfig_invalidRangeSize() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - assertFalse(calculator.isInWhitelist(1500L)); // Should not match invalid range - assertTrue(calculator.isInWhitelist(2500L)); // Should match valid range + assertFalse(calculator.isInAllowlist(1500L)); // Should not match invalid range + assertTrue(calculator.isInAllowlist(2500L)); // Should match valid range } @Test @@ -491,7 +491,7 @@ void testIsInTrafficCalcConfig_multipleRanges() throws Exception { // Setup - load traffic calc config with multiple ranges String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [1000, 2000], [3000, 4000], [5000, 6000] @@ -504,10 +504,10 @@ void testIsInTrafficCalcConfig_multipleRanges() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - assertTrue(calculator.isInWhitelist(1500L)); // In first range - assertTrue(calculator.isInWhitelist(3500L)); // In second range - assertTrue(calculator.isInWhitelist(5500L)); // In third range - assertFalse(calculator.isInWhitelist(2500L)); // Between first and second + assertTrue(calculator.isInAllowlist(1500L)); // In first range + assertTrue(calculator.isInAllowlist(3500L)); // In second range + assertTrue(calculator.isInAllowlist(5500L)); // In third range + assertFalse(calculator.isInAllowlist(2500L)); // Between first and second } // ============================================================================ @@ -521,7 +521,7 @@ void testGetTrafficCalcConfigDuration_noRanges() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Assert - assertEquals(0L, calculator.getWhitelistDuration(10000L, 5000L)); // 0 duration when no ranges + assertEquals(0L, calculator.getAllowlistDuration(10000L, 5000L)); // 0 duration when no ranges } @Test @@ -529,7 +529,7 @@ void testGetTrafficCalcConfigDuration_rangeFullyWithinWindow() throws Exception // Setup - range fully within window String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [6000, 7000] ] } @@ -540,7 +540,7 @@ void testGetTrafficCalcConfigDuration_rangeFullyWithinWindow() throws Exception cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], range [6000, 7000] - long duration = calculator.getWhitelistDuration(10000L, 5000L); + long duration = calculator.getAllowlistDuration(10000L, 5000L); // Assert - full range duration assertEquals(1000L, duration); @@ -551,7 +551,7 @@ void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsStart() throws Excep // Setup - range partially overlaps start of window String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [3000, 7000] ] } @@ -562,7 +562,7 @@ void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsStart() throws Excep cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], range [3000, 7000] - long duration = calculator.getWhitelistDuration(10000L, 5000L); + long duration = calculator.getAllowlistDuration(10000L, 5000L); // Assert - should clip to [5000, 7000] = 2000 assertEquals(2000L, duration); @@ -573,7 +573,7 @@ void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsEnd() throws Excepti // Setup - range partially overlaps end of window String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [8000, 12000] ] } @@ -584,7 +584,7 @@ void testGetTrafficCalcConfigDuration_rangePartiallyOverlapsEnd() throws Excepti cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], range [8000, 12000] - long duration = calculator.getWhitelistDuration(10000L, 5000L); + long duration = calculator.getAllowlistDuration(10000L, 5000L); // Assert - should clip to [8000, 10000] = 2000 assertEquals(2000L, duration); @@ -595,7 +595,7 @@ void testGetTrafficCalcConfigDuration_rangeCompletelyOutsideWindow() throws Exce // Setup - range completely outside window String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [1000, 2000] ] } @@ -606,7 +606,7 @@ void testGetTrafficCalcConfigDuration_rangeCompletelyOutsideWindow() throws Exce cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], range [1000, 2000] - long duration = calculator.getWhitelistDuration(10000L, 5000L); + long duration = calculator.getAllowlistDuration(10000L, 5000L); // Assert - 0 duration when range completely outside window assertEquals(0L, duration); @@ -617,7 +617,7 @@ void testGetTrafficCalcConfigDuration_multipleRanges() throws Exception { // Setup - multiple ranges String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [6000, 7000], [8000, 9000] ] @@ -629,7 +629,7 @@ void testGetTrafficCalcConfigDuration_multipleRanges() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], ranges [6000, 7000] and [8000, 9000] - long duration = calculator.getWhitelistDuration(10000L, 5000L); + long duration = calculator.getAllowlistDuration(10000L, 5000L); // Assert - 1000 + 1000 = 2000 assertEquals(2000L, duration); @@ -640,7 +640,7 @@ void testGetTrafficCalcConfigDuration_rangeSpansEntireWindow() throws Exception // Setup - range spans entire window String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [3000, 12000] ] } @@ -651,7 +651,7 @@ void testGetTrafficCalcConfigDuration_rangeSpansEntireWindow() throws Exception cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - window [5000, 10000], range [3000, 12000] - long duration = calculator.getWhitelistDuration(10000L, 5000L); + long duration = calculator.getAllowlistDuration(10000L, 5000L); // Assert - entire window is in traffic calc config ranges = 5000 assertEquals(5000L, duration); @@ -783,7 +783,7 @@ void testReloadTrafficCalcConfig_success() throws Exception { // Setup - initial traffic calc config String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [1000, 2000], [3000, 4000] ] @@ -797,7 +797,7 @@ void testReloadTrafficCalcConfig_success() throws Exception { // Change the traffic calc config to a new range String newTrafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [5000, 6000] ] } @@ -808,7 +808,7 @@ void testReloadTrafficCalcConfig_success() throws Exception { calculator.reloadTrafficCalcConfig(); // Assert - verify new traffic calc config is loaded - assertTrue(calculator.isInWhitelist(5500L)); + assertTrue(calculator.isInAllowlist(5500L)); } @Test @@ -816,7 +816,7 @@ void testReloadTrafficCalcConfig_failure() throws Exception { // Setup - initial traffic calc config String trafficCalcConfigJson = """ { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [1000, 2000] ] } @@ -843,19 +843,19 @@ public void testReloadTrafficCalcConfig_failure_missingKeys() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act & Assert missing threshold multiplier - createTrafficConfigFile("{\"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_baseline_traffic\": 100, \"traffic_calc_whitelist_ranges\": [ [1000, 2000] ]}"); + createTrafficConfigFile("{\"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_baseline_traffic\": 100, \"traffic_calc_allowlist_ranges\": [ [1000, 2000] ]}"); assertThrows(MalformedTrafficCalcConfigException.class, () -> { calculator.reloadTrafficCalcConfig(); }); // Act & Assert missing evaluation window seconds - createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_baseline_traffic\": 100, \"traffic_calc_whitelist_ranges\": [ [1000, 2000] ]}"); + createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_baseline_traffic\": 100, \"traffic_calc_allowlist_ranges\": [ [1000, 2000] ]}"); assertThrows(MalformedTrafficCalcConfigException.class, () -> { calculator.reloadTrafficCalcConfig(); }); // Act & Assert missing baseline traffic - createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_whitelist_ranges\": [ [1000, 2000] ]}"); + createTrafficConfigFile("{\"traffic_calc_threshold_multiplier\": 5, \"traffic_calc_evaluation_window_seconds\": 86400, \"traffic_calc_allowlist_ranges\": [ [1000, 2000] ]}"); assertThrows(MalformedTrafficCalcConfigException.class, () -> { calculator.reloadTrafficCalcConfig(); }); @@ -872,7 +872,7 @@ public void testReloadTrafficCalcConfig_failure_misorderedRanges() throws Except // Setup - misordered ranges OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - createConfigFromPartialJson("{\"traffic_calc_whitelist_ranges\": [ [2000, 1000] ]}"); + createConfigFromPartialJson("{\"traffic_calc_allowlist_ranges\": [ [2000, 1000] ]}"); // Act & Assert assertThrows(MalformedTrafficCalcConfigException.class, () -> { @@ -885,7 +885,7 @@ public void testReloadTrafficCalcConfig_failure_rangeTooLong() throws Exception // Setup - range greater than 24 hours OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - createConfigFromPartialJson("{\"traffic_calc_whitelist_ranges\": [ [1000, 200000] ]}"); + createConfigFromPartialJson("{\"traffic_calc_allowlist_ranges\": [ [1000, 200000] ]}"); // Act & Assert assertThrows(MalformedTrafficCalcConfigException.class, () -> { @@ -1126,7 +1126,7 @@ void testCalculateStatus_withTrafficCalcConfig() throws Exception { // Traffic calc config that covers part of window String trafficCalcConfigJson = String.format(""" { - "traffic_calc_whitelist_ranges": [ + "traffic_calc_allowlist_ranges": [ [%d, %d] ] } From d7db764ce06c08f78dab3cde6c8b56839ede45c6 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 25 Nov 2025 17:23:28 -0700 Subject: [PATCH 18/28] small comment/name update --- .../java/com/uid2/optout/vertx/OptOutTrafficCalculator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 2cf2792e..1809703e 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -223,9 +223,9 @@ public TrafficStatus calculateStatus(List sqsMessages) { boolean shouldStop = false; for (long ts : timestamps) { - // Stop condition: record is older than our 48h window + // Stop condition: record is older than our window if (ts < windowStart) { - LOGGER.debug("Stopping delta file processing at timestamp {} (older than t-48h)", ts); + LOGGER.debug("Stopping delta file processing at timestamp {} (older than window start {})", ts, windowStart); break; } @@ -234,7 +234,7 @@ public TrafficStatus calculateStatus(List sqsMessages) { continue; } - // Count for sumCurrent: [t-24h, t+5m] + // increment sum if record is in window if (ts >= windowStart && ts <= t) { sum++; } From a17854044becc8dd64b06b79707ad52cf4c96118 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 25 Nov 2025 17:26:00 -0700 Subject: [PATCH 19/28] naming update --- .../optout/vertx/OptOutTrafficCalculator.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 1809703e..b7e514cd 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -468,21 +468,21 @@ private void evictOldCacheEntries(long cutoffTimestamp) { /** * Determine traffic status based on current vs past counts */ - TrafficStatus determineStatus(int sumCurrent, int sumPast) { - if (sumPast == 0) { + TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { + if (baselineTraffic == 0) { // Avoid division by zero - if no baseline traffic, return DEFAULT status - LOGGER.warn("sumPast is 0, cannot detect thresholdcrossing. Returning DEFAULT status."); + LOGGER.warn("baselineTraffic is 0, cannot detect threshold crossing. Returning DEFAULT status."); return TrafficStatus.DEFAULT; } - if (sumCurrent >= thresholdMultiplier * sumPast) { - LOGGER.warn("DELAYED_PROCESSING threshold breached: sumCurrent={} >= {}×sumPast={}", - sumCurrent, thresholdMultiplier, sumPast); + if (sumCurrent >= thresholdMultiplier * baselineTraffic) { + LOGGER.warn("DELAYED_PROCESSING threshold breached: sumCurrent={} >= {}×baselineTraffic={}", + sumCurrent, thresholdMultiplier, baselineTraffic); return TrafficStatus.DELAYED_PROCESSING; } - LOGGER.info("Traffic within normal range: sumCurrent={} < {}×sumPast={}", - sumCurrent, thresholdMultiplier, sumPast); + LOGGER.info("Traffic within normal range: sumCurrent={} < {}×baselineTraffic={}", + sumCurrent, thresholdMultiplier, baselineTraffic); return TrafficStatus.DEFAULT; } From df7ab91b56ed485bed727b5c07a31db891063b63 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 25 Nov 2025 17:27:00 -0700 Subject: [PATCH 20/28] naming update --- .../java/com/uid2/optout/vertx/OptOutTrafficCalculator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index b7e514cd..8633c813 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -469,9 +469,9 @@ private void evictOldCacheEntries(long cutoffTimestamp) { * Determine traffic status based on current vs past counts */ TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { - if (baselineTraffic == 0) { + if (baselineTraffic == 0 || thresholdMultiplier == 0) { // Avoid division by zero - if no baseline traffic, return DEFAULT status - LOGGER.warn("baselineTraffic is 0, cannot detect threshold crossing. Returning DEFAULT status."); + LOGGER.warn("baselineTraffic is 0 or thresholdMultiplier is 0 returning DEFAULT status."); return TrafficStatus.DEFAULT; } From 38c86b37fd363d0004a88f5dc0e9a804a0f1bc34 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Mon, 1 Dec 2025 23:06:57 -0700 Subject: [PATCH 21/28] add newest delta file logic --- .../optout/vertx/OptOutTrafficCalculator.java | 68 +++++++++++++------ 1 file changed, 49 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 8633c813..0b11c8db 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -191,6 +191,9 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic /** * Calculate traffic status based on delta files and SQS queue messages. * + * Uses the newest delta file timestamp to anchor the 24-hour delta traffic window, + * and the oldest queue timestamp to anchor the 5-minute queue window. + * * @param sqsMessages List of SQS messages * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) */ @@ -205,17 +208,21 @@ public TrafficStatus calculateStatus(List sqsMessages) { return TrafficStatus.DEFAULT; } - // Find t = oldest SQS queue message timestamp - long t = findOldestQueueTimestamp(sqsMessages); - LOGGER.info("Traffic calculation starting with t={} (oldest SQS message)", t); + // Find newest delta file timestamp for delta traffic window + long newestDeltaTs = findNewestDeltaTimestamp(deltaS3Paths); + LOGGER.info("Traffic calculation: newestDeltaTs={}", newestDeltaTs); + + // Find oldest SQS queue message timestamp for queue window + long oldestQueueTs = findOldestQueueTimestamp(sqsMessages); + LOGGER.info("Traffic calculation: oldestQueueTs={}", oldestQueueTs); - // define start time of the evaluation window [t-24h, t+5m] - long windowStart = t - this.evaluationWindowSeconds - getAllowlistDuration(t, t - this.evaluationWindowSeconds); + // Define start time of the delta evaluation window [newestDeltaTs - 24h, newestDeltaTs] + long deltaWindowStart = newestDeltaTs - this.evaluationWindowSeconds - getAllowlistDuration(newestDeltaTs, newestDeltaTs - this.evaluationWindowSeconds); - // Evict old cache entries (older than window start) - evictOldCacheEntries(windowStart); + // Evict old cache entries (older than delta window start) + evictOldCacheEntries(deltaWindowStart); - // Process delta files and count + // Process delta files and count records in [deltaWindowStart, newestDeltaTs] int sum = 0; for (String s3Path : deltaS3Paths) { @@ -224,8 +231,8 @@ public TrafficStatus calculateStatus(List sqsMessages) { boolean shouldStop = false; for (long ts : timestamps) { // Stop condition: record is older than our window - if (ts < windowStart) { - LOGGER.debug("Stopping delta file processing at timestamp {} (older than window start {})", ts, windowStart); + if (ts < deltaWindowStart) { + LOGGER.debug("Stopping delta file processing at timestamp {} (older than window start {})", ts, deltaWindowStart); break; } @@ -234,8 +241,8 @@ public TrafficStatus calculateStatus(List sqsMessages) { continue; } - // increment sum if record is in window - if (ts >= windowStart && ts <= t) { + // increment sum if record is in delta window + if (ts >= deltaWindowStart) { sum++; } @@ -246,10 +253,9 @@ public TrafficStatus calculateStatus(List sqsMessages) { } } - // Count SQS messages in [t, t+5m] + // Count SQS messages in [oldestQueueTs, oldestQueueTs + 5m] if (sqsMessages != null && !sqsMessages.isEmpty()) { - int sqsCount = countSqsMessages( - sqsMessages, t); + int sqsCount = countSqsMessages(sqsMessages, oldestQueueTs); sum += sqsCount; } @@ -267,6 +273,29 @@ public TrafficStatus calculateStatus(List sqsMessages) { } } + /** + * Find the newest timestamp from delta files. + * Reads the newest delta file and returns its maximum timestamp. + */ + private long findNewestDeltaTimestamp(List deltaS3Paths) throws IOException { + if (deltaS3Paths == null || deltaS3Paths.isEmpty()) { + return System.currentTimeMillis() / 1000; + } + + // Delta files are sorted (ISO 8601 format, lexicographically sortable) so first file is newest + String newestDeltaPath = deltaS3Paths.get(0); + List timestamps = getTimestampsFromFile(newestDeltaPath); + + if (timestamps.isEmpty()) { + LOGGER.warn("Newest delta file has no timestamps: {}", newestDeltaPath); + return System.currentTimeMillis() / 1000; + } + + long newestTs = Collections.max(timestamps); + LOGGER.debug("Found newest delta timestamp {} from file {}", newestTs, newestDeltaPath); + return newestTs; + } + /** * List all delta files from S3, sorted newest to oldest */ @@ -400,16 +429,17 @@ private Long extractTimestampFromMessage(Message msg) { } /** - * Count SQS messages from t to t+5 minutes + * Count SQS messages from oldestQueueTs to oldestQueueTs + 5 minutes */ - private int countSqsMessages(List sqsMessages, long t) { + private int countSqsMessages(List sqsMessages, long oldestQueueTs) { int count = 0; + long windowEnd = oldestQueueTs + 5 * 60; for (Message msg : sqsMessages) { Long ts = extractTimestampFromMessage(msg); - if (ts < t || ts > t + 5 * 60) { + if (ts < oldestQueueTs || ts > windowEnd) { continue; } @@ -420,7 +450,7 @@ private int countSqsMessages(List sqsMessages, long t) { } - LOGGER.info("SQS messages: {} in window [t={}, t+5(minutes)={}]", count, t, t + 5 * 60); + LOGGER.info("SQS messages: {} in window [oldestQueueTs={}, oldestQueueTs+5m={}]", count, oldestQueueTs, windowEnd); return count; } From 962b7cc39680457b8b22abfacf3e69da98c3adfe Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Mon, 1 Dec 2025 23:15:41 -0700 Subject: [PATCH 22/28] update comments --- src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 0b11c8db..7795021c 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -177,7 +177,6 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic } } - // Sort ranges by start time for efficient lookups ranges.sort(Comparator.comparing(range -> range.get(0))); } catch (Exception e) { From 16dfe5302684659812b43e42d5d109bb6a9a6c9f Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Mon, 1 Dec 2025 23:37:11 -0700 Subject: [PATCH 23/28] add traffic filter --- .../optout/vertx/OptOutTrafficFilter.java | 167 +++++++ .../optout/vertx/OptOutTrafficFilterTest.java | 424 ++++++++++++++++++ 2 files changed, 591 insertions(+) create mode 100644 src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java create mode 100644 src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java new file mode 100644 index 00000000..70bd1a09 --- /dev/null +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java @@ -0,0 +1,167 @@ +package com.uid2.optout.vertx; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Collections; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.charset.StandardCharsets; +import io.vertx.core.json.JsonObject; +import io.vertx.core.json.JsonArray; + +public class OptOutTrafficFilter { + private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficFilter.class); + + private final String trafficFilterConfigPath; + List filterRules; + + /** + * Traffic filter rule defining a time range and a list of IP addresses to exclude + */ + private static class TrafficFilterRule { + private final List range; + private final List ipAddresses; + + TrafficFilterRule(List range, List ipAddresses) { + this.range = range; + this.ipAddresses = ipAddresses; + } + + public long getRangeStart() { + return range.get(0); + } + public long getRangeEnd() { + return range.get(1); + } + public List getIpAddresses() { + return ipAddresses; + } + } + + public static class MalformedTrafficFilterConfigException extends Exception { + public MalformedTrafficFilterConfigException(String message) { + super(message); + } + } + + /** + * Constructor for OptOutTrafficFilter + * + * @param trafficFilterConfigPath S3 path for traffic filter config + * @throws MalformedTrafficFilterConfigException if the traffic filter config is invalid + */ + public OptOutTrafficFilter(String trafficFilterConfigPath) throws MalformedTrafficFilterConfigException { + this.trafficFilterConfigPath = trafficFilterConfigPath; + // Initial filter rules load + this.filterRules = Collections.emptyList(); // start empty + reloadTrafficFilterConfig(); // load ConfigMap + + LOGGER.info("OptOutTrafficFilter initialized: filterRules={}", + filterRules.size()); + } + + /** + * Reload traffic filter config from ConfigMap. + * Expected format: + * { + * "blacklist_requests": [ + * {range: [startTimestamp, endTimestamp], IPs: ["ip1"]}, + * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip2"]}, + * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip3"]}, + * ] + * } + * + * Can be called periodically to pick up config changes without restarting. + */ + public void reloadTrafficFilterConfig() throws MalformedTrafficFilterConfigException { + LOGGER.info("Loading traffic filter config from ConfigMap"); + try (InputStream is = Files.newInputStream(Paths.get(trafficFilterConfigPath))) { + String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); + JsonObject filterConfigJson = new JsonObject(content); + + this.filterRules = parseFilterRules(filterConfigJson); + + LOGGER.info("Successfully loaded traffic filter config from ConfigMap: filterRules={}", + filterRules.size()); + + } catch (Exception e) { + LOGGER.warn("No traffic filter config found at: {}", trafficFilterConfigPath, e); + throw new MalformedTrafficFilterConfigException(e.getMessage()); + } + } + + /** + * Parse request filtering rules from JSON config + */ + List parseFilterRules(JsonObject config) throws MalformedTrafficFilterConfigException { + List rules = new ArrayList<>(); + try { + JsonArray blacklistRequests = config.getJsonArray("blacklist_requests"); + if (blacklistRequests == null) { + LOGGER.error("Invalid traffic filter config: blacklist_requests is null"); + throw new MalformedTrafficFilterConfigException("Invalid traffic filter config: blacklist_requests is null"); + } + for (int i = 0; i < blacklistRequests.size(); i++) { + JsonObject ruleJson = blacklistRequests.getJsonObject(i); + + // parse range + var rangeJson = ruleJson.getJsonArray("range"); + List range = new ArrayList<>(); + if (rangeJson != null && rangeJson.size() == 2) { + long start = rangeJson.getLong(0); + long end = rangeJson.getLong(1); + + if (start >= end) { + LOGGER.error("Invalid traffic filter rule: range start must be less than end: {}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule: range start must be less than end"); + } + range.add(start); + range.add(end); + } + + // parse IPs + var ipAddressesJson = ruleJson.getJsonArray("IPs"); + List ipAddresses = new ArrayList<>(); + if (ipAddressesJson != null) { + for (int j = 0; j < ipAddressesJson.size(); j++) { + ipAddresses.add(ipAddressesJson.getString(j)); + } + } + + // log error and throw exception if rule is invalid + if (range.size() != 2 || ipAddresses.size() == 0 || range.get(1) - range.get(0) > 86400) { // range must be 24 hours or less + LOGGER.error("Invalid traffic filter rule: {}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule"); + } + + TrafficFilterRule rule = new TrafficFilterRule(range, ipAddresses); + + LOGGER.info("Loaded traffic filter rule: range=[{}, {}], IPs={}", rule.getRangeStart(), rule.getRangeEnd(), rule.getIpAddresses()); + rules.add(rule); + } + return rules; + } catch (Exception e) { + LOGGER.error("Failed to parse traffic filter rules: config={}, error={}", config.encode(), e.getMessage()); + throw new MalformedTrafficFilterConfigException(e.getMessage()); + } + } + + public boolean isBlacklisted(SqsParsedMessage message) { + long timestamp = message.getTimestamp(); + String clientIp = message.getClientIp(); + + for (TrafficFilterRule rule : filterRules) { + if(timestamp >= rule.getRangeStart() && timestamp <= rule.getRangeEnd()) { + if(rule.getIpAddresses().contains(clientIp)) { + return true; + } + }; + } + return false; + } + +} \ No newline at end of file diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java new file mode 100644 index 00000000..491e9967 --- /dev/null +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java @@ -0,0 +1,424 @@ +package com.uid2.optout.vertx; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import software.amazon.awssdk.services.sqs.model.Message; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.Assert.*; + +public class OptOutTrafficFilterTest { + + private static final String TEST_CONFIG_PATH = "./traffic-config.json"; + + @Before + public void setUp() { + try { + Files.deleteIfExists(Path.of(TEST_CONFIG_PATH)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @After + public void tearDown() { + try { + Files.deleteIfExists(Path.of(TEST_CONFIG_PATH)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + public void testParseFilterRules_emptyRules() throws Exception { + // Setup - empty blacklist + String config = """ + { + "blacklist_requests": [] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - no rules + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertEquals(0, filter.filterRules.size()); + } + + @Test + public void testParseFilterRules_singleRule() throws Exception { + // Setup - config with one rule + String config = """ + { + "blacklist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - one rule + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertEquals(1, filter.filterRules.size()); + } + + @Test + public void testParseFilterRules_multipleRules() throws Exception { + // Setup - config with multiple rules + String config = """ + { + "blacklist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + }, + { + "range": [1700010000, 1700013600], + "IPs": ["10.0.0.1", "10.0.0.2"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - two rules + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertEquals(2, filter.filterRules.size()); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_missingBlacklistRequests() throws Exception { + // Setup - config without blacklist_requests field + String config = """ + { + "other_field": "value" + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_invalidRange_startAfterEnd() throws Exception { + // Setup - range where start > end + String config = """ + { + "blacklist_requests": [ + { + "range": [1700003600, 1700000000], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_invalidRange_startEqualsEnd() throws Exception { + // Setup - range where start == end + String config = """ + { + "blacklist_requests": [ + { + "range": [1700000000, 1700000000], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_rangeExceeds24Hours() throws Exception { + // Setup - range longer than 24 hours (86400 seconds) + String config = """ + { + "blacklist_requests": [ + { + "range": [1700000000, 1700086401], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_emptyIPs() throws Exception { + // Setup - rule with empty IP list + String config = """ + { + "blacklist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": [] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testParseFilterRules_missingIPs() throws Exception { + // Setup - rule without IPs field + String config = """ + { + "blacklist_requests": [ + { + "range": [1700000000, 1700003600] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - throws exception + new OptOutTrafficFilter(TEST_CONFIG_PATH); + } + + @Test + public void testIsBlacklisted_matchingIPAndTimestamp() throws Exception { + // Setup - filter with blacklist rule + String config = """ + { + "blacklist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1", "10.0.0.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + SqsParsedMessage message = createTestMessage(1700001800, "192.168.1.1"); + + // Act & Assert - blacklisted + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertTrue(filter.isBlacklisted(message)); + } + + @Test + public void testIsBlacklisted_matchingIPOutsideTimeRange() throws Exception { + // Setup - filter with blacklist rule + String config = """ + { + "blacklist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + + // Act & Assert - message before range not blacklisted + SqsParsedMessage messageBefore = createTestMessage(1699999999, "192.168.1.1"); + assertFalse(filter.isBlacklisted(messageBefore)); + // Act & Assert - message after range not blacklisted + SqsParsedMessage messageAfter = createTestMessage(1700003601, "192.168.1.1"); + assertFalse(filter.isBlacklisted(messageAfter)); + } + + @Test + public void testIsBlacklisted_nonMatchingIP() throws Exception { + // Setup - filter with blacklist rule + String config = """ + { + "blacklist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + + // Act & Assert - non-matching IP not blacklisted + SqsParsedMessage message = createTestMessage(1700001800, "10.0.0.1"); + assertFalse(filter.isBlacklisted(message)); + } + + @Test + public void testIsBlacklisted_atRangeBoundaries() throws Exception { + // Setup - filter with blacklist rule + String config = """ + { + "blacklist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + + // Act & Assert - message at start boundary (inclusive) blacklisted + SqsParsedMessage messageAtStart = createTestMessage(1700000000, "192.168.1.1"); + assertTrue(filter.isBlacklisted(messageAtStart)); + + // Act & Assert - message at end boundary (inclusive) blacklisted + SqsParsedMessage messageAtEnd = createTestMessage(1700003600, "192.168.1.1"); + assertTrue(filter.isBlacklisted(messageAtEnd)); + } + + @Test + public void testIsBlacklisted_multipleRules() throws Exception { + // Setup - multiple blacklist rules + String config = """ + { + "blacklist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + }, + { + "range": [1700010000, 1700013600], + "IPs": ["10.0.0.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + + // Act & Assert - message matches first rule + SqsParsedMessage msg1 = createTestMessage(1700001800, "192.168.1.1"); + assertTrue(filter.isBlacklisted(msg1)); + + // Act & Assert - message matches second rule + SqsParsedMessage msg2 = createTestMessage(1700011800, "10.0.0.1"); + assertTrue(filter.isBlacklisted(msg2)); + + // Act & Assert - message matches neither rule + SqsParsedMessage msg3 = createTestMessage(1700005000, "172.16.0.1"); + assertFalse(filter.isBlacklisted(msg3)); + } + + @Test + public void testIsBlacklisted_nullClientIp() throws Exception { + // Setup - filter with blacklist rule + String config = """ + { + "blacklist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + + // Act & Assert - message with null IP not blacklisted + SqsParsedMessage message = createTestMessage(1700001800, null); + assertFalse(filter.isBlacklisted(message)); + } + + @Test + public void testReloadTrafficFilterConfig_success() throws Exception { + // Setup - config with one rule + String initialConfig = """ + { + "blacklist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), initialConfig); + + // Act & Assert - one rule + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertEquals(1, filter.filterRules.size()); + + // Setup - update config + String updatedConfig = """ + { + "blacklist_requests": [ + { + "range": [1700000000, 1700003600], + "IPs": ["192.168.1.1"] + }, + { + "range": [1700010000, 1700013600], + "IPs": ["10.0.0.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), updatedConfig); + + // Act & Assert - two rules + filter.reloadTrafficFilterConfig(); + assertEquals(2, filter.filterRules.size()); + } + + @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) + public void testReloadTrafficFilterConfig_fileNotFound() throws Exception { + // Setup, Act & Assert - try to create filter with non-existent config + new OptOutTrafficFilter("./non-existent-file.json"); + } + + @Test + public void testParseFilterRules_maxValidRange() throws Exception { + // Setup - range exactly 24 hours (86400 seconds) - should be valid + String config = """ + { + "blacklist_requests": [ + { + "range": [1700000000, 1700086400], + "IPs": ["192.168.1.1"] + } + ] + } + """; + Files.writeString(Path.of(TEST_CONFIG_PATH), config); + + // Act & Assert - one rule + OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); + assertEquals(1, filter.filterRules.size()); + } + + /** + * Helper method to create test SqsParsedMessage + */ + private SqsParsedMessage createTestMessage(long timestamp, String clientIp) { + Message mockMessage = Message.builder().build(); + byte[] hash = new byte[32]; + byte[] id = new byte[32]; + return new SqsParsedMessage(mockMessage, hash, id, timestamp, null, null, clientIp, null); + } +} From 6f4cf769a1ec992bf94fff64122be88192438435 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Mon, 1 Dec 2025 23:45:20 -0700 Subject: [PATCH 24/28] merge --- src/main/java/com/uid2/optout/Const.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/com/uid2/optout/Const.java b/src/main/java/com/uid2/optout/Const.java index dd78e0ab..02875a39 100644 --- a/src/main/java/com/uid2/optout/Const.java +++ b/src/main/java/com/uid2/optout/Const.java @@ -22,10 +22,6 @@ public static class Config extends com.uid2.shared.Const.Config { public static final String OptOutSqsS3FolderProp = "optout_sqs_s3_folder"; // Default: "sqs-delta" - folder within same S3 bucket as regular optout public static final String OptOutSqsMaxMessagesPerPollProp = "optout_sqs_max_messages_per_poll"; public static final String OptOutSqsVisibilityTimeoutProp = "optout_sqs_visibility_timeout"; - public static final String OptOutTrafficCalcBaselineTrafficProp = "traffic_calc_baseline_traffic"; - public static final String OptOutTrafficCalcThresholdMultiplierProp = "traffic_calc_threshold_multiplier"; - public static final String OptOutTrafficCalcEvaluationWindowSecondsProp = "traffic_calc_evaluation_window_seconds"; - public static final String OptOutTrafficCalcAllowlistRangesProp = "traffic_calc_allowlist_ranges"; public static final String OptOutDeltaJobTimeoutSecondsProp = "optout_delta_job_timeout_seconds"; public static final String OptOutS3BucketDroppedRequestsProp = "optout_s3_bucket_dropped_requests"; public static final String TrafficFilterConfigPathProp = "traffic_filter_config_path"; From f6b7f46e8a2f7f0203b41c4bffb0959add4c5866 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Thu, 4 Dec 2025 13:14:33 -0700 Subject: [PATCH 25/28] update traffic calculator --- .../optout/vertx/OptOutTrafficCalculator.java | 45 ++++- .../optout/vertx/OptOutTrafficFilter.java | 167 ++++++++++++++++++ .../vertx/OptOutTrafficCalculatorTest.java | 150 ++++++++++++++++ 3 files changed, 360 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 7795021c..f0c7a7c5 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -179,6 +179,20 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic ranges.sort(Comparator.comparing(range -> range.get(0))); + // Validate no overlapping ranges + for (int i = 0; i < ranges.size() - 1; i++) { + long currentEnd = ranges.get(i).get(1); + long nextStart = ranges.get(i + 1).get(0); + if (currentEnd >= nextStart) { + LOGGER.error("Overlapping allowlist ranges detected: [{}, {}] overlaps with [{}, {}]", + ranges.get(i).get(0), currentEnd, nextStart, ranges.get(i + 1).get(1)); + throw new MalformedTrafficCalcConfigException( + "Overlapping allowlist ranges detected at indices " + i + " and " + (i + 1)); + } + } + + } catch (MalformedTrafficCalcConfigException e) { + throw e; } catch (Exception e) { LOGGER.error("Failed to parse allowlist ranges", e); throw new MalformedTrafficCalcConfigException("Failed to parse allowlist ranges: " + e.getMessage()); @@ -215,8 +229,10 @@ public TrafficStatus calculateStatus(List sqsMessages) { long oldestQueueTs = findOldestQueueTimestamp(sqsMessages); LOGGER.info("Traffic calculation: oldestQueueTs={}", oldestQueueTs); - // Define start time of the delta evaluation window [newestDeltaTs - 24h, newestDeltaTs] - long deltaWindowStart = newestDeltaTs - this.evaluationWindowSeconds - getAllowlistDuration(newestDeltaTs, newestDeltaTs - this.evaluationWindowSeconds); + // Define start time of the delta evaluation window + // We need evaluationWindowSeconds of non-allowlisted time, so we iteratively extend + // the window to account for any allowlist ranges in the extended portion + long deltaWindowStart = calculateWindowStartWithAllowlist(newestDeltaTs, this.evaluationWindowSeconds); // Evict old cache entries (older than delta window start) evictOldCacheEntries(deltaWindowStart); @@ -391,6 +407,31 @@ long getAllowlistDuration(long t, long windowStart) { return totalDuration; } + /** + * Calculate the window start time that provides evaluationWindowSeconds of non-allowlisted time. + * Iteratively extends the window to account for allowlist ranges that may fall in extended portions. + */ + long calculateWindowStartWithAllowlist(long newestDeltaTs, int evaluationWindowSeconds) { + long allowlistDuration = getAllowlistDuration(newestDeltaTs, newestDeltaTs - evaluationWindowSeconds); + + // Each iteration discovers at least one new allowlist range, so max iterations = number of ranges + int maxIterations = this.allowlistRanges.size() + 1; + + for (int i = 0; i < maxIterations && allowlistDuration > 0; i++) { + long newWindowStart = newestDeltaTs - evaluationWindowSeconds - allowlistDuration; + long newAllowlistDuration = getAllowlistDuration(newestDeltaTs, newWindowStart); + + if (newAllowlistDuration == allowlistDuration) { + // No new allowlist time in extended portion, we've converged + break; + } + + allowlistDuration = newAllowlistDuration; + } + + return newestDeltaTs - evaluationWindowSeconds - allowlistDuration; + } + /** * Find the oldest SQS queue message timestamp */ diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java new file mode 100644 index 00000000..70bd1a09 --- /dev/null +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java @@ -0,0 +1,167 @@ +package com.uid2.optout.vertx; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Collections; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.charset.StandardCharsets; +import io.vertx.core.json.JsonObject; +import io.vertx.core.json.JsonArray; + +public class OptOutTrafficFilter { + private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficFilter.class); + + private final String trafficFilterConfigPath; + List filterRules; + + /** + * Traffic filter rule defining a time range and a list of IP addresses to exclude + */ + private static class TrafficFilterRule { + private final List range; + private final List ipAddresses; + + TrafficFilterRule(List range, List ipAddresses) { + this.range = range; + this.ipAddresses = ipAddresses; + } + + public long getRangeStart() { + return range.get(0); + } + public long getRangeEnd() { + return range.get(1); + } + public List getIpAddresses() { + return ipAddresses; + } + } + + public static class MalformedTrafficFilterConfigException extends Exception { + public MalformedTrafficFilterConfigException(String message) { + super(message); + } + } + + /** + * Constructor for OptOutTrafficFilter + * + * @param trafficFilterConfigPath S3 path for traffic filter config + * @throws MalformedTrafficFilterConfigException if the traffic filter config is invalid + */ + public OptOutTrafficFilter(String trafficFilterConfigPath) throws MalformedTrafficFilterConfigException { + this.trafficFilterConfigPath = trafficFilterConfigPath; + // Initial filter rules load + this.filterRules = Collections.emptyList(); // start empty + reloadTrafficFilterConfig(); // load ConfigMap + + LOGGER.info("OptOutTrafficFilter initialized: filterRules={}", + filterRules.size()); + } + + /** + * Reload traffic filter config from ConfigMap. + * Expected format: + * { + * "blacklist_requests": [ + * {range: [startTimestamp, endTimestamp], IPs: ["ip1"]}, + * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip2"]}, + * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip3"]}, + * ] + * } + * + * Can be called periodically to pick up config changes without restarting. + */ + public void reloadTrafficFilterConfig() throws MalformedTrafficFilterConfigException { + LOGGER.info("Loading traffic filter config from ConfigMap"); + try (InputStream is = Files.newInputStream(Paths.get(trafficFilterConfigPath))) { + String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); + JsonObject filterConfigJson = new JsonObject(content); + + this.filterRules = parseFilterRules(filterConfigJson); + + LOGGER.info("Successfully loaded traffic filter config from ConfigMap: filterRules={}", + filterRules.size()); + + } catch (Exception e) { + LOGGER.warn("No traffic filter config found at: {}", trafficFilterConfigPath, e); + throw new MalformedTrafficFilterConfigException(e.getMessage()); + } + } + + /** + * Parse request filtering rules from JSON config + */ + List parseFilterRules(JsonObject config) throws MalformedTrafficFilterConfigException { + List rules = new ArrayList<>(); + try { + JsonArray blacklistRequests = config.getJsonArray("blacklist_requests"); + if (blacklistRequests == null) { + LOGGER.error("Invalid traffic filter config: blacklist_requests is null"); + throw new MalformedTrafficFilterConfigException("Invalid traffic filter config: blacklist_requests is null"); + } + for (int i = 0; i < blacklistRequests.size(); i++) { + JsonObject ruleJson = blacklistRequests.getJsonObject(i); + + // parse range + var rangeJson = ruleJson.getJsonArray("range"); + List range = new ArrayList<>(); + if (rangeJson != null && rangeJson.size() == 2) { + long start = rangeJson.getLong(0); + long end = rangeJson.getLong(1); + + if (start >= end) { + LOGGER.error("Invalid traffic filter rule: range start must be less than end: {}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule: range start must be less than end"); + } + range.add(start); + range.add(end); + } + + // parse IPs + var ipAddressesJson = ruleJson.getJsonArray("IPs"); + List ipAddresses = new ArrayList<>(); + if (ipAddressesJson != null) { + for (int j = 0; j < ipAddressesJson.size(); j++) { + ipAddresses.add(ipAddressesJson.getString(j)); + } + } + + // log error and throw exception if rule is invalid + if (range.size() != 2 || ipAddresses.size() == 0 || range.get(1) - range.get(0) > 86400) { // range must be 24 hours or less + LOGGER.error("Invalid traffic filter rule: {}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule"); + } + + TrafficFilterRule rule = new TrafficFilterRule(range, ipAddresses); + + LOGGER.info("Loaded traffic filter rule: range=[{}, {}], IPs={}", rule.getRangeStart(), rule.getRangeEnd(), rule.getIpAddresses()); + rules.add(rule); + } + return rules; + } catch (Exception e) { + LOGGER.error("Failed to parse traffic filter rules: config={}, error={}", config.encode(), e.getMessage()); + throw new MalformedTrafficFilterConfigException(e.getMessage()); + } + } + + public boolean isBlacklisted(SqsParsedMessage message) { + long timestamp = message.getTimestamp(); + String clientIp = message.getClientIp(); + + for (TrafficFilterRule rule : filterRules) { + if(timestamp >= rule.getRangeStart() && timestamp <= rule.getRangeEnd()) { + if(rule.getIpAddresses().contains(clientIp)) { + return true; + } + }; + } + return false; + } + +} \ No newline at end of file diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java index 0824c3b2..f977233d 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -319,6 +319,61 @@ void testParseTrafficCalcConfigRanges_nullArray() throws Exception { assertTrue(result.isEmpty()); } + @Test + void testParseTrafficCalcConfigRanges_overlappingRanges() throws Exception { + // Setup - overlapping ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L).add(2000L)) + .add(new JsonArray().add(1500L).add(2500L)); // Overlaps with first range + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act & Assert - should throw exception due to overlap + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.parseAllowlistRanges(configWithRanges); + }); + } + + @Test + void testParseTrafficCalcConfigRanges_adjacentRangesWithSameBoundary() throws Exception { + // Setup - ranges where end of first equals start of second (touching but not overlapping semantically, but we treat as overlap) + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L).add(2000L)) + .add(new JsonArray().add(2000L).add(3000L)); // Starts exactly where first ends + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act & Assert - should throw exception because ranges touch at boundary + assertThrows(MalformedTrafficCalcConfigException.class, () -> { + calculator.parseAllowlistRanges(configWithRanges); + }); + } + + @Test + void testParseTrafficCalcConfigRanges_nonOverlappingRanges() throws Exception { + // Setup - ranges that don't overlap (with gap between them) + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + JsonObject configWithRanges = new JsonObject(); + JsonArray ranges = new JsonArray() + .add(new JsonArray().add(1000L).add(2000L)) + .add(new JsonArray().add(2001L).add(3000L)); // Starts after first ends + configWithRanges.put("traffic_calc_allowlist_ranges", ranges); + + // Act + List> result = calculator.parseAllowlistRanges(configWithRanges); + + // Assert - should succeed with 2 ranges + assertEquals(2, result.size()); + } + // ============================================================================ // SECTION 3: isInTrafficCalcConfig() // ============================================================================ @@ -657,6 +712,101 @@ void testGetTrafficCalcConfigDuration_rangeSpansEntireWindow() throws Exception assertEquals(5000L, duration); } + // ============================================================================ + // SECTION 4.5: calculateWindowStartWithAllowlist() + // ============================================================================ + + @Test + void testCalculateWindowStartWithAllowlist_noAllowlist() throws Exception { + // Setup - no allowlist ranges + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - window should be [3, 8] with no extension + long windowStart = calculator.calculateWindowStartWithAllowlist(8L, 5); + + // Assert - no allowlist, so window start is simply newestDeltaTs - evaluationWindowSeconds + assertEquals(3L, windowStart); + } + + @Test + void testCalculateWindowStartWithAllowlist_allowlistInOriginalWindowOnly() throws Exception { + // Setup - allowlist range only in original window, not in extended portion + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [6, 7] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - newestDeltaTs=8, evaluationWindow=5 + // Original window [3, 8] has [6,7] allowlisted (1 hour) + // Extended portion [2, 3] has no allowlist + // So window start should be 8 - 5 - 1 = 2 + long windowStart = calculator.calculateWindowStartWithAllowlist(8L, 5); + + assertEquals(2L, windowStart); + } + + @Test + void testCalculateWindowStartWithAllowlist_allowlistInExtendedPortion() throws Exception { + // Setup - allowlist ranges in both original window AND extended portion + // This is the user's example: evaluationWindow=5, newestDeltaTs=8, allowlist={[2,3], [6,7]} + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [2, 3], + [6, 7] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + // Original window [3, 8]: [6,7] allowlisted = 1 hour + // First extension to [2, 8]: [2,3] and [6,7] allowlisted = 2 hours total + // Second extension to [1, 8]: still [2,3] and [6,7] = 2 hours (no new allowlist) + // Final: windowStart = 8 - 5 - 2 = 1 + long windowStart = calculator.calculateWindowStartWithAllowlist(8L, 5); + + assertEquals(1L, windowStart); + } + + @Test + void testCalculateWindowStartWithAllowlist_allowlistBeforeWindow() throws Exception { + // Setup - allowlist range entirely before the initial window + // This tests that we don't over-extend when allowlist is old + // evaluationWindow=5, newestDeltaTs=20, allowlist=[10,13] + String trafficCalcConfigJson = """ + { + "traffic_calc_allowlist_ranges": [ + [10, 13] + ] + } + """; + createConfigFromPartialJson(trafficCalcConfigJson); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act + // Initial window [15, 20]: no allowlist overlap, allowlistDuration = 0 + // No extension needed + // Final: windowStart = 20 - 5 - 0 = 15 + long windowStart = calculator.calculateWindowStartWithAllowlist(20L, 5); + + // Verify: window [15, 20] has 5 hours, 0 allowlisted = 5 non-allowlisted + assertEquals(15L, windowStart); + } + // ============================================================================ // SECTION 5: determineStatus() // ============================================================================ From a93b051a8761308d4c5d990afe3ddff90a334436 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Thu, 4 Dec 2025 13:24:09 -0700 Subject: [PATCH 26/28] address comments --- .../optout/vertx/OptOutTrafficFilter.java | 25 +++-- .../optout/vertx/OptOutTrafficFilterTest.java | 98 +++++++++---------- 2 files changed, 64 insertions(+), 59 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java index 70bd1a09..e8bd04b8 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java @@ -68,7 +68,7 @@ public OptOutTrafficFilter(String trafficFilterConfigPath) throws MalformedTraff * Reload traffic filter config from ConfigMap. * Expected format: * { - * "blacklist_requests": [ + * "denylist_requests": [ * {range: [startTimestamp, endTimestamp], IPs: ["ip1"]}, * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip2"]}, * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip3"]}, @@ -100,13 +100,13 @@ public void reloadTrafficFilterConfig() throws MalformedTrafficFilterConfigExcep List parseFilterRules(JsonObject config) throws MalformedTrafficFilterConfigException { List rules = new ArrayList<>(); try { - JsonArray blacklistRequests = config.getJsonArray("blacklist_requests"); - if (blacklistRequests == null) { - LOGGER.error("Invalid traffic filter config: blacklist_requests is null"); - throw new MalformedTrafficFilterConfigException("Invalid traffic filter config: blacklist_requests is null"); + JsonArray denylistRequests = config.getJsonArray("denylist_requests"); + if (denylistRequests == null) { + LOGGER.error("Invalid traffic filter config: denylist_requests is null"); + throw new MalformedTrafficFilterConfigException("Invalid traffic filter config: denylist_requests is null"); } - for (int i = 0; i < blacklistRequests.size(); i++) { - JsonObject ruleJson = blacklistRequests.getJsonObject(i); + for (int i = 0; i < denylistRequests.size(); i++) { + JsonObject ruleJson = denylistRequests.getJsonObject(i); // parse range var rangeJson = ruleJson.getJsonArray("range"); @@ -134,8 +134,8 @@ List parseFilterRules(JsonObject config) throws MalformedTraf // log error and throw exception if rule is invalid if (range.size() != 2 || ipAddresses.size() == 0 || range.get(1) - range.get(0) > 86400) { // range must be 24 hours or less - LOGGER.error("Invalid traffic filter rule: {}", ruleJson.encode()); - throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule"); + LOGGER.error("Invalid traffic filter rule, range must be 24 hours or less: {}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule, range must be 24 hours or less"); } TrafficFilterRule rule = new TrafficFilterRule(range, ipAddresses); @@ -150,10 +150,15 @@ List parseFilterRules(JsonObject config) throws MalformedTraf } } - public boolean isBlacklisted(SqsParsedMessage message) { + public boolean isDenylisted(SqsParsedMessage message) { long timestamp = message.getTimestamp(); String clientIp = message.getClientIp(); + if (clientIp == null || clientIp.isEmpty()) { + LOGGER.error("Request does not contain client IP, timestamp={}", timestamp); + return false; + } + for (TrafficFilterRule rule : filterRules) { if(timestamp >= rule.getRangeStart() && timestamp <= rule.getRangeEnd()) { if(rule.getIpAddresses().contains(clientIp)) { diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java index 491e9967..63f6807c 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java @@ -34,10 +34,10 @@ public void tearDown() { @Test public void testParseFilterRules_emptyRules() throws Exception { - // Setup - empty blacklist + // Setup - empty denylist String config = """ { - "blacklist_requests": [] + "denylist_requests": [] } """; Files.writeString(Path.of(TEST_CONFIG_PATH), config); @@ -52,7 +52,7 @@ public void testParseFilterRules_singleRule() throws Exception { // Setup - config with one rule String config = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700000000, 1700003600], "IPs": ["192.168.1.1"] @@ -72,7 +72,7 @@ public void testParseFilterRules_multipleRules() throws Exception { // Setup - config with multiple rules String config = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700000000, 1700003600], "IPs": ["192.168.1.1"] @@ -92,8 +92,8 @@ public void testParseFilterRules_multipleRules() throws Exception { } @Test(expected = OptOutTrafficFilter.MalformedTrafficFilterConfigException.class) - public void testParseFilterRules_missingBlacklistRequests() throws Exception { - // Setup - config without blacklist_requests field + public void testParseFilterRules_missingDenylistRequests() throws Exception { + // Setup - config without denylist_requests field String config = """ { "other_field": "value" @@ -110,7 +110,7 @@ public void testParseFilterRules_invalidRange_startAfterEnd() throws Exception { // Setup - range where start > end String config = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700003600, 1700000000], "IPs": ["192.168.1.1"] @@ -129,7 +129,7 @@ public void testParseFilterRules_invalidRange_startEqualsEnd() throws Exception // Setup - range where start == end String config = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700000000, 1700000000], "IPs": ["192.168.1.1"] @@ -148,7 +148,7 @@ public void testParseFilterRules_rangeExceeds24Hours() throws Exception { // Setup - range longer than 24 hours (86400 seconds) String config = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700000000, 1700086401], "IPs": ["192.168.1.1"] @@ -167,7 +167,7 @@ public void testParseFilterRules_emptyIPs() throws Exception { // Setup - rule with empty IP list String config = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700000000, 1700003600], "IPs": [] @@ -186,7 +186,7 @@ public void testParseFilterRules_missingIPs() throws Exception { // Setup - rule without IPs field String config = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700000000, 1700003600] } @@ -200,11 +200,11 @@ public void testParseFilterRules_missingIPs() throws Exception { } @Test - public void testIsBlacklisted_matchingIPAndTimestamp() throws Exception { - // Setup - filter with blacklist rule + public void testIsDenylisted_matchingIPAndTimestamp() throws Exception { + // Setup - filter with denylist rule String config = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700000000, 1700003600], "IPs": ["192.168.1.1", "10.0.0.1"] @@ -215,17 +215,17 @@ public void testIsBlacklisted_matchingIPAndTimestamp() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); SqsParsedMessage message = createTestMessage(1700001800, "192.168.1.1"); - // Act & Assert - blacklisted + // Act & Assert - denylisted OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - assertTrue(filter.isBlacklisted(message)); + assertTrue(filter.isDenylisted(message)); } @Test - public void testIsBlacklisted_matchingIPOutsideTimeRange() throws Exception { - // Setup - filter with blacklist rule + public void testIsDenylisted_matchingIPOutsideTimeRange() throws Exception { + // Setup - filter with denylist rule String config = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700000000, 1700003600], "IPs": ["192.168.1.1"] @@ -236,20 +236,20 @@ public void testIsBlacklisted_matchingIPOutsideTimeRange() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - // Act & Assert - message before range not blacklisted + // Act & Assert - message before range not denylisted SqsParsedMessage messageBefore = createTestMessage(1699999999, "192.168.1.1"); - assertFalse(filter.isBlacklisted(messageBefore)); - // Act & Assert - message after range not blacklisted + assertFalse(filter.isDenylisted(messageBefore)); + // Act & Assert - message after range not denylisted SqsParsedMessage messageAfter = createTestMessage(1700003601, "192.168.1.1"); - assertFalse(filter.isBlacklisted(messageAfter)); + assertFalse(filter.isDenylisted(messageAfter)); } @Test - public void testIsBlacklisted_nonMatchingIP() throws Exception { - // Setup - filter with blacklist rule + public void testIsDenylisted_nonMatchingIP() throws Exception { + // Setup - filter with denylist rule String config = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700000000, 1700003600], "IPs": ["192.168.1.1"] @@ -260,17 +260,17 @@ public void testIsBlacklisted_nonMatchingIP() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - // Act & Assert - non-matching IP not blacklisted + // Act & Assert - non-matching IP not denylisted SqsParsedMessage message = createTestMessage(1700001800, "10.0.0.1"); - assertFalse(filter.isBlacklisted(message)); + assertFalse(filter.isDenylisted(message)); } @Test - public void testIsBlacklisted_atRangeBoundaries() throws Exception { - // Setup - filter with blacklist rule + public void testIsDenylisted_atRangeBoundaries() throws Exception { + // Setup - filter with denylist rule String config = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700000000, 1700003600], "IPs": ["192.168.1.1"] @@ -281,21 +281,21 @@ public void testIsBlacklisted_atRangeBoundaries() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - // Act & Assert - message at start boundary (inclusive) blacklisted + // Act & Assert - message at start boundary (inclusive) denylisted SqsParsedMessage messageAtStart = createTestMessage(1700000000, "192.168.1.1"); - assertTrue(filter.isBlacklisted(messageAtStart)); + assertTrue(filter.isDenylisted(messageAtStart)); - // Act & Assert - message at end boundary (inclusive) blacklisted + // Act & Assert - message at end boundary (inclusive) denylisted SqsParsedMessage messageAtEnd = createTestMessage(1700003600, "192.168.1.1"); - assertTrue(filter.isBlacklisted(messageAtEnd)); + assertTrue(filter.isDenylisted(messageAtEnd)); } @Test - public void testIsBlacklisted_multipleRules() throws Exception { - // Setup - multiple blacklist rules + public void testIsDenylisted_multipleRules() throws Exception { + // Setup - multiple denylist rules String config = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700000000, 1700003600], "IPs": ["192.168.1.1"] @@ -313,23 +313,23 @@ public void testIsBlacklisted_multipleRules() throws Exception { // Act & Assert - message matches first rule SqsParsedMessage msg1 = createTestMessage(1700001800, "192.168.1.1"); - assertTrue(filter.isBlacklisted(msg1)); + assertTrue(filter.isDenylisted(msg1)); // Act & Assert - message matches second rule SqsParsedMessage msg2 = createTestMessage(1700011800, "10.0.0.1"); - assertTrue(filter.isBlacklisted(msg2)); + assertTrue(filter.isDenylisted(msg2)); // Act & Assert - message matches neither rule SqsParsedMessage msg3 = createTestMessage(1700005000, "172.16.0.1"); - assertFalse(filter.isBlacklisted(msg3)); + assertFalse(filter.isDenylisted(msg3)); } @Test - public void testIsBlacklisted_nullClientIp() throws Exception { - // Setup - filter with blacklist rule + public void testIsDenylisted_nullClientIp() throws Exception { + // Setup - filter with denylist rule String config = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700000000, 1700003600], "IPs": ["192.168.1.1"] @@ -340,9 +340,9 @@ public void testIsBlacklisted_nullClientIp() throws Exception { Files.writeString(Path.of(TEST_CONFIG_PATH), config); OptOutTrafficFilter filter = new OptOutTrafficFilter(TEST_CONFIG_PATH); - // Act & Assert - message with null IP not blacklisted + // Act & Assert - message with null IP not denylisted SqsParsedMessage message = createTestMessage(1700001800, null); - assertFalse(filter.isBlacklisted(message)); + assertFalse(filter.isDenylisted(message)); } @Test @@ -350,7 +350,7 @@ public void testReloadTrafficFilterConfig_success() throws Exception { // Setup - config with one rule String initialConfig = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700000000, 1700003600], "IPs": ["192.168.1.1"] @@ -367,7 +367,7 @@ public void testReloadTrafficFilterConfig_success() throws Exception { // Setup - update config String updatedConfig = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700000000, 1700003600], "IPs": ["192.168.1.1"] @@ -397,7 +397,7 @@ public void testParseFilterRules_maxValidRange() throws Exception { // Setup - range exactly 24 hours (86400 seconds) - should be valid String config = """ { - "blacklist_requests": [ + "denylist_requests": [ { "range": [1700000000, 1700086400], "IPs": ["192.168.1.1"] From 019f42664ccaa3983ca0ce37a30b6ee32fdac052 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Thu, 4 Dec 2025 13:25:55 -0700 Subject: [PATCH 27/28] undo accidental commit --- .../optout/vertx/OptOutTrafficFilter.java | 167 ------------------ 1 file changed, 167 deletions(-) delete mode 100644 src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java deleted file mode 100644 index 70bd1a09..00000000 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java +++ /dev/null @@ -1,167 +0,0 @@ -package com.uid2.optout.vertx; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Collections; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.charset.StandardCharsets; -import io.vertx.core.json.JsonObject; -import io.vertx.core.json.JsonArray; - -public class OptOutTrafficFilter { - private static final Logger LOGGER = LoggerFactory.getLogger(OptOutTrafficFilter.class); - - private final String trafficFilterConfigPath; - List filterRules; - - /** - * Traffic filter rule defining a time range and a list of IP addresses to exclude - */ - private static class TrafficFilterRule { - private final List range; - private final List ipAddresses; - - TrafficFilterRule(List range, List ipAddresses) { - this.range = range; - this.ipAddresses = ipAddresses; - } - - public long getRangeStart() { - return range.get(0); - } - public long getRangeEnd() { - return range.get(1); - } - public List getIpAddresses() { - return ipAddresses; - } - } - - public static class MalformedTrafficFilterConfigException extends Exception { - public MalformedTrafficFilterConfigException(String message) { - super(message); - } - } - - /** - * Constructor for OptOutTrafficFilter - * - * @param trafficFilterConfigPath S3 path for traffic filter config - * @throws MalformedTrafficFilterConfigException if the traffic filter config is invalid - */ - public OptOutTrafficFilter(String trafficFilterConfigPath) throws MalformedTrafficFilterConfigException { - this.trafficFilterConfigPath = trafficFilterConfigPath; - // Initial filter rules load - this.filterRules = Collections.emptyList(); // start empty - reloadTrafficFilterConfig(); // load ConfigMap - - LOGGER.info("OptOutTrafficFilter initialized: filterRules={}", - filterRules.size()); - } - - /** - * Reload traffic filter config from ConfigMap. - * Expected format: - * { - * "blacklist_requests": [ - * {range: [startTimestamp, endTimestamp], IPs: ["ip1"]}, - * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip2"]}, - * {range: [startTimestamp, endTimestamp], IPs: ["ip1", "ip3"]}, - * ] - * } - * - * Can be called periodically to pick up config changes without restarting. - */ - public void reloadTrafficFilterConfig() throws MalformedTrafficFilterConfigException { - LOGGER.info("Loading traffic filter config from ConfigMap"); - try (InputStream is = Files.newInputStream(Paths.get(trafficFilterConfigPath))) { - String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); - JsonObject filterConfigJson = new JsonObject(content); - - this.filterRules = parseFilterRules(filterConfigJson); - - LOGGER.info("Successfully loaded traffic filter config from ConfigMap: filterRules={}", - filterRules.size()); - - } catch (Exception e) { - LOGGER.warn("No traffic filter config found at: {}", trafficFilterConfigPath, e); - throw new MalformedTrafficFilterConfigException(e.getMessage()); - } - } - - /** - * Parse request filtering rules from JSON config - */ - List parseFilterRules(JsonObject config) throws MalformedTrafficFilterConfigException { - List rules = new ArrayList<>(); - try { - JsonArray blacklistRequests = config.getJsonArray("blacklist_requests"); - if (blacklistRequests == null) { - LOGGER.error("Invalid traffic filter config: blacklist_requests is null"); - throw new MalformedTrafficFilterConfigException("Invalid traffic filter config: blacklist_requests is null"); - } - for (int i = 0; i < blacklistRequests.size(); i++) { - JsonObject ruleJson = blacklistRequests.getJsonObject(i); - - // parse range - var rangeJson = ruleJson.getJsonArray("range"); - List range = new ArrayList<>(); - if (rangeJson != null && rangeJson.size() == 2) { - long start = rangeJson.getLong(0); - long end = rangeJson.getLong(1); - - if (start >= end) { - LOGGER.error("Invalid traffic filter rule: range start must be less than end: {}", ruleJson.encode()); - throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule: range start must be less than end"); - } - range.add(start); - range.add(end); - } - - // parse IPs - var ipAddressesJson = ruleJson.getJsonArray("IPs"); - List ipAddresses = new ArrayList<>(); - if (ipAddressesJson != null) { - for (int j = 0; j < ipAddressesJson.size(); j++) { - ipAddresses.add(ipAddressesJson.getString(j)); - } - } - - // log error and throw exception if rule is invalid - if (range.size() != 2 || ipAddresses.size() == 0 || range.get(1) - range.get(0) > 86400) { // range must be 24 hours or less - LOGGER.error("Invalid traffic filter rule: {}", ruleJson.encode()); - throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule"); - } - - TrafficFilterRule rule = new TrafficFilterRule(range, ipAddresses); - - LOGGER.info("Loaded traffic filter rule: range=[{}, {}], IPs={}", rule.getRangeStart(), rule.getRangeEnd(), rule.getIpAddresses()); - rules.add(rule); - } - return rules; - } catch (Exception e) { - LOGGER.error("Failed to parse traffic filter rules: config={}, error={}", config.encode(), e.getMessage()); - throw new MalformedTrafficFilterConfigException(e.getMessage()); - } - } - - public boolean isBlacklisted(SqsParsedMessage message) { - long timestamp = message.getTimestamp(); - String clientIp = message.getClientIp(); - - for (TrafficFilterRule rule : filterRules) { - if(timestamp >= rule.getRangeStart() && timestamp <= rule.getRangeEnd()) { - if(rule.getIpAddresses().contains(clientIp)) { - return true; - } - }; - } - return false; - } - -} \ No newline at end of file From cabdce5b8c9e42f6d82650f854444fb7aee06210 Mon Sep 17 00:00:00 2001 From: caroline-ttd <157654071+caroline-ttd@users.noreply.github.com> Date: Thu, 4 Dec 2025 14:04:34 -0800 Subject: [PATCH 28/28] Temporarily Suppress libpng CVE-2025-64720 and CVE-2025-65018 (#255) * Temporarily Suppress libpng CVE-2025-64720 and CVE-2025-65018 * Upgrade uid2-shared --- .trivyignore | 8 ++++++-- pom.xml | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.trivyignore b/.trivyignore index 223846fb..82ae41f4 100644 --- a/.trivyignore +++ b/.trivyignore @@ -5,5 +5,9 @@ # UID2-6097 CVE-2025-59375 exp:2025-12-15 -# UID2-6128 -CVE-2025-55163 exp:2025-11-30 +# UID2-6340 +CVE-2025-64720 exp:2026-06-05 + +# UID2-6340 +CVE-2025-65018 exp:2026-06-05 + diff --git a/pom.xml b/pom.xml index 2d85c1fd..7e3305b8 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ 4.5.21 1.1.0 - 11.1.13 + 11.1.91 ${project.version} 5.10.1 5.10.1