Confiant AdQuality: Add config to exclude bidders from scanning, analytics tags (prebid#2670)

Author: IvanTsymbal

extra/modules/confiant-ad-quality/README.md

@@ -52,6 +52,7 @@ And configure
   - `long-interval-attempts` - Maximum attempts with long interval value to try to reconnect to Confiant's Redis server in case any connection error happens. This attempts are used when short-attempts were not successful.
   - `long-interval` - Long time interval in milliseconds after which another one attempt to connect to Redis will be executed.
 - `scan-state-check-interval` - Time interval in milliseconds between periodic calls to check if scan state is enabled on the side of Redis server.
+- `bidders-to-exclude-from-scan` - List of bidders which won't be scanned by Confiant
 
 ```yaml
 hooks:
@@ -73,6 +74,9 @@ hooks:
         long-interval-attempts: 336
         long-interval: 1800000
       scan-state-check-interval: 100000
+      bidders-to-exclude-from-scan: >
+        adyoulike,
+        rtbhouse    
 ```
 
 ## Maintainer contacts

extra/modules/confiant-ad-quality/src/main/java/org/prebid/server/hooks/modules/com/confiant/adquality/config/ConfiantAdQualityModuleConfiguration.java

@@ -1,11 +1,12 @@
 package org.prebid.server.hooks.modules.com.confiant.adquality.config;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import io.vertx.core.Promise;
 import io.vertx.core.Vertx;
 import org.prebid.server.auction.PrivacyEnforcementService;
 import org.prebid.server.hooks.modules.com.confiant.adquality.core.BidsScanner;
-import org.prebid.server.hooks.modules.com.confiant.adquality.core.RedisScanStateChecker;
 import org.prebid.server.hooks.modules.com.confiant.adquality.core.RedisClient;
+import org.prebid.server.hooks.modules.com.confiant.adquality.core.RedisScanStateChecker;
 import org.prebid.server.hooks.modules.com.confiant.adquality.model.RedisConfig;
 import org.prebid.server.hooks.modules.com.confiant.adquality.model.RedisConnectionConfig;
 import org.prebid.server.hooks.modules.com.confiant.adquality.model.RedisRetryConfig;
@@ -32,18 +33,21 @@ public class ConfiantAdQualityModuleConfiguration {
     ConfiantAdQualityModule confiantAdQualityModule(
             @Value("${hooks.modules.confiant-ad-quality.api-key}") String apiKey,
             @Value("${hooks.modules.confiant-ad-quality.scan-state-check-interval}") int scanStateCheckInterval,
+            @Value("${hooks.modules.confiant-ad-quality.bidders-to-exclude-from-scan}") List<String> biddersToExcludeFromScan,
             RedisConfig redisConfig,
             RedisRetryConfig retryConfig,
             Vertx vertx,
-            PrivacyEnforcementService privacyEnforcementService) {
+            PrivacyEnforcementService privacyEnforcementService,
+            ObjectMapper objectMapper) {
+
         final RedisConnectionConfig writeNodeConfig = redisConfig.getWriteNode();
         final RedisClient writeRedisNode = new RedisClient(
                 vertx, writeNodeConfig.getHost(), writeNodeConfig.getPort(), writeNodeConfig.getPassword(), retryConfig, "write node");
         final RedisConnectionConfig readNodeConfig = redisConfig.getReadNode();
         final RedisClient readRedisNode = new RedisClient(
                 vertx, readNodeConfig.getHost(), readNodeConfig.getPort(), readNodeConfig.getPassword(), retryConfig, "read node");
 
-        final BidsScanner bidsScanner = new BidsScanner(writeRedisNode, readRedisNode, apiKey);
+        final BidsScanner bidsScanner = new BidsScanner(writeRedisNode, readRedisNode, apiKey, objectMapper);
         final RedisScanStateChecker redisScanStateChecker = new RedisScanStateChecker(bidsScanner, scanStateCheckInterval, vertx);
 
         final Promise<Void> scannerPromise = Promise.promise();
@@ -52,9 +56,14 @@ ConfiantAdQualityModule confiantAdQualityModule(
         bidsScanner.start(scannerPromise);
 
         return new ConfiantAdQualityModule(List.of(
-                new ConfiantAdQualityBidResponsesScanHook(bidsScanner, privacyEnforcementService)));
+                new ConfiantAdQualityBidResponsesScanHook(bidsScanner, biddersToExcludeFromScan, privacyEnforcementService)));
     }
 
+    @Bean
+    ObjectMapper objectMapper() {
+        return new ObjectMapper();
+    };
+
     @Bean
     @ConfigurationProperties(prefix = "hooks.modules.confiant-ad-quality.redis-config")
     RedisConfig redisConfig() {

extra/modules/confiant-ad-quality/src/main/java/org/prebid/server/hooks/modules/com/confiant/adquality/core/AnalyticsMapper.java

@@ -0,0 +1,71 @@
+package org.prebid.server.hooks.modules.com.confiant.adquality.core;
+
+import com.iab.openrtb.response.Bid;
+import org.prebid.server.auction.model.BidderResponse;
+import org.prebid.server.bidder.model.BidderBid;
+import org.prebid.server.hooks.modules.com.confiant.adquality.v1.model.analytics.ActivityImpl;
+import org.prebid.server.hooks.modules.com.confiant.adquality.v1.model.analytics.AppliedToImpl;
+import org.prebid.server.hooks.modules.com.confiant.adquality.v1.model.analytics.ResultImpl;
+import org.prebid.server.hooks.modules.com.confiant.adquality.v1.model.analytics.TagsImpl;
+import org.prebid.server.hooks.v1.analytics.AppliedTo;
+import org.prebid.server.hooks.v1.analytics.Result;
+import org.prebid.server.hooks.v1.analytics.Tags;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class AnalyticsMapper {
+
+    private static final String AD_QUALITY_SCAN = "ad-scan";
+    private static final String SUCCESS_STATUS = "success";
+    private static final String SKIPPED = "skipped";
+    private static final String INSPECTED_HAS_ISSUE = "inspected-has-issue";
+    private static final String INSPECTED_NO_ISSUES = "inspected-no-issues";
+
+    public static Tags toAnalyticsTags(List<BidderResponse> bidderResponsesWithIssues,
+                                       List<BidderResponse> bidderResponsesWithoutIssues,
+                                       List<BidderResponse> bidderResponsesNotScanned) {
+
+        return TagsImpl.of(Collections.singletonList(ActivityImpl.of(
+                AD_QUALITY_SCAN,
+                SUCCESS_STATUS,
+                toActivityResults(bidderResponsesWithIssues, bidderResponsesWithoutIssues, bidderResponsesNotScanned))));
+    }
+
+    private static List<Result> toActivityResults(List<BidderResponse> bidderResponsesWithIssues,
+                                                  List<BidderResponse> bidderResponsesWithoutIssues,
+                                                  List<BidderResponse> bidderResponsesNotScanned) {
+
+        final List<Result> results = new ArrayList<>();
+        if (!bidderResponsesNotScanned.isEmpty()) {
+            results.add(ResultImpl.of(SKIPPED, null, toAppliedTo(bidderResponsesNotScanned)));
+        }
+        if (!bidderResponsesWithIssues.isEmpty()) {
+            results.add(ResultImpl.of(INSPECTED_HAS_ISSUE, null, toAppliedTo(bidderResponsesWithIssues)));
+        }
+        if (!bidderResponsesWithoutIssues.isEmpty()) {
+            results.add(ResultImpl.of(INSPECTED_NO_ISSUES, null, toAppliedTo(bidderResponsesWithoutIssues)));
+        }
+
+        return results;
+    }
+
+    private static AppliedTo toAppliedTo(List<BidderResponse> bidderResponses) {
+        final List<Bid> bids = toBids(bidderResponses);
+        return AppliedToImpl.builder()
+                .bidders(bidderResponses.stream().map(BidderResponse::getBidder).toList())
+                .impIds(bids.stream().map(Bid::getImpid).toList())
+                .bidIds(bids.stream().map(Bid::getId).toList())
+                .build();
+    }
+
+    private static List<Bid> toBids(List<BidderResponse> bidderResponses) {
+        return bidderResponses.stream()
+                .map(BidderResponse::getSeatBid)
+                .flatMap(seatBid -> seatBid.getBids().stream())
+                .map(BidderBid::getBid)
+                .collect(Collectors.toList());
+    }
+}

extra/modules/confiant-ad-quality/src/main/java/org/prebid/server/hooks/modules/com/confiant/adquality/core/BidsScanResult.java

@@ -1,49 +1,49 @@
 package org.prebid.server.hooks.modules.com.confiant.adquality.core;
 
+import lombok.Builder;
+import lombok.Value;
 import org.prebid.server.auction.model.BidderResponse;
 import org.prebid.server.hooks.modules.com.confiant.adquality.model.BidScanResult;
-import org.prebid.server.hooks.modules.com.confiant.adquality.model.OperationResult;
+import org.prebid.server.hooks.modules.com.confiant.adquality.model.GroupByIssues;
 
+import java.util.ArrayList;
 import java.util.List;
-import java.util.Optional;
-import java.util.stream.IntStream;
 
+@Builder
+@Value(staticConstructor = "of")
 public class BidsScanResult {
 
-    private final OperationResult<List<BidScanResult>> result;
+    List<BidScanResult> bidScanResults;
 
-    public BidsScanResult(OperationResult<List<BidScanResult>> result) {
-        this.result = result;
-    }
+    List<String> debugMessages;
 
-    public boolean hasIssues() {
-        return Optional.ofNullable(result.getValue())
-                .map(scanResults -> scanResults.stream()
-                        .anyMatch(result -> Optional.ofNullable(result.getIssues())
-                                .map(issues -> !issues.isEmpty())
-                                .orElse(false)))
-                .orElse(false);
-    }
+    public GroupByIssues<BidderResponse> toGroupByIssues(List<BidderResponse> bidderResponses) {
+        final List<BidderResponse> bidderResponsesWithIssues = new ArrayList<>();
+        final List<BidderResponse> bidderResponsesWithoutIssues = new ArrayList<>();
+        final int groupSize = bidderResponses.size();
 
-    public List<BidderResponse> filterValidResponses(List<BidderResponse> responses) {
-        return IntStream.range(0, responses.size())
-                .filter(ind -> !hasIssuesByBidIndex(ind))
-                .mapToObj(responses::get)
-                .toList();
+        for (int i = 0; i < groupSize; i++) {
+            if (hasIssuesByBidIndex(i)) {
+                bidderResponsesWithIssues.add(bidderResponses.get(i));
+            } else {
+                bidderResponsesWithoutIssues.add(bidderResponses.get(i));
+            }
+        }
+
+        return GroupByIssues.<BidderResponse>builder()
+                .withIssues(bidderResponsesWithIssues)
+                .withoutIssues(bidderResponsesWithoutIssues)
+                .build();
     }
 
     public List<String> getIssuesMessages() {
-        return result.getValue().stream()
+        return bidScanResults.stream()
                 .map(r -> r.getTagKey() + ": " + (r.getIssues() == null ? "no issues" : r.getIssues().toString()))
                 .toList();
     }
 
-    public List<String> getDebugMessages() {
-        return result.getDebugMessages();
-    }
-
     private boolean hasIssuesByBidIndex(Integer ind) {
-        final BidScanResult bidResult = result.getValue().get(ind);
-        return bidResult != null && bidResult.getIssues() != null && bidResult.getIssues().size() > 0;
+        final BidScanResult bidResult = bidScanResults.size() > ind ? bidScanResults.get(ind) : null;
+        return bidResult != null && bidResult.getIssues() != null && !bidResult.getIssues().isEmpty();
     }
 }

extra/modules/confiant-ad-quality/src/main/java/org/prebid/server/hooks/modules/com/confiant/adquality/core/BidsScanner.java

@@ -7,15 +7,14 @@
 import io.vertx.redis.client.RedisAPI;
 import io.vertx.redis.client.Response;
 import org.prebid.server.hooks.modules.com.confiant.adquality.model.BidScanResult;
-import org.prebid.server.hooks.modules.com.confiant.adquality.model.OperationResult;
 import org.prebid.server.hooks.modules.com.confiant.adquality.model.RedisBidsData;
 
 import java.util.Collections;
 import java.util.List;
 
 public class BidsScanner {
 
-    private final RedisParser redisParser = new RedisParser();
+    private final RedisParser redisParser;
 
     private final String apiKey;
 
@@ -25,12 +24,19 @@ public class BidsScanner {
 
     private volatile Boolean isScanDisabled = true;
 
-    private final ObjectMapper objectMapper = new ObjectMapper();
+    private final ObjectMapper objectMapper;
+
+    public BidsScanner(
+            RedisClient writeRedisNode,
+            RedisClient readRedisNode,
+            String apiKey,
+            ObjectMapper objectMapper) {
 
-    public BidsScanner(RedisClient writeRedisNode, RedisClient readRedisNode, String apiKey) {
         this.writeRedisNode = writeRedisNode;
         this.readRedisNode = readRedisNode;
         this.apiKey = apiKey;
+        this.objectMapper = objectMapper;
+        this.redisParser = new RedisParser(objectMapper);
     }
 
     public void start(Promise<Void> startFuture) {
@@ -63,15 +69,15 @@ public Future<BidsScanResult> submitBids(RedisBidsData bids) {
 
                     readRedisNodeAPI.evalsha(readArgs, response -> {
                         if (response.result() != null) {
-                            final OperationResult<List<BidScanResult>> parserResult = redisParser
+                            final BidsScanResult parserResult = redisParser
                                     .parseBidsScanResult(response.result().toString());
-                            final boolean isAnyRoSkipped = parserResult.getValue()
+                            final boolean isAnyRoSkipped = parserResult.getBidScanResults()
                                     .stream().anyMatch(BidScanResult::isRoSkipped);
 
                             if (isAnyRoSkipped) {
                                 reSubmitBidsToWriteNode(readArgs, scanResult);
                             } else {
-                                scanResult.complete(new BidsScanResult(parserResult));
+                                scanResult.complete(parserResult);
                             }
                         } else {
                             scanResult.complete(getEmptyScanResult());
@@ -94,10 +100,10 @@ private void reSubmitBidsToWriteNode(List<String> readArgs, Promise<BidsScanResu
             final List<String> writeArgs = readArgs.stream().limit(4).toList();
             writeRedisAPI.evalsha(writeArgs, response -> {
                 if (response.result() != null) {
-                    final OperationResult<List<BidScanResult>> parserResult = redisParser
+                    final BidsScanResult parserResult = redisParser
                             .parseBidsScanResult(response.result().toString());
 
-                    scanResult.complete(new BidsScanResult(parserResult));
+                    scanResult.complete(parserResult);
                 } else {
                     scanResult.complete(getEmptyScanResult());
                 }
@@ -132,9 +138,9 @@ private String toBidsAsJson(RedisBidsData bids) {
     }
 
     private BidsScanResult getEmptyScanResult() {
-        return new BidsScanResult(OperationResult.<List<BidScanResult>>builder()
-                .value(Collections.emptyList())
+        return BidsScanResult.builder()
+                .bidScanResults(Collections.emptyList())
                 .debugMessages(Collections.emptyList())
-                .build());
+                .build();
     }
 }

extra/modules/confiant-ad-quality/src/main/java/org/prebid/server/hooks/modules/com/confiant/adquality/core/RedisClient.java

@@ -35,6 +35,7 @@ public RedisClient(
             String password,
             RedisRetryConfig retryConfig,
             String type) {
+
         this.vertx = vertx;
         this.retryConfig = retryConfig;
         this.options = new RedisOptions().setConnectionString("redis://:" + password + "@" + host + ":" + port);

extra/modules/confiant-ad-quality/src/main/java/org/prebid/server/hooks/modules/com/confiant/adquality/core/RedisParser.java

@@ -5,7 +5,6 @@
 import io.vertx.core.logging.Logger;
 import io.vertx.core.logging.LoggerFactory;
 import org.prebid.server.hooks.modules.com.confiant.adquality.model.BidScanResult;
-import org.prebid.server.hooks.modules.com.confiant.adquality.model.OperationResult;
 import org.prebid.server.hooks.modules.com.confiant.adquality.model.RedisError;
 
 import java.util.Arrays;
@@ -16,18 +15,22 @@ public class RedisParser {
 
     private static final Logger logger = LoggerFactory.getLogger(RedisParser.class);
 
-    public OperationResult<List<BidScanResult>> parseBidsScanResult(String redisResponse) {
-        final ObjectMapper objectMapper = new ObjectMapper();
+    private final ObjectMapper objectMapper;
 
+    public RedisParser(ObjectMapper objectMapper) {
+        this.objectMapper = objectMapper;
+    }
+
+    public BidsScanResult parseBidsScanResult(String redisResponse) {
         try {
             final BidScanResult[][][] draftResponse = objectMapper.readValue(redisResponse, BidScanResult[][][].class);
             final List<BidScanResult> scanResultsFlat = Arrays.stream(draftResponse)
                     .flatMap(Arrays::stream)
                     .map(array -> array[0])
                     .toList();
 
-            return OperationResult.<List<BidScanResult>>builder()
-                    .value(scanResultsFlat)
+            return BidsScanResult.builder()
+                    .bidScanResults(scanResultsFlat)
                     .debugMessages(Collections.emptyList())
                     .build();
         } catch (JsonProcessingException resultParse) {
@@ -39,8 +42,8 @@ public OperationResult<List<BidScanResult>> parseBidsScanResult(String redisResp
                 message = String.format("Error during parse redis response: %s", redisResponse);
             }
             logger.info(message);
-            return OperationResult.<List<BidScanResult>>builder()
-                    .value(Collections.emptyList())
+            return BidsScanResult.builder()
+                    .bidScanResults(Collections.emptyList())
                     .debugMessages(List.of(message))
                     .build();
         }

extra/modules/confiant-ad-quality/src/main/java/org/prebid/server/hooks/modules/com/confiant/adquality/core/RedisScanStateChecker.java

@@ -14,6 +14,7 @@ public RedisScanStateChecker(
             BidsScanner bidsScanner,
             long scanStateCheckInterval,
             Vertx vertx) {
+
         this.bidsScanner = bidsScanner;
         this.scanStateCheckInterval = scanStateCheckInterval;
         this.vertx = vertx;

extra/modules/confiant-ad-quality/src/main/java/org/prebid/server/hooks/modules/com/confiant/adquality/model/GroupByIssues.java

@@ -0,0 +1,15 @@
+package org.prebid.server.hooks.modules.com.confiant.adquality.model;
+
+import lombok.Builder;
+import lombok.Value;
+
+import java.util.List;
+
+@Builder(toBuilder = true)
+@Value
+public class GroupByIssues<T> {
+
+    List<T> withIssues;
+
+    List<T> withoutIssues;
+}