Core: Improve traceability for Prebid Cache stores

src/main/java/org/prebid/server/cache/CoreCacheService.java

@@ -7,6 +7,7 @@
 import io.vertx.core.Future;
 import io.vertx.core.MultiMap;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.prebid.server.auction.model.AuctionContext;
 import org.prebid.server.auction.model.BidInfo;
@@ -63,6 +64,8 @@ public class CoreCacheService {
     private static final Logger logger = LoggerFactory.getLogger(CoreCacheService.class);
 
     private static final String BID_WURL_ATTRIBUTE = "wurl";
+    private static final String TRACE_INFO_SEPARATOR = "-";
+    private static final int MAX_DATACENTER_REGION_LENGTH = 4;
 
     private final HttpClient httpClient;
     private final URL endpointUrl;
@@ -78,13 +81,18 @@ public class CoreCacheService {
     private final MultiMap cacheHeaders;
     private final Map<String, List<String>> debugHeaders;
 
+    private final boolean appendTraceInfoToCacheId;
+    private final String datacenterRegion;
+
     public CoreCacheService(
             HttpClient httpClient,
             URL endpointUrl,
             String cachedAssetUrlTemplate,
             long expectedCacheTimeMs,
             String apiKey,
             boolean isApiKeySecured,
+            boolean appendTraceInfoToCacheId,
+            String datacenterRegion,
             VastModifier vastModifier,
             EventsService eventsService,
             Metrics metrics,
@@ -107,6 +115,9 @@ public CoreCacheService(
                 ? HttpUtil.headers().add(HttpUtil.X_PBC_API_KEY_HEADER, Objects.requireNonNull(apiKey))
                 : HttpUtil.headers();
         debugHeaders = HttpUtil.toDebugHeaders(cacheHeaders);
+
+        this.appendTraceInfoToCacheId = appendTraceInfoToCacheId;
+        this.datacenterRegion = normalizeDatacenterRegion(datacenterRegion);
     }
 
     public String getEndpointHost() {
@@ -138,8 +149,10 @@ public String cacheVideoDebugLog(CachedDebugLog cachedDebugLog, Integer videoCac
         return cacheKey;
     }
 
-    private CachedCreative makeDebugCacheCreative(CachedDebugLog videoCacheDebugLog, String hbCacheId,
+    private CachedCreative makeDebugCacheCreative(CachedDebugLog videoCacheDebugLog,
+                                                  String hbCacheId,
                                                   Integer videoCacheTtl) {
+
         final JsonNode value = mapper.mapper().valueToTree(videoCacheDebugLog.buildCacheBody());
         videoCacheDebugLog.setCacheKey(hbCacheId);
         return CachedCreative.of(BidPutObject.builder()
@@ -211,6 +224,7 @@ private List<CachedCreative> updatePutObjects(List<BidPutObject> bidPutObjects,
                         .bidid(null)
                         .bidder(null)
                         .timestamp(null)
+                        .key(resolveCacheKey(accountId, putObject.getKey()))
                         .value(vastModifier.modifyVastXml(isEventsEnabled,
                                 allowedBidders,
                                 putObject,
@@ -268,7 +282,8 @@ private Future<CacheServiceResult> doCacheOpenrtb(List<CacheBid> bids,
         final List<CachedCreative> cachedCreatives = Stream.concat(
                         bids.stream().map(cacheBid ->
                                 createJsonPutObjectOpenrtb(cacheBid, accountId, eventsContext)),
-                        videoBids.stream().map(videoBid -> createXmlPutObjectOpenrtb(videoBid, requestId, hbCacheId)))
+                        videoBids.stream().map(videoBid ->
+                                createXmlPutObjectOpenrtb(videoBid, requestId, hbCacheId, accountId)))
                 .collect(Collectors.toCollection(ArrayList::new));
 
         if (cachedCreatives.isEmpty()) {
@@ -385,38 +400,43 @@ private CachedCreative createJsonPutObjectOpenrtb(CacheBid cacheBid,
             bidObjectNode.put(BID_WURL_ATTRIBUTE, eventUrl);
         }
 
+        final String resolvedCacheKey = resolveCacheKey(accountId);
+
         final BidPutObject payload = BidPutObject.builder()
                 .aid(eventsContext.getAuctionId())
                 .type("json")
+                .key(resolvedCacheKey)
                 .value(bidObjectNode)
                 .ttlseconds(cacheBid.getTtl())
                 .build();
 
         return CachedCreative.of(payload, creativeSizeFromAdm(bid.getAdm()));
     }
 
-    private CachedCreative createXmlPutObjectOpenrtb(CacheBid cacheBid, String requestId, String hbCacheId) {
+    private CachedCreative createXmlPutObjectOpenrtb(CacheBid cacheBid,
+                                                     String requestId,
+                                                     String hbCacheId,
+                                                     String accountId) {
+
         final BidInfo bidInfo = cacheBid.getBidInfo();
         final Bid bid = bidInfo.getBid();
         final String vastXml = bid.getAdm();
 
-        final String customCacheKey = resolveCustomCacheKey(hbCacheId, bidInfo.getCategory());
-
         final BidPutObject payload = BidPutObject.builder()
                 .aid(requestId)
                 .type("xml")
+                .key(resolveCacheKey(accountId, hbCacheId, bidInfo.getCategory()))
                 .value(vastXml != null ? new TextNode(vastXml) : null)
                 .ttlseconds(cacheBid.getTtl())
-                .key(customCacheKey)
                 .build();
 
         return CachedCreative.of(payload, creativeSizeFromTextNode(payload.getValue()));
     }
 
-    private static String resolveCustomCacheKey(String hbCacheId, String category) {
+    private static String formatCategoryMappedCacheKey(String hbCacheId, String category) {
         return StringUtils.isNoneEmpty(category, hbCacheId)
                 ? "%s_%s".formatted(category, hbCacheId)
-                : null;
+                : hbCacheId;
     }
 
     private String generateWinUrl(String bidId,
@@ -515,10 +535,16 @@ private static String resolveVideoBidUuid(String uuid, String hbCacheId) {
     }
 
     private void updateCreativeMetrics(String accountId, List<CachedCreative> cachedCreatives) {
-        for (final CachedCreative cachedCreative : cachedCreatives) {
-            metrics.updateCacheCreativeSize(accountId,
-                    cachedCreative.getSize(),
-                    resolveCreativeTypeName(cachedCreative.getPayload()));
+        for (CachedCreative cachedCreative : cachedCreatives) {
+            final BidPutObject payload = cachedCreative.getPayload();
+            final MetricName creativeType = resolveCreativeTypeName(payload);
+            final Integer creativeTtl = ObjectUtils.defaultIfNull(payload.getTtlseconds(), payload.getExpiry());
+
+            if (creativeTtl != null) {
+                metrics.updateCacheCreativeTtl(accountId, creativeTtl, creativeType);
+            }
+
+            metrics.updateCacheCreativeSize(accountId, cachedCreative.getSize(), creativeType);
         }
     }
 
@@ -553,4 +579,48 @@ private BidCacheRequest toBidCacheRequest(List<CachedCreative> cachedCreatives)
                 .map(CachedCreative::getPayload)
                 .toList());
     }
+
+    private String resolveCacheKey(String accountId, String existingKey, String category) {
+        final String resolvedCacheKey = resolveCacheKey(accountId, existingKey);
+        return formatCategoryMappedCacheKey(resolvedCacheKey, category);
+
+    }
+
+    private String resolveCacheKey(String accountId) {
+        return resolveCacheKey(accountId, null);
+    }
+
+    private String resolveCacheKey(String accountId, String existingCacheKey) {
+        if (!appendTraceInfoToCacheId || existingCacheKey != null) {
+            return existingCacheKey;
+        }
+
+        final boolean isDatacenterNamePopulated = StringUtils.isNotBlank(datacenterRegion);
+        final int separatorCount = isDatacenterNamePopulated ? 2 : 1;
+        final int accountIdLength = accountId.length();
+        final int traceInfoLength = isDatacenterNamePopulated
+                ? accountIdLength + datacenterRegion.length() + separatorCount
+                : accountIdLength + separatorCount;
+
+        final String cacheKey = idGenerator.generateId();
+        if (cacheKey == null || traceInfoLength >= (cacheKey.length() / 2)) {
+            return null;
+        }
+
+        final String substring = cacheKey.substring(0, cacheKey.length() - traceInfoLength);
+        return isDatacenterNamePopulated
+                ? accountId + TRACE_INFO_SEPARATOR + datacenterRegion + TRACE_INFO_SEPARATOR + substring
+                : accountId + TRACE_INFO_SEPARATOR + substring;
+    }
+
+    private static String normalizeDatacenterRegion(String datacenterRegion) {
+        if (datacenterRegion == null) {
+            return null;
+        }
+
+        final String trimmedDatacenterRegion = datacenterRegion.trim();
+        return trimmedDatacenterRegion.length() > MAX_DATACENTER_REGION_LENGTH
+                ? trimmedDatacenterRegion.substring(0, MAX_DATACENTER_REGION_LENGTH)
+                : trimmedDatacenterRegion;
+    }
 }

src/main/java/org/prebid/server/metric/CacheCreativeTtlMetrics.java

@@ -0,0 +1,19 @@
+package org.prebid.server.metric;
+
+import com.codahale.metrics.MetricRegistry;
+
+import java.util.Objects;
+import java.util.function.Function;
+
+public class CacheCreativeTtlMetrics extends UpdatableMetrics {
+
+    CacheCreativeTtlMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) {
+        super(Objects.requireNonNull(metricRegistry),
+                Objects.requireNonNull(counterType),
+                nameCreator(Objects.requireNonNull(prefix)));
+    }
+
+    private static Function<MetricName, String> nameCreator(String prefix) {
+        return metricName -> "%s.creative_ttl.%s".formatted(prefix, metricName);
+    }
+}

src/main/java/org/prebid/server/metric/CacheMetrics.java

@@ -12,6 +12,7 @@ class CacheMetrics extends UpdatableMetrics {
 
     private final RequestMetrics requestsMetrics;
     private final CacheCreativeSizeMetrics cacheCreativeSizeMetrics;
+    private final CacheCreativeTtlMetrics cacheCreativeTtlMetrics;
 
     CacheMetrics(MetricRegistry metricRegistry, CounterType counterType) {
         super(
@@ -21,6 +22,7 @@ class CacheMetrics extends UpdatableMetrics {
 
         requestsMetrics = new RequestMetrics(metricRegistry, counterType, createPrefix());
         cacheCreativeSizeMetrics = new CacheCreativeSizeMetrics(metricRegistry, counterType, createPrefix());
+        cacheCreativeTtlMetrics = new CacheCreativeTtlMetrics(metricRegistry, counterType, createPrefix());
     }
 
     CacheMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) {
@@ -31,6 +33,7 @@ class CacheMetrics extends UpdatableMetrics {
 
         requestsMetrics = new RequestMetrics(metricRegistry, counterType, createPrefix(prefix));
         cacheCreativeSizeMetrics = new CacheCreativeSizeMetrics(metricRegistry, counterType, createPrefix(prefix));
+        cacheCreativeTtlMetrics = new CacheCreativeTtlMetrics(metricRegistry, counterType, createPrefix(prefix));
     }
 
     private static String createPrefix(String prefix) {
@@ -52,4 +55,8 @@ RequestMetrics requests() {
     CacheCreativeSizeMetrics creativeSize() {
         return cacheCreativeSizeMetrics;
     }
+
+    CacheCreativeTtlMetrics creativeTtl() {
+        return cacheCreativeTtlMetrics;
+    }
 }

src/main/java/org/prebid/server/metric/Metrics.java

@@ -694,6 +694,11 @@ public void updateAccountModuleDurationMetric(Account account, String moduleCode
         }
     }
 
+    public void updateCacheCreativeTtl(String accountId, Integer creativeTtl, MetricName creativeType) {
+        cache().creativeTtl().updateHistogram(creativeType, creativeTtl);
+        forAccount(accountId).cache().creativeTtl().updateHistogram(creativeType, creativeTtl);
+    }
+
     private static class HookMetricMapper {
 
         private static final EnumMap<ExecutionStatus, MetricName> STATUS_TO_METRIC =

src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java

@@ -164,6 +164,8 @@ CoreCacheService cacheService(
             @Value("${auction.cache.expected-request-time-ms}") long expectedCacheTimeMs,
             @Value("${pbc.api.key:#{null}}") String apiKey,
             @Value("${cache.api-key-secured:false}") boolean apiKeySecured,
+            @Value("${cache.append-trace-info-to-cache-id:false}") boolean appendTraceInfoToCacheId,
+            @Value("${datacenter-region:#{null}}") String datacenterRegion,
             VastModifier vastModifier,
             EventsService eventsService,
             HttpClient httpClient,
@@ -178,6 +180,8 @@ CoreCacheService cacheService(
                 expectedCacheTimeMs,
                 apiKey,
                 apiKeySecured,
+                appendTraceInfoToCacheId,
+                datacenterRegion,
                 vastModifier,
                 eventsService,
                 metrics,

src/test/groovy/org/prebid/server/functional/model/request/cache/BidCachePut.groovy

@@ -14,4 +14,5 @@ class BidCachePut implements ObjectMapperWrapper {
     String bidder
     Long timestamp
     String aid
+    String key
 }