diff --git a/extra/modules/optable-targeting/src/main/java/org/prebid/server/hooks/modules/optable/targeting/v1/core/Cache.java b/extra/modules/optable-targeting/src/main/java/org/prebid/server/hooks/modules/optable/targeting/v1/core/Cache.java index 6aab8698a4a..97fdddf4a89 100644 --- a/extra/modules/optable-targeting/src/main/java/org/prebid/server/hooks/modules/optable/targeting/v1/core/Cache.java +++ b/extra/modules/optable-targeting/src/main/java/org/prebid/server/hooks/modules/optable/targeting/v1/core/Cache.java @@ -11,8 +11,8 @@ public class Cache { - private static final String APP_CODE = "prebid-Java"; - private static final String APPLICATION = "optable-targeting"; + private static final String APPLICATION = "prebid-Java"; + private static final String APP_CODE = "optable-targeting"; private final PbcStorageService cacheService; private final JacksonMapper mapper; diff --git a/src/main/java/org/prebid/server/cache/BasicPbcStorageService.java b/src/main/java/org/prebid/server/cache/BasicPbcStorageService.java index 956330cd1ca..a665f00c845 100644 --- a/src/main/java/org/prebid/server/cache/BasicPbcStorageService.java +++ b/src/main/java/org/prebid/server/cache/BasicPbcStorageService.java @@ -11,10 +11,13 @@ import org.prebid.server.exception.PreBidException; import org.prebid.server.json.DecodeException; import org.prebid.server.json.JacksonMapper; +import org.prebid.server.metric.MetricName; +import org.prebid.server.metric.Metrics; import org.prebid.server.util.HttpUtil; import org.prebid.server.vertx.httpclient.HttpClient; import java.net.URL; +import java.time.Clock; import java.util.Objects; public class BasicPbcStorageService implements PbcStorageService { @@ -27,18 +30,24 @@ public class BasicPbcStorageService implements PbcStorageService { private final String apiKey; private final int callTimeoutMs; private final JacksonMapper mapper; + private final Clock clock; + private final Metrics metrics; public BasicPbcStorageService(HttpClient httpClient, URL endpointUrl, String apiKey, int callTimeoutMs, - JacksonMapper mapper) { + JacksonMapper mapper, + Clock clock, + Metrics metrics) { this.httpClient = Objects.requireNonNull(httpClient); this.endpointUrl = Objects.requireNonNull(endpointUrl); this.apiKey = Objects.requireNonNull(apiKey); this.callTimeoutMs = callTimeoutMs; this.mapper = Objects.requireNonNull(mapper); + this.clock = Objects.requireNonNull(clock); + this.metrics = Objects.requireNonNull(metrics); } @Override @@ -55,20 +64,28 @@ public Future storeEntry(String key, return Future.failedFuture(e); } + final String valueToStore = prepareValueForStoring(value, type); final ModuleCacheRequest moduleCacheRequest = ModuleCacheRequest.of( constructEntryKey(key, appCode), type, - prepareValueForStoring(value, type), + valueToStore, application, ttlseconds); + updateCreativeMetrics(valueToStore, type, ttlseconds, appCode); + + final long startTime = clock.millis(); return httpClient.post( endpointUrl.toString(), securedCallHeaders(), mapper.encodeToString(moduleCacheRequest), callTimeoutMs) - .compose(response -> processStoreResponse(response.getStatusCode(), response.getBody())); + .compose(response -> processStoreResponse( + response.getStatusCode(), + response.getBody(), + startTime, + appCode)); } @@ -99,6 +116,19 @@ private static void validateStoreData(String key, } } + private void updateCreativeMetrics(String value, StorageDataType type, Integer ttlseconds, String appCode) { + final MetricName metricName = switch (type) { + case XML -> MetricName.xml; + case JSON -> MetricName.json; + case TEXT -> MetricName.text; + }; + + metrics.updateModuleStorageCacheEntrySize(appCode, value.length(), metricName); + if (ttlseconds != null) { + metrics.updateModuleStorageCacheEntryTtl(appCode, ttlseconds, metricName); + } + } + private static String prepareValueForStoring(String value, StorageDataType type) { return type == StorageDataType.TEXT ? new String(Base64.encodeBase64(value.getBytes())) @@ -114,31 +144,35 @@ private String constructEntryKey(String key, String moduleCode) { return MODULE_KEY_PREFIX + MODULE_KEY_DELIMETER + moduleCode + MODULE_KEY_DELIMETER + key; } - private Future processStoreResponse(int statusCode, String responseBody) { + private Future processStoreResponse(int statusCode, String responseBody, long startTime, String appCode) { if (statusCode != 204) { + metrics.updateModuleStorageCacheWriteRequestTime(appCode, clock.millis() - startTime, MetricName.err); throw new PreBidException("HTTP status code: '%s', body: '%s' " .formatted(statusCode, responseBody)); } + metrics.updateModuleStorageCacheWriteRequestTime(appCode, clock.millis() - startTime, MetricName.ok); return Future.succeededFuture(); } @Override - public Future retrieveEntry(String key, - String appCode, - String application) { - + public Future retrieveEntry(String key, String appCode, String application) { try { validateRetrieveData(key, application, appCode); } catch (PreBidException e) { return Future.failedFuture(e); } + final long startTime = clock.millis(); return httpClient.get( getRetrieveEndpoint(key, appCode, application), securedCallHeaders(), callTimeoutMs) - .map(response -> toModuleCacheResponse(response.getStatusCode(), response.getBody())); + .map(response -> toModuleCacheResponse( + response.getStatusCode(), + response.getBody(), + startTime, + appCode)); } @@ -165,11 +199,18 @@ private String getRetrieveEndpoint(String key, + "&a=" + application; } - private ModuleCacheResponse toModuleCacheResponse(int statusCode, String responseBody) { + private ModuleCacheResponse toModuleCacheResponse(int statusCode, + String responseBody, + long startTime, + String application) { + if (statusCode != 200) { + metrics.updateModuleStorageCacheReadRequestTime(application, clock.millis() - startTime, MetricName.err); throw new PreBidException("HTTP status code " + statusCode); } + metrics.updateModuleStorageCacheReadRequestTime(application, clock.millis() - startTime, MetricName.ok); + final ModuleCacheResponse moduleCacheResponse; try { moduleCacheResponse = mapper.decodeValue(responseBody, ModuleCacheResponse.class); diff --git a/src/main/java/org/prebid/server/cache/CoreCacheService.java b/src/main/java/org/prebid/server/cache/CoreCacheService.java index 908fe9e2905..e3769216570 100644 --- a/src/main/java/org/prebid/server/cache/CoreCacheService.java +++ b/src/main/java/org/prebid/server/cache/CoreCacheService.java @@ -208,13 +208,6 @@ private BidCacheResponse processVtrackWriteCacheResponse(int statusCode, return bidCacheResponse; } - private Future failVtrackCacheWriteResponse(Throwable exception, String accountId, long startTime) { - if (exception instanceof PreBidException) { - metrics.updateVtrackCacheWriteRequestTime(accountId, clock.millis() - startTime, MetricName.err); - } - return failResponse(exception); - } - public Future cachePutObjects(List bidPutObjects, Boolean isEventsEnabled, Set biddersAllowingVastUpdate, @@ -671,7 +664,7 @@ public Future getCachedObject(String key, String ch, Timeout final long startTime = clock.millis(); return httpClient.get(url, cacheHeaders, remainingTimeout) .map(response -> processVtrackReadResponse(response, startTime)) - .recover(CoreCacheService::failResponse); + .recover(exception -> failVtrackCacheReadResponse(exception, startTime)); } private HttpClientResponse processVtrackReadResponse(HttpClientResponse response, long startTime) { @@ -683,16 +676,29 @@ private HttpClientResponse processVtrackReadResponse(HttpClientResponse response return response; } - metrics.updateVtrackCacheReadRequestTime(clock.millis() - startTime, MetricName.err); - try { final CacheErrorResponse errorResponse = mapper.decodeValue(body, CacheErrorResponse.class); + metrics.updateVtrackCacheReadRequestTime(clock.millis() - startTime, MetricName.err); return HttpClientResponse.of(statusCode, response.getHeaders(), errorResponse.getMessage()); } catch (DecodeException e) { throw new PreBidException("Cannot parse response: " + body, e); } } + private Future failVtrackCacheWriteResponse(Throwable exception, String accountId, long startTime) { + if (exception instanceof PreBidException) { + metrics.updateVtrackCacheWriteRequestTime(accountId, clock.millis() - startTime, MetricName.err); + } + return failResponse(exception); + } + + private Future failVtrackCacheReadResponse(Throwable exception, long startTime) { + if (exception instanceof PreBidException) { + metrics.updateVtrackCacheReadRequestTime(clock.millis() - startTime, MetricName.err); + } + return failResponse(exception); + } + private static Future failResponse(Throwable exception) { logger.warn("Error occurred while interacting with cache service: {}", exception.getMessage()); logger.debug("Error occurred while interacting with cache service", exception); diff --git a/src/main/java/org/prebid/server/metric/CacheCreativeSizeMetrics.java b/src/main/java/org/prebid/server/metric/CacheCreativeSizeMetrics.java index 5791b370740..f0d56074944 100644 --- a/src/main/java/org/prebid/server/metric/CacheCreativeSizeMetrics.java +++ b/src/main/java/org/prebid/server/metric/CacheCreativeSizeMetrics.java @@ -1,18 +1,23 @@ package org.prebid.server.metric; import com.codahale.metrics.MetricRegistry; +import org.prebid.server.metric.model.CacheCreativeType; import java.util.Objects; import java.util.function.Function; public class CacheCreativeSizeMetrics extends UpdatableMetrics { - CacheCreativeSizeMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) { + CacheCreativeSizeMetrics(MetricRegistry metricRegistry, + CounterType counterType, + String prefix, + CacheCreativeType type) { + super(Objects.requireNonNull(metricRegistry), Objects.requireNonNull(counterType), - nameCreator(Objects.requireNonNull(prefix))); + nameCreator(Objects.requireNonNull(prefix), Objects.requireNonNull(type))); } - private static Function nameCreator(String prefix) { - return metricName -> "%s.creative_size.%s".formatted(prefix, metricName); + private static Function nameCreator(String prefix, CacheCreativeType type) { + return metricName -> "%s.%s_size.%s".formatted(prefix, type.getType(), metricName); } } diff --git a/src/main/java/org/prebid/server/metric/CacheCreativeTtlMetrics.java b/src/main/java/org/prebid/server/metric/CacheCreativeTtlMetrics.java index f5325c382aa..f79ce39f96a 100644 --- a/src/main/java/org/prebid/server/metric/CacheCreativeTtlMetrics.java +++ b/src/main/java/org/prebid/server/metric/CacheCreativeTtlMetrics.java @@ -1,19 +1,24 @@ package org.prebid.server.metric; import com.codahale.metrics.MetricRegistry; +import org.prebid.server.metric.model.CacheCreativeType; import java.util.Objects; import java.util.function.Function; public class CacheCreativeTtlMetrics extends UpdatableMetrics { - CacheCreativeTtlMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) { + CacheCreativeTtlMetrics(MetricRegistry metricRegistry, + CounterType counterType, + String prefix, + CacheCreativeType type) { + super(Objects.requireNonNull(metricRegistry), Objects.requireNonNull(counterType), - nameCreator(Objects.requireNonNull(prefix))); + nameCreator(Objects.requireNonNull(prefix), Objects.requireNonNull(type))); } - private static Function nameCreator(String prefix) { - return metricName -> "%s.creative_ttl.%s".formatted(prefix, metricName); + private static Function nameCreator(String prefix, CacheCreativeType type) { + return metricName -> "%s.%s_ttl.%s".formatted(prefix, type.getType(), metricName); } } diff --git a/src/main/java/org/prebid/server/metric/CacheMetrics.java b/src/main/java/org/prebid/server/metric/CacheMetrics.java index 4838c0848f6..6719bc995d2 100644 --- a/src/main/java/org/prebid/server/metric/CacheMetrics.java +++ b/src/main/java/org/prebid/server/metric/CacheMetrics.java @@ -1,7 +1,10 @@ package org.prebid.server.metric; import com.codahale.metrics.MetricRegistry; +import org.prebid.server.metric.model.CacheCreativeType; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import java.util.function.Function; @@ -14,6 +17,8 @@ class CacheMetrics extends UpdatableMetrics { private final CacheCreativeSizeMetrics cacheCreativeSizeMetrics; private final CacheCreativeTtlMetrics cacheCreativeTtlMetrics; private final CacheVtrackMetrics cacheVtrackMetrics; + private final Map cacheModuleStorageMetrics; + private final Function cacheModuleStorageMetricsCreator; CacheMetrics(MetricRegistry metricRegistry, CounterType counterType) { super( @@ -22,9 +27,14 @@ class CacheMetrics extends UpdatableMetrics { nameCreator(createPrefix())); requestsMetrics = new RequestMetrics(metricRegistry, counterType, createPrefix()); - cacheCreativeSizeMetrics = new CacheCreativeSizeMetrics(metricRegistry, counterType, createPrefix()); - cacheCreativeTtlMetrics = new CacheCreativeTtlMetrics(metricRegistry, counterType, createPrefix()); + cacheCreativeSizeMetrics = new CacheCreativeSizeMetrics( + metricRegistry, counterType, createPrefix(), CacheCreativeType.CREATIVE); + cacheCreativeTtlMetrics = new CacheCreativeTtlMetrics( + metricRegistry, counterType, createPrefix(), CacheCreativeType.CREATIVE); cacheVtrackMetrics = new CacheVtrackMetrics(metricRegistry, counterType, createPrefix()); + cacheModuleStorageMetrics = new HashMap<>(); + cacheModuleStorageMetricsCreator = moduleCode -> + new CacheModuleStorageMetrics(metricRegistry, counterType, createPrefix(), moduleCode); } CacheMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) { @@ -34,9 +44,14 @@ class CacheMetrics extends UpdatableMetrics { nameCreator(createPrefix(Objects.requireNonNull(prefix)))); requestsMetrics = new RequestMetrics(metricRegistry, counterType, createPrefix(prefix)); - cacheCreativeSizeMetrics = new CacheCreativeSizeMetrics(metricRegistry, counterType, createPrefix(prefix)); - cacheCreativeTtlMetrics = new CacheCreativeTtlMetrics(metricRegistry, counterType, createPrefix(prefix)); + cacheCreativeSizeMetrics = new CacheCreativeSizeMetrics( + metricRegistry, counterType, createPrefix(prefix), CacheCreativeType.CREATIVE); + cacheCreativeTtlMetrics = new CacheCreativeTtlMetrics( + metricRegistry, counterType, createPrefix(prefix), CacheCreativeType.CREATIVE); cacheVtrackMetrics = new CacheVtrackMetrics(metricRegistry, counterType, createPrefix(prefix)); + cacheModuleStorageMetrics = new HashMap<>(); + cacheModuleStorageMetricsCreator = moduleCode -> + new CacheModuleStorageMetrics(metricRegistry, counterType, createPrefix(), moduleCode); } private static String createPrefix(String prefix) { @@ -66,4 +81,8 @@ CacheCreativeTtlMetrics creativeTtl() { CacheVtrackMetrics vtrack() { return cacheVtrackMetrics; } + + CacheModuleStorageMetrics moduleStorage(String moduleCode) { + return cacheModuleStorageMetrics.computeIfAbsent(moduleCode, cacheModuleStorageMetricsCreator); + } } diff --git a/src/main/java/org/prebid/server/metric/CacheModuleStorageMetrics.java b/src/main/java/org/prebid/server/metric/CacheModuleStorageMetrics.java new file mode 100644 index 00000000000..5bc52b503cb --- /dev/null +++ b/src/main/java/org/prebid/server/metric/CacheModuleStorageMetrics.java @@ -0,0 +1,54 @@ +package org.prebid.server.metric; + +import com.codahale.metrics.MetricRegistry; +import org.prebid.server.metric.model.CacheCreativeType; + +import java.util.Objects; +import java.util.function.Function; + +class CacheModuleStorageMetrics extends UpdatableMetrics { + + private final CacheReadMetrics readMetrics; + private final CacheWriteMetrics writeMetrics; + private final CacheCreativeSizeMetrics entrySizeMetrics; + private final CacheCreativeTtlMetrics entryTtlMetrics; + + CacheModuleStorageMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix, String module) { + super( + Objects.requireNonNull(metricRegistry), + Objects.requireNonNull(counterType), + nameCreator(createPrefix(Objects.requireNonNull(prefix), Objects.requireNonNull(module)))); + + readMetrics = new CacheReadMetrics(metricRegistry, counterType, createPrefix(prefix, module)); + writeMetrics = new CacheWriteMetrics(metricRegistry, counterType, createPrefix(prefix, module)); + entrySizeMetrics = new CacheCreativeSizeMetrics( + metricRegistry, counterType, createPrefix(prefix, module), CacheCreativeType.ENTRY); + entryTtlMetrics = new CacheCreativeTtlMetrics( + metricRegistry, counterType, createPrefix(prefix, module), CacheCreativeType.ENTRY); + } + + private static Function nameCreator(String prefix) { + return metricName -> "%s.%s".formatted(prefix, metricName); + } + + private static String createPrefix(String prefix, String moduleCode) { + return "%s.module_storage.%s".formatted(prefix, moduleCode); + } + + CacheReadMetrics read() { + return readMetrics; + } + + CacheWriteMetrics write() { + return writeMetrics; + } + + CacheCreativeSizeMetrics entrySize() { + return entrySizeMetrics; + } + + CacheCreativeTtlMetrics entryTtl() { + return entryTtlMetrics; + } + +} diff --git a/src/main/java/org/prebid/server/metric/CacheVtrackMetrics.java b/src/main/java/org/prebid/server/metric/CacheVtrackMetrics.java index ba1644fa136..e3a44639297 100644 --- a/src/main/java/org/prebid/server/metric/CacheVtrackMetrics.java +++ b/src/main/java/org/prebid/server/metric/CacheVtrackMetrics.java @@ -1,6 +1,7 @@ package org.prebid.server.metric; import com.codahale.metrics.MetricRegistry; +import org.prebid.server.metric.model.CacheCreativeType; import java.util.Objects; import java.util.function.Function; @@ -20,8 +21,10 @@ class CacheVtrackMetrics extends UpdatableMetrics { readMetrics = new CacheReadMetrics(metricRegistry, counterType, createPrefix(prefix)); writeMetrics = new CacheWriteMetrics(metricRegistry, counterType, createPrefix(prefix)); - creativeSizeMetrics = new CacheCreativeSizeMetrics(metricRegistry, counterType, createPrefix(prefix)); - creativeTtlMetrics = new CacheCreativeTtlMetrics(metricRegistry, counterType, createPrefix(prefix)); + creativeSizeMetrics = new CacheCreativeSizeMetrics( + metricRegistry, counterType, createPrefix(prefix), CacheCreativeType.CREATIVE); + creativeTtlMetrics = new CacheCreativeTtlMetrics( + metricRegistry, counterType, createPrefix(prefix), CacheCreativeType.CREATIVE); } private static Function nameCreator(String prefix) { diff --git a/src/main/java/org/prebid/server/metric/MetricName.java b/src/main/java/org/prebid/server/metric/MetricName.java index 4d562aa4122..2f0950c5ec3 100644 --- a/src/main/java/org/prebid/server/metric/MetricName.java +++ b/src/main/java/org/prebid/server/metric/MetricName.java @@ -121,6 +121,7 @@ public enum MetricName { // cache creative types json, xml, + text, // account.*.requests. rejected_by_invalid_account("rejected.invalid-account"), diff --git a/src/main/java/org/prebid/server/metric/Metrics.java b/src/main/java/org/prebid/server/metric/Metrics.java index 5c57214ee4c..e7b290bc14a 100644 --- a/src/main/java/org/prebid/server/metric/Metrics.java +++ b/src/main/java/org/prebid/server/metric/Metrics.java @@ -631,6 +631,22 @@ public void updateVtrackCacheCreativeTtl(String accountId, Integer creativeTtl, forAccount(accountId).cache().vtrack().creativeTtl().updateHistogram(creativeType, creativeTtl); } + public void updateModuleStorageCacheReadRequestTime(String moduleCode, long timeElapsed, MetricName metricName) { + cache().moduleStorage(moduleCode).read().updateTimer(metricName, timeElapsed); + } + + public void updateModuleStorageCacheWriteRequestTime(String moduleCode, long timeElapsed, MetricName metricName) { + cache().moduleStorage(moduleCode).write().updateTimer(metricName, timeElapsed); + } + + public void updateModuleStorageCacheEntrySize(String moduleCode, int entrySize, MetricName type) { + cache().moduleStorage(moduleCode).entrySize().updateHistogram(type, entrySize); + } + + public void updateModuleStorageCacheEntryTtl(String moduleCode, Integer entryTtl, MetricName type) { + cache().moduleStorage(moduleCode).entryTtl().updateHistogram(type, entryTtl); + } + public void updateAuctionCacheRequestTime(String accountId, long timeElapsed, MetricName metricName) { cache().requests().updateTimer(metricName, timeElapsed); forAccount(accountId).cache().requests().updateTimer(metricName, timeElapsed); diff --git a/src/main/java/org/prebid/server/metric/model/CacheCreativeType.java b/src/main/java/org/prebid/server/metric/model/CacheCreativeType.java new file mode 100644 index 00000000000..cd7b8ec78e2 --- /dev/null +++ b/src/main/java/org/prebid/server/metric/model/CacheCreativeType.java @@ -0,0 +1,17 @@ +package org.prebid.server.metric.model; + +public enum CacheCreativeType { + + ENTRY("entry"), + CREATIVE("creative"); + + private final String type; + + CacheCreativeType(String type) { + this.type = type; + } + + public String getType() { + return this.type; + } +} diff --git a/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java b/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java index 9021beab946..228d4702435 100644 --- a/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java @@ -248,14 +248,18 @@ PbcStorageService basicModuleCacheService( @Value("${storage.pbc.call-timeout-ms}") int callTimeoutMs, @Value("${pbc.api.key}") String apiKey, HttpClient httpClient, - JacksonMapper mapper) { + JacksonMapper mapper, + Clock clock, + Metrics metrics) { return new BasicPbcStorageService( httpClient, CacheServiceUtil.getCacheEndpointUrl(scheme, host, path), apiKey, callTimeoutMs, - mapper); + mapper, + clock, + metrics); } @Bean diff --git a/src/test/groovy/org/prebid/server/functional/model/ModuleName.groovy b/src/test/groovy/org/prebid/server/functional/model/ModuleName.groovy index 9aee1c69c68..19a29ae0058 100644 --- a/src/test/groovy/org/prebid/server/functional/model/ModuleName.groovy +++ b/src/test/groovy/org/prebid/server/functional/model/ModuleName.groovy @@ -8,6 +8,7 @@ enum ModuleName { PB_RESPONSE_CORRECTION ("pb-response-correction"), ORTB2_BLOCKING("ortb2-blocking"), PB_REQUEST_CORRECTION('pb-request-correction'), + OPTABLE_TARGETING('optable-targeting'), PB_RULE_ENGINE('pb-rule-engine') @JsonValue diff --git a/src/test/groovy/org/prebid/server/functional/model/config/Audience.groovy b/src/test/groovy/org/prebid/server/functional/model/config/Audience.groovy new file mode 100644 index 00000000000..9ea6345ff0e --- /dev/null +++ b/src/test/groovy/org/prebid/server/functional/model/config/Audience.groovy @@ -0,0 +1,10 @@ +package org.prebid.server.functional.model.config + +import groovy.transform.ToString + +@ToString(includeNames = true, ignoreNulls = true) +class Audience { + + String provider + List ids +} diff --git a/src/test/groovy/org/prebid/server/functional/model/config/AudienceId.groovy b/src/test/groovy/org/prebid/server/functional/model/config/AudienceId.groovy new file mode 100644 index 00000000000..e964b9f84b5 --- /dev/null +++ b/src/test/groovy/org/prebid/server/functional/model/config/AudienceId.groovy @@ -0,0 +1,9 @@ +package org.prebid.server.functional.model.config + +import groovy.transform.ToString + +@ToString(includeNames = true, ignoreNulls = true) +class AudienceId { + + String id +} diff --git a/src/test/groovy/org/prebid/server/functional/model/config/CacheProperties.groovy b/src/test/groovy/org/prebid/server/functional/model/config/CacheProperties.groovy new file mode 100644 index 00000000000..e16548b57c5 --- /dev/null +++ b/src/test/groovy/org/prebid/server/functional/model/config/CacheProperties.groovy @@ -0,0 +1,21 @@ +package org.prebid.server.functional.model.config + +import com.fasterxml.jackson.databind.PropertyNamingStrategies +import com.fasterxml.jackson.databind.annotation.JsonNaming +import groovy.transform.ToString +import org.prebid.server.functional.util.PBSUtils + +@ToString(includeNames = true, ignoreNulls = true) +@JsonNaming(PropertyNamingStrategies.LowerCaseStrategy) +class CacheProperties { + + Boolean enabled + Integer ttlSeconds + + static CacheProperties getDefault() { + new CacheProperties().tap { + enabled = true + ttlSeconds = PBSUtils.getRandomNumber(0, 1000) + } + } +} diff --git a/src/test/groovy/org/prebid/server/functional/model/config/ExecutionGroup.groovy b/src/test/groovy/org/prebid/server/functional/model/config/ExecutionGroup.groovy index 09e95d7753d..eb32b75e729 100644 --- a/src/test/groovy/org/prebid/server/functional/model/config/ExecutionGroup.groovy +++ b/src/test/groovy/org/prebid/server/functional/model/config/ExecutionGroup.groovy @@ -18,7 +18,7 @@ class ExecutionGroup { static ExecutionGroup getModuleExecutionGroup(ModuleName name, Stage stage) { new ExecutionGroup().tap { - timeout = 100 + timeout = 1000 hookSequence = [new HookId(moduleCode: name.code, hookImplCode: ModuleHookImplementation.forValue(name, stage).code)] } } diff --git a/src/test/groovy/org/prebid/server/functional/model/config/IdentifierType.groovy b/src/test/groovy/org/prebid/server/functional/model/config/IdentifierType.groovy new file mode 100644 index 00000000000..7148c869c77 --- /dev/null +++ b/src/test/groovy/org/prebid/server/functional/model/config/IdentifierType.groovy @@ -0,0 +1,49 @@ +package org.prebid.server.functional.model.config + +import com.fasterxml.jackson.annotation.JsonValue + +import static org.prebid.server.functional.model.config.OperatingSystem.ANDROID +import static org.prebid.server.functional.model.config.OperatingSystem.FIRE +import static org.prebid.server.functional.model.config.OperatingSystem.IOS +import static org.prebid.server.functional.model.config.OperatingSystem.ROKU +import static org.prebid.server.functional.model.config.OperatingSystem.TIZEN + +enum IdentifierType { + + EMAIL_ADDRESS("e"), + PHONE_NUMBER("p"), + POSTAL_CODE("z"), + APPLE_IDFA("a"), + GOOGLE_GAID("g"), + ROKU_RIDA("r"), + SAMSUNG_TIFA("s"), + AMAZON_AFAI("f"), + NET_ID("n"), + ID5("id5"), + UTIQ("utiq"), + OPTABLE_VID("v") + + @JsonValue + final String value + + IdentifierType(String value) { + this.value = value + } + + static IdentifierType fromOS(OperatingSystem os) { + switch (os) { + case IOS: + return APPLE_IDFA + case ANDROID: + return GOOGLE_GAID + case ROKU: + return ROKU_RIDA + case TIZEN: + return SAMSUNG_TIFA + case FIRE: + return AMAZON_AFAI + default: + throw new IllegalArgumentException("Unsupported OS: " + os); + } + } +} diff --git a/src/test/groovy/org/prebid/server/functional/model/config/ModuleHookImplementation.groovy b/src/test/groovy/org/prebid/server/functional/model/config/ModuleHookImplementation.groovy index 4d9424e1c39..af7bf670c95 100644 --- a/src/test/groovy/org/prebid/server/functional/model/config/ModuleHookImplementation.groovy +++ b/src/test/groovy/org/prebid/server/functional/model/config/ModuleHookImplementation.groovy @@ -11,6 +11,7 @@ enum ModuleHookImplementation { ORTB2_BLOCKING_BIDDER_REQUEST("ortb2-blocking-bidder-request"), ORTB2_BLOCKING_RAW_BIDDER_RESPONSE("ortb2-blocking-raw-bidder-response"), PB_REQUEST_CORRECTION_PROCESSED_AUCTION_REQUEST("pb-request-correction-processed-auction-request"), + OPTABLE_TARGETING_PROCESSED_AUCTION_REQUEST("optable-targeting-processed-auction-request-hook"), PB_RULES_ENGINE_PROCESSED_AUCTION_REQUEST("pb-rule-engine-processed-auction-request") @JsonValue diff --git a/src/test/groovy/org/prebid/server/functional/model/config/OperatingSystem.groovy b/src/test/groovy/org/prebid/server/functional/model/config/OperatingSystem.groovy new file mode 100644 index 00000000000..f10e28a4cb6 --- /dev/null +++ b/src/test/groovy/org/prebid/server/functional/model/config/OperatingSystem.groovy @@ -0,0 +1,19 @@ +package org.prebid.server.functional.model.config + +import com.fasterxml.jackson.annotation.JsonValue + +enum OperatingSystem { + + IOS("ios"), + ANDROID("android"), + ROKU("roku"), + TIZEN("tizen"), + FIRE("fire") + + @JsonValue + final String value + + OperatingSystem(String value) { + this.value = value; + } +} diff --git a/src/test/groovy/org/prebid/server/functional/model/config/OptableTargetingConfig.groovy b/src/test/groovy/org/prebid/server/functional/model/config/OptableTargetingConfig.groovy new file mode 100644 index 00000000000..56e6f3415b8 --- /dev/null +++ b/src/test/groovy/org/prebid/server/functional/model/config/OptableTargetingConfig.groovy @@ -0,0 +1,33 @@ +package org.prebid.server.functional.model.config + +import com.fasterxml.jackson.databind.PropertyNamingStrategies +import com.fasterxml.jackson.databind.annotation.JsonNaming +import groovy.transform.ToString +import org.prebid.server.functional.util.PBSUtils + +@ToString(includeNames = true, ignoreNulls = true) +@JsonNaming(PropertyNamingStrategies.KebabCaseStrategy) +class OptableTargetingConfig { + + String apiEndpoint + String apiKey + String tenant + String origin + Map ppidMapping + Boolean adserverTargeting + Long timeout + String idPrefixOrder + CacheProperties cache + + static OptableTargetingConfig getDefault(Map ppidMapping) { + new OptableTargetingConfig().tap { + it.apiKey = PBSUtils.randomString + it.tenant = PBSUtils.randomString + it.origin = PBSUtils.randomString + it.apiEndpoint = PBSUtils.randomString + it.adserverTargeting = true + it.ppidMapping = ppidMapping + it.cache = CacheProperties.default + } + } +} diff --git a/src/test/groovy/org/prebid/server/functional/model/config/PbsModulesConfig.groovy b/src/test/groovy/org/prebid/server/functional/model/config/PbsModulesConfig.groovy index 801178fd4d4..ac2685742b6 100644 --- a/src/test/groovy/org/prebid/server/functional/model/config/PbsModulesConfig.groovy +++ b/src/test/groovy/org/prebid/server/functional/model/config/PbsModulesConfig.groovy @@ -13,5 +13,6 @@ class PbsModulesConfig { Ortb2BlockingConfig ortb2Blocking PbResponseCorrection pbResponseCorrection PbRequestCorrectionConfig pbRequestCorrection + OptableTargetingConfig optableTargeting PbRulesEngine pbRuleEngine } diff --git a/src/test/groovy/org/prebid/server/functional/model/config/TargetingOrtb.groovy b/src/test/groovy/org/prebid/server/functional/model/config/TargetingOrtb.groovy new file mode 100644 index 00000000000..0e6b8d93119 --- /dev/null +++ b/src/test/groovy/org/prebid/server/functional/model/config/TargetingOrtb.groovy @@ -0,0 +1,10 @@ +package org.prebid.server.functional.model.config + +import groovy.transform.ToString +import org.prebid.server.functional.model.request.auction.User + +@ToString(includeNames = true, ignoreNulls = true) +class TargetingOrtb { + + User user +} diff --git a/src/test/groovy/org/prebid/server/functional/model/config/TargetingResult.groovy b/src/test/groovy/org/prebid/server/functional/model/config/TargetingResult.groovy new file mode 100644 index 00000000000..d9f2bc7b30d --- /dev/null +++ b/src/test/groovy/org/prebid/server/functional/model/config/TargetingResult.groovy @@ -0,0 +1,10 @@ +package org.prebid.server.functional.model.config + +import groovy.transform.ToString + +@ToString(includeNames = true, ignoreNulls = true) +class TargetingResult { + + List audience + TargetingOrtb ortb2 +} diff --git a/src/test/groovy/org/prebid/server/functional/testcontainers/scaffolding/StoredCache.groovy b/src/test/groovy/org/prebid/server/functional/testcontainers/scaffolding/StoredCache.groovy new file mode 100644 index 00000000000..c8ad7caa924 --- /dev/null +++ b/src/test/groovy/org/prebid/server/functional/testcontainers/scaffolding/StoredCache.groovy @@ -0,0 +1,119 @@ +package org.prebid.server.functional.testcontainers.scaffolding + +import org.mockserver.matchers.TimeToLive +import org.mockserver.matchers.Times +import org.mockserver.model.HttpRequest +import org.mockserver.model.HttpStatusCode +import org.prebid.server.functional.model.config.Audience +import org.prebid.server.functional.model.config.AudienceId +import org.prebid.server.functional.model.config.IdentifierType +import org.prebid.server.functional.model.config.OptableTargetingConfig +import org.prebid.server.functional.model.config.TargetingOrtb +import org.prebid.server.functional.model.config.TargetingResult +import org.prebid.server.functional.model.request.auction.BidRequest +import org.prebid.server.functional.model.request.auction.User +import org.prebid.server.functional.util.PBSUtils +import org.testcontainers.containers.MockServerContainer + +import java.nio.charset.StandardCharsets + +import static org.mockserver.model.HttpRequest.request +import static org.mockserver.model.HttpResponse.response +import static org.mockserver.model.HttpStatusCode.NO_CONTENT_204 +import static org.mockserver.model.HttpStatusCode.OK_200 + +class StoredCache extends NetworkScaffolding { + + private static final String CACHE_ENDPOINT = "/stored-cache" + + StoredCache(MockServerContainer mockServerContainer) { + super(mockServerContainer, CACHE_ENDPOINT) + } + + @Override + protected HttpRequest getRequest(String impId) {} + + @Override + HttpRequest getRequest() { + request().withMethod("GET") + .withPath(endpoint) + } + + @Override + void setResponse() {} + + TargetingResult setTargetingResponse(BidRequest bidRequest, OptableTargetingConfig config) { + def targetingResult = getBodyByRequest(bidRequest) + mockServerClient.when(request() + .withMethod("GET") + .withPath("$endpoint${QueryBuilder.buildQuery(bidRequest, config)}"), Times.unlimited(), TimeToLive.unlimited(), -10) + .respond { response().withStatusCode(OK_200.code()).withBody(encode(targetingResult)) } + targetingResult + } + + TargetingResult setCachedTargetingResponse(BidRequest bidRequest) { + def targetingResult = getBodyByRequest(bidRequest) + mockServerClient.when(request() + .withMethod("GET") + .withPath(endpoint), Times.unlimited(), TimeToLive.unlimited(), -10) + .respond { response().withStatusCode(OK_200.code()).withBody(encode(targetingResult)) } + targetingResult + } + + void setCachingResponse(HttpStatusCode statusCode = NO_CONTENT_204) { + mockServerClient.when(request() + .withMethod("POST") + .withPath(endpoint), Times.unlimited(), TimeToLive.unlimited(), -10) + .respond { response().withStatusCode(statusCode.code()) } + } + + private static TargetingResult getBodyByRequest(BidRequest bidRequest) { + new TargetingResult().tap { + it.audience = [new Audience(ids: [new AudienceId(id: PBSUtils.randomString)], provider: PBSUtils.randomString)] + it.ortb2 = new TargetingOrtb(user: new User(data: bidRequest.user.data, eids: bidRequest.user.eids)) + } + } + + private class QueryBuilder { + + static String buildQuery(BidRequest bidRequest, OptableTargetingConfig config) { + buildIdsString(config) + buildAttributesString(bidRequest, config) + } + + private static String buildIdsString(OptableTargetingConfig config) { + def ppids = config.ppidMapping + if (!ppids) { + return '' + } + + def reorderedIds = reorderIds(ppids.keySet(), config.idPrefixOrder) + + reorderedIds.collect { id -> + def value = ppids[id] + "&id=${URLEncoder.encode("${id.value}:${value}", StandardCharsets.UTF_8)}" + }.join('') + } + + private static Set reorderIds(Set ids, String idPrefixOrder) { + if (!idPrefixOrder) { + return ids + } + def prefixOrder = idPrefixOrder.split(',') as List + def prefixToPriority = prefixOrder.collectEntries { v, i -> [(v): i] } + ids.sort { prefixToPriority.get(it.value, Integer.MAX_VALUE) } + } + + private static String buildAttributesString(BidRequest bidRequest, OptableTargetingConfig config) { + def regs = bidRequest.regs + def gdpr = regs?.gdpr + def gdprConsent = bidRequest.user?.consent + + [gdprConsent != null ? "&gdpr_consent=${gdprConsent}" : null, + "&gdpr=${gdpr ? 1 : 0}", + regs?.gpp ? "&gpp=${regs.gpp}" : null, + regs?.gppSid ? "&gpp_sid=${regs.gppSid.first()}" : null, + config?.timeout ? "&timeout=${config.timeout}ms" : null, + "&osdk=prebid-server"].findAll().join('') + } + } +} diff --git a/src/test/groovy/org/prebid/server/functional/tests/module/ModuleBaseSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/module/ModuleBaseSpec.groovy index f721b93eac0..c0933a238e7 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/module/ModuleBaseSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/module/ModuleBaseSpec.groovy @@ -7,7 +7,9 @@ import org.prebid.server.functional.model.response.auction.AnalyticResult import org.prebid.server.functional.model.response.auction.BidResponse import org.prebid.server.functional.model.response.auction.InvocationResult import org.prebid.server.functional.tests.BaseSpec +import org.prebid.server.functional.util.PBSUtils +import static org.prebid.server.functional.model.ModuleName.OPTABLE_TARGETING import static org.prebid.server.functional.model.ModuleName.ORTB2_BLOCKING import static org.prebid.server.functional.model.ModuleName.PB_RESPONSE_CORRECTION import static org.prebid.server.functional.model.ModuleName.PB_RICHMEDIA_FILTER @@ -16,6 +18,7 @@ import static org.prebid.server.functional.model.ModuleName.PB_RULE_ENGINE import static org.prebid.server.functional.model.config.Endpoint.OPENRTB2_AUCTION import static org.prebid.server.functional.model.config.Stage.ALL_PROCESSED_BID_RESPONSES import static org.prebid.server.functional.model.config.Stage.PROCESSED_AUCTION_REQUEST +import static org.prebid.server.functional.testcontainers.Dependencies.getNetworkServiceContainer class ModuleBaseSpec extends BaseSpec { @@ -56,6 +59,15 @@ class ModuleBaseSpec extends BaseSpec { .collectEntries { key, value -> [(key.toString()): value.toString()] } } + protected static Map getOptableTargetingSettings(boolean isEnabled = true, Endpoint endpoint = OPENRTB2_AUCTION) { + ["hooks.${OPTABLE_TARGETING.code}.enabled": isEnabled as String, + "hooks.modules.${OPTABLE_TARGETING.code}.api-endpoint" : "$networkServiceContainer.rootUri/stored-cache".toString(), + "hooks.modules.${OPTABLE_TARGETING.code}.tenant" : PBSUtils.randomString, + "hooks.modules.${OPTABLE_TARGETING.code}.origin" : PBSUtils.randomString, + "hooks.host-execution-plan" : encode(ExecutionPlan.getSingleEndpointExecutionPlan(endpoint, [(PROCESSED_AUCTION_REQUEST): [OPTABLE_TARGETING]]))] + .collectEntries { key, value -> [(key.toString()): value.toString()] } + } + protected static Map getOrtb2BlockingSettings(boolean isEnabled = true) { ["hooks.${ORTB2_BLOCKING.code}.enabled": isEnabled as String] } diff --git a/src/test/groovy/org/prebid/server/functional/tests/module/optabletargeting/CacheStorageSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/module/optabletargeting/CacheStorageSpec.groovy new file mode 100644 index 00000000000..9a71d42ba7f --- /dev/null +++ b/src/test/groovy/org/prebid/server/functional/tests/module/optabletargeting/CacheStorageSpec.groovy @@ -0,0 +1,207 @@ +package org.prebid.server.functional.tests.module.optabletargeting + +import org.prebid.server.functional.model.config.AccountConfig +import org.prebid.server.functional.model.config.AccountHooksConfiguration +import org.prebid.server.functional.model.config.IdentifierType +import org.prebid.server.functional.model.config.OperatingSystem +import org.prebid.server.functional.model.config.OptableTargetingConfig +import org.prebid.server.functional.model.config.PbsModulesConfig +import org.prebid.server.functional.model.db.Account +import org.prebid.server.functional.model.request.auction.BidRequest +import org.prebid.server.functional.model.request.auction.Data +import org.prebid.server.functional.model.request.auction.Device +import org.prebid.server.functional.model.request.auction.Eid +import org.prebid.server.functional.model.request.auction.Geo +import org.prebid.server.functional.model.request.auction.PublicCountryIp +import org.prebid.server.functional.model.request.auction.User +import org.prebid.server.functional.service.PrebidServerService +import org.prebid.server.functional.testcontainers.scaffolding.StoredCache +import org.prebid.server.functional.tests.module.ModuleBaseSpec +import org.prebid.server.functional.util.PBSUtils + +import static org.apache.commons.codec.binary.Base64.encodeBase64 +import static org.mockserver.model.HttpStatusCode.NOT_FOUND_404 +import static org.prebid.server.functional.model.ModuleName.OPTABLE_TARGETING +import static org.prebid.server.functional.testcontainers.Dependencies.getNetworkServiceContainer + +class CacheStorageSpec extends ModuleBaseSpec { + + private static final String METRIC_CREATIVE_SIZE_TEXT = "prebid_cache.module_storage.${OPTABLE_TARGETING.code}.entry_size.text" + private static final String METRIC_CREATIVE_TTL_TEXT = "prebid_cache.module_storage.${OPTABLE_TARGETING.code}.entry_ttl.text" + + private static final String METRIC_CREATIVE_READ_OK = "prebid_cache.module_storage.${OPTABLE_TARGETING.code}.read.ok" + private static final String METRIC_CREATIVE_READ_ERR = "prebid_cache.module_storage.${OPTABLE_TARGETING.code}.read.err" + private static final String METRIC_CREATIVE_WRITE_OK = "prebid_cache.module_storage.${OPTABLE_TARGETING.code}.write.ok" + private static final String METRIC_CREATIVE_WRITE_ERR = "prebid_cache.module_storage.${OPTABLE_TARGETING.code}.write.err" + + private static final StoredCache storedCache = new StoredCache(networkServiceContainer) + + private static final Map CACHE_STORAGE_CONFIG = ['storage.pbc.path' : "$networkServiceContainer.rootUri/stored-cache".toString(), + 'storage.pbc.call-timeout-ms': '1000', + 'storage.pbc.enabled' : 'true', + 'cache.module.enabled' : 'true', + 'pbc.api.key' : PBSUtils.randomString, + 'cache.api-key-secured' : 'false'] + private static final Map MODULE_STORAGE_CACHE_CONFIG = getOptableTargetingSettings() + CACHE_STORAGE_CONFIG + private static final PrebidServerService prebidServerStoredCacheService = pbsServiceFactory.getService(MODULE_STORAGE_CACHE_CONFIG) + + def setup() { + storedCache.reset() + } + + def cleanupSpec() { + pbsServiceFactory.removeContainer(MODULE_STORAGE_CACHE_CONFIG) + } + + def "PBS should update error metrics when no cached requests present"() { + given: "Default BidRequest with cache and device info" + def randomIfa = PBSUtils.randomString + def system = PBSUtils.getRandomEnum(OperatingSystem) + def bidRequest = getBidRequestForModuleCacheStorage(randomIfa, system) + + and: "Account with optable targeting module" + def targetingConfig = OptableTargetingConfig.getDefault([(IdentifierType.fromOS(system)): randomIfa]) + def account = createAccountWithRequestCorrectionConfig(bidRequest, targetingConfig) + accountDao.save(account) + + and: "Flash metrics" + flushMetrics(prebidServerStoredCacheService) + + when: "PBS processes auction request" + prebidServerStoredCacheService.sendAuctionRequest(bidRequest) + + then: "PBS should update metrics for new saved text storage cache" + def metrics = prebidServerStoredCacheService.sendCollectedMetricsRequest() + assert metrics[METRIC_CREATIVE_READ_ERR] == 1 + + and: "No updates for success metrics" + assert !metrics[METRIC_CREATIVE_SIZE_TEXT] + assert !metrics[METRIC_CREATIVE_TTL_TEXT] + assert !metrics[METRIC_CREATIVE_READ_OK] + } + + def "PBS should update error metrics when external service responded with invalid values"() { + given: "Default BidRequest with cache and device info" + def randomIfa = PBSUtils.randomString + def system = PBSUtils.getRandomEnum(OperatingSystem) + def bidRequest = getBidRequestForModuleCacheStorage(randomIfa, system) + + and: "Account with optable targeting module" + def targetingConfig = OptableTargetingConfig.getDefault([(IdentifierType.fromOS(system)): randomIfa]) + def account = createAccountWithRequestCorrectionConfig(bidRequest, targetingConfig) + accountDao.save(account) + + and: "Mocked external request" + storedCache.setTargetingResponse(bidRequest, targetingConfig) + storedCache.setCachingResponse(NOT_FOUND_404) + + and: "Flash metrics" + flushMetrics(prebidServerStoredCacheService) + + when: "PBS processes auction request" + prebidServerStoredCacheService.sendAuctionRequest(bidRequest) + + then: "PBS should update error metrics" + def metrics = prebidServerStoredCacheService.sendCollectedMetricsRequest() + assert metrics[METRIC_CREATIVE_WRITE_ERR] == 1 + + and: "No updates for success metrics" + assert !metrics[METRIC_CREATIVE_WRITE_OK] + } + + def "PBS should update metrics for new saved text storage cache when no cached requests"() { + given: "Current value of metric prebid cache" + def okInitialValue = getCurrentMetricValue(prebidServerStoredCacheService, METRIC_CREATIVE_WRITE_OK) + + and: "Default BidRequest with cache and device info" + def randomIfa = PBSUtils.randomString + def system = PBSUtils.getRandomEnum(OperatingSystem) + def bidRequest = getBidRequestForModuleCacheStorage(randomIfa, system) + + and: "Account with optable targeting module" + def targetingConfig = OptableTargetingConfig.getDefault([(IdentifierType.fromOS(system)): randomIfa]) + def account = createAccountWithRequestCorrectionConfig(bidRequest, targetingConfig) + accountDao.save(account) + + and: "Mocked external request" + def targetingResult = storedCache.setTargetingResponse(bidRequest, targetingConfig) + storedCache.setCachingResponse() + + and: "Flash metrics" + flushMetrics(prebidServerStoredCacheService) + + when: "PBS processes auction request" + prebidServerStoredCacheService.sendAuctionRequest(bidRequest) + + then: "PBS should update metrics for new saved text storage cache" + def metrics = prebidServerStoredCacheService.sendCollectedMetricsRequest() + assert metrics[METRIC_CREATIVE_SIZE_TEXT] == new String(encodeBase64(encode(targetingResult).bytes)).size() + assert metrics[METRIC_CREATIVE_WRITE_OK] == okInitialValue + 1 + + and: "PBS should include histogram metric" + assert metrics[METRIC_CREATIVE_TTL_TEXT] + } + + def "PBS should update metrics for stored cached requests cache when proper record present"() { + given: "Current value of metric prebid cache" + def textInitialValue = getCurrentMetricValue(prebidServerStoredCacheService, METRIC_CREATIVE_SIZE_TEXT) + def ttlInitialValue = getCurrentMetricValue(prebidServerStoredCacheService, METRIC_CREATIVE_TTL_TEXT) + def writeInitialValue = getCurrentMetricValue(prebidServerStoredCacheService, METRIC_CREATIVE_WRITE_OK) + def readErrorInitialValue = getCurrentMetricValue(prebidServerStoredCacheService, METRIC_CREATIVE_READ_ERR) + def writeErrorInitialValue = getCurrentMetricValue(prebidServerStoredCacheService, METRIC_CREATIVE_WRITE_ERR) + + and: "Default BidRequest with cache and device info" + def randomIfa = PBSUtils.randomString + def system = PBSUtils.getRandomEnum(OperatingSystem) + def bidRequest = getBidRequestForModuleCacheStorage(randomIfa, system) + + and: "Account with optable targeting module" + def targetingConfig = OptableTargetingConfig.getDefault([(IdentifierType.fromOS(system)): randomIfa]) + def account = createAccountWithRequestCorrectionConfig(bidRequest, targetingConfig) + accountDao.save(account) + + and: "Mocked external request" + storedCache.setCachedTargetingResponse(bidRequest) + storedCache.setCachingResponse() + + and: "Flash metrics" + flushMetrics(prebidServerStoredCacheService) + + when: "PBS processes auction request" + prebidServerStoredCacheService.sendAuctionRequest(bidRequest) + + then: "PBS should update metrics for stored cached requests" + def metrics = prebidServerStoredCacheService.sendCollectedMetricsRequest() + assert metrics[METRIC_CREATIVE_READ_OK] == 1 + + and: "No updates for new saved text storage metrics" + assert metrics[METRIC_CREATIVE_SIZE_TEXT] == textInitialValue + assert metrics[METRIC_CREATIVE_TTL_TEXT] == ttlInitialValue + assert metrics[METRIC_CREATIVE_WRITE_OK] == writeInitialValue + + and: "No update for error metrics" + assert metrics[METRIC_CREATIVE_READ_ERR] == readErrorInitialValue + assert metrics[METRIC_CREATIVE_WRITE_ERR] == writeErrorInitialValue + } + + private static BidRequest getBidRequestForModuleCacheStorage(String ifa, OperatingSystem os) { + BidRequest.defaultBidRequest.tap { + it.enableCache() + it.user = new User(id: PBSUtils.randomString, data: [Data.defaultData], eids: [Eid.defaultEid]) + it.device = new Device(geo: Geo.FPDGeo, + ip: PBSUtils.getRandomEnum(PublicCountryIp.class).v4, + ifa: ifa, + ua: PBSUtils.randomString, + os: os) + } + } + + private static Account createAccountWithRequestCorrectionConfig(BidRequest bidRequest, + OptableTargetingConfig optableTargetingConfig) { + + def pbsModulesConfig = new PbsModulesConfig(optableTargeting: optableTargetingConfig) + def accountHooksConfig = new AccountHooksConfiguration(modules: pbsModulesConfig) + def accountConfig = new AccountConfig(hooks: accountHooksConfig) + new Account(uuid: bidRequest.accountId, config: accountConfig) + } +} diff --git a/src/test/java/org/prebid/server/cache/BasicPbcStorageServiceTest.java b/src/test/java/org/prebid/server/cache/BasicPbcStorageServiceTest.java index c7b09518ae7..a995429ba79 100644 --- a/src/test/java/org/prebid/server/cache/BasicPbcStorageServiceTest.java +++ b/src/test/java/org/prebid/server/cache/BasicPbcStorageServiceTest.java @@ -15,20 +15,27 @@ import org.prebid.server.cache.proto.request.module.StorageDataType; import org.prebid.server.cache.proto.response.module.ModuleCacheResponse; import org.prebid.server.exception.PreBidException; +import org.prebid.server.metric.MetricName; +import org.prebid.server.metric.Metrics; import org.prebid.server.util.HttpUtil; import org.prebid.server.vertx.httpclient.HttpClient; import org.prebid.server.vertx.httpclient.model.HttpClientResponse; import java.net.MalformedURLException; import java.net.URL; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneId; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; import static org.mockito.Mock.Strictness.LENIENT; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; @ExtendWith(MockitoExtension.class) public class BasicPbcStorageServiceTest extends VertxTest { @@ -36,6 +43,9 @@ public class BasicPbcStorageServiceTest extends VertxTest { @Mock(strictness = LENIENT) private HttpClient httpClient; + @Mock(strictness = LENIENT) + private Metrics metrics; + private BasicPbcStorageService target; @BeforeEach @@ -45,15 +55,40 @@ public void setUp() throws MalformedURLException, JsonProcessingException { new URL("http://cache-service/cache"), "pbc-api-key", 10, - jacksonMapper); + jacksonMapper, + Clock.fixed(Instant.now(), ZoneId.systemDefault()), + metrics); given(httpClient.post(anyString(), any(), any(), anyLong())).willReturn(Future.succeededFuture( - HttpClientResponse.of(200, null, "someBody"))); + HttpClientResponse.of(204, null, null))); given(httpClient.get(anyString(), any(), anyLong())).willReturn(Future.succeededFuture( HttpClientResponse.of(200, null, mapper.writeValueAsString( ModuleCacheResponse.of("some-key", StorageDataType.JSON, "some-value"))))); } + @Test + public void storeModuleEntryShouldReturnFailureResponse() { + //given + given(httpClient.post(anyString(), any(), any(), anyLong())).willReturn(Future.succeededFuture( + HttpClientResponse.of(500, null, null))); + + // when + target.storeEntry("some-key", + "some-value", + StorageDataType.TEXT, + 12, + "some-application", + "some-module-code"); + + // then + final ModuleCacheRequest result = captureModuleCacheRequest(); + assertThat(result.getKey()).isEqualTo("module.some-module-code.some-key"); + + verify(metrics).updateModuleStorageCacheEntryTtl("some-module-code", 12, MetricName.text); + verify(metrics).updateModuleStorageCacheEntrySize("some-module-code", 16, MetricName.text); + verify(metrics).updateModuleStorageCacheWriteRequestTime(eq("some-module-code"), anyLong(), eq(MetricName.err)); + } + @Test public void storeModuleEntryShouldStoreExpectedKey() { // when @@ -67,6 +102,10 @@ public void storeModuleEntryShouldStoreExpectedKey() { // then final ModuleCacheRequest result = captureModuleCacheRequest(); assertThat(result.getKey()).isEqualTo("module.some-module-code.some-key"); + + verify(metrics).updateModuleStorageCacheEntryTtl("some-module-code", 12, MetricName.text); + verify(metrics).updateModuleStorageCacheEntrySize("some-module-code", 16, MetricName.text); + verify(metrics).updateModuleStorageCacheWriteRequestTime(eq("some-module-code"), anyLong(), eq(MetricName.ok)); } @Test @@ -82,6 +121,10 @@ public void storeModuleEntryShouldStoreExpectedValue() { // then final ModuleCacheRequest result = captureModuleCacheRequest(); assertThat(result.getValue()).isEqualTo("c29tZS12YWx1ZQ=="); + + verify(metrics).updateModuleStorageCacheEntryTtl("some-module-code", 12, MetricName.text); + verify(metrics).updateModuleStorageCacheEntrySize("some-module-code", 16, MetricName.text); + verify(metrics).updateModuleStorageCacheWriteRequestTime(eq("some-module-code"), anyLong(), eq(MetricName.ok)); } @Test @@ -97,10 +140,14 @@ public void storeModuleEntryShouldStoreExpectedApplication() { // then final ModuleCacheRequest result = captureModuleCacheRequest(); assertThat(result.getApplication()).isEqualTo("some-application"); + + verify(metrics).updateModuleStorageCacheEntryTtl("some-module-code", 12, MetricName.text); + verify(metrics).updateModuleStorageCacheEntrySize("some-module-code", 16, MetricName.text); + verify(metrics).updateModuleStorageCacheWriteRequestTime(eq("some-module-code"), anyLong(), eq(MetricName.ok)); } @Test - public void storeModuleEntryShouldStoreExpectedMediaType() { + public void storeModuleEntryShouldStoreTextMediaType() { // when target.storeEntry("some-key", "some-value", @@ -112,6 +159,48 @@ public void storeModuleEntryShouldStoreExpectedMediaType() { // then final ModuleCacheRequest result = captureModuleCacheRequest(); assertThat(result.getType()).isEqualTo(StorageDataType.TEXT); + + verify(metrics).updateModuleStorageCacheEntryTtl("some-module-code", 12, MetricName.text); + verify(metrics).updateModuleStorageCacheEntrySize("some-module-code", 16, MetricName.text); + verify(metrics).updateModuleStorageCacheWriteRequestTime(eq("some-module-code"), anyLong(), eq(MetricName.ok)); + } + + @Test + public void storeModuleEntryShouldStoreXmlMediaType() { + // when + target.storeEntry("some-key", + "", + StorageDataType.XML, + 12, + "some-application", + "some-module-code"); + + // then + final ModuleCacheRequest result = captureModuleCacheRequest(); + assertThat(result.getType()).isEqualTo(StorageDataType.XML); + + verify(metrics).updateModuleStorageCacheEntryTtl("some-module-code", 12, MetricName.xml); + verify(metrics).updateModuleStorageCacheEntrySize("some-module-code", 13, MetricName.xml); + verify(metrics).updateModuleStorageCacheWriteRequestTime(eq("some-module-code"), anyLong(), eq(MetricName.ok)); + } + + @Test + public void storeModuleEntryShouldStoreJsonMediaType() { + // when + target.storeEntry("some-key", + "{}", + StorageDataType.JSON, + 12, + "some-application", + "some-module-code"); + + // then + final ModuleCacheRequest result = captureModuleCacheRequest(); + assertThat(result.getType()).isEqualTo(StorageDataType.JSON); + + verify(metrics).updateModuleStorageCacheEntryTtl("some-module-code", 12, MetricName.json); + verify(metrics).updateModuleStorageCacheEntrySize("some-module-code", 2, MetricName.json); + verify(metrics).updateModuleStorageCacheWriteRequestTime(eq("some-module-code"), anyLong(), eq(MetricName.ok)); } @Test @@ -127,6 +216,10 @@ public void storeModuleEntryShouldStoreExpectedTtl() { // then final ModuleCacheRequest result = captureModuleCacheRequest(); assertThat(result.getTtlseconds()).isEqualTo(12); + + verify(metrics).updateModuleStorageCacheEntryTtl("some-module-code", 12, MetricName.text); + verify(metrics).updateModuleStorageCacheEntrySize("some-module-code", 16, MetricName.text); + verify(metrics).updateModuleStorageCacheWriteRequestTime(eq("some-module-code"), anyLong(), eq(MetricName.ok)); } @Test @@ -143,6 +236,8 @@ public void storeEntryShouldReturnFailedFutureIfKeyIsMissed() { assertThat(result.failed()).isTrue(); assertThat(result.cause()).isInstanceOf(PreBidException.class); assertThat(result.cause().getMessage()).isEqualTo("Module cache 'key' can not be blank"); + + verifyNoInteractions(metrics); } @Test @@ -159,6 +254,8 @@ public void storeEntryShouldReturnFailedFutureIfValueIsMissed() { assertThat(result.failed()).isTrue(); assertThat(result.cause()).isInstanceOf(PreBidException.class); assertThat(result.cause().getMessage()).isEqualTo("Module cache 'value' can not be blank"); + + verifyNoInteractions(metrics); } @Test @@ -175,6 +272,8 @@ public void storeEntryShouldReturnFailedFutureIfApplicationIsMissed() { assertThat(result.failed()).isTrue(); assertThat(result.cause()).isInstanceOf(PreBidException.class); assertThat(result.cause().getMessage()).isEqualTo("Module cache 'application' can not be blank"); + + verifyNoInteractions(metrics); } @Test @@ -191,6 +290,8 @@ public void storeEntryShouldReturnFailedFutureIfTypeIsMissed() { assertThat(result.failed()).isTrue(); assertThat(result.cause()).isInstanceOf(PreBidException.class); assertThat(result.cause().getMessage()).isEqualTo("Module cache 'type' can not be empty"); + + verifyNoInteractions(metrics); } @Test @@ -207,6 +308,8 @@ public void storeModuleEntryShouldReturnFailedFutureIfCodeIsMissed() { assertThat(result.failed()).isTrue(); assertThat(result.cause()).isInstanceOf(PreBidException.class); assertThat(result.cause().getMessage()).isEqualTo("Module cache 'moduleCode' can not be blank"); + + verifyNoInteractions(metrics); } @Test @@ -222,6 +325,10 @@ public void storeEntryShouldCreateCallWithApiKeyInHeader() { // then final MultiMap result = captureStoreRequestHeaders(); assertThat(result.get(HttpUtil.X_PBC_API_KEY_HEADER)).isEqualTo("pbc-api-key"); + + verify(metrics).updateModuleStorageCacheEntryTtl("some-module-code", 12, MetricName.text); + verify(metrics).updateModuleStorageCacheEntrySize("some-module-code", 16, MetricName.text); + verify(metrics).updateModuleStorageCacheWriteRequestTime(eq("some-module-code"), anyLong(), eq(MetricName.ok)); } @Test @@ -234,6 +341,8 @@ public void retrieveModuleEntryShouldReturnFailedFutureIfKeyIsMissed() { assertThat(result.failed()).isTrue(); assertThat(result.cause()).isInstanceOf(PreBidException.class); assertThat(result.cause().getMessage()).isEqualTo("Module cache 'key' can not be blank"); + + verifyNoInteractions(metrics); } @Test @@ -246,6 +355,8 @@ public void retrieveModuleEntryShouldReturnFailedFutureIfApplicationIsMissed() { assertThat(result.failed()).isTrue(); assertThat(result.cause()).isInstanceOf(PreBidException.class); assertThat(result.cause().getMessage()).isEqualTo("Module cache 'application' can not be blank"); + + verifyNoInteractions(metrics); } @Test @@ -258,6 +369,8 @@ public void retrieveModuleEntryShouldReturnFailedFutureIfCodeIsMissed() { assertThat(result.failed()).isTrue(); assertThat(result.cause()).isInstanceOf(PreBidException.class); assertThat(result.cause().getMessage()).isEqualTo("Module cache 'moduleCode' can not be blank"); + + verifyNoInteractions(metrics); } @Test @@ -268,6 +381,8 @@ public void retrieveEntryShouldCreateCallWithApiKeyInHeader() { // then final MultiMap result = captureRetrieveRequestHeaders(); assertThat(result.get(HttpUtil.X_PBC_API_KEY_HEADER)).isEqualTo("pbc-api-key"); + + verify(metrics).updateModuleStorageCacheReadRequestTime(eq("some-module-code"), anyLong(), eq(MetricName.ok)); } @Test @@ -279,6 +394,8 @@ public void retrieveEntryShouldCreateCallWithKeyInParams() { final String result = captureRetrieveUrl(); assertThat(result) .isEqualTo("http://cache-service/cache?k=module.some-module-code.some-key&a=some-app"); + + verify(metrics).updateModuleStorageCacheReadRequestTime(eq("some-module-code"), anyLong(), eq(MetricName.ok)); } @Test @@ -290,6 +407,24 @@ public void retrieveEntryShouldReturnExpectedResponse() { // then assertThat(result.result()) .isEqualTo(ModuleCacheResponse.of("some-key", StorageDataType.JSON, "some-value")); + + verify(metrics).updateModuleStorageCacheReadRequestTime(eq("some-module-code"), anyLong(), eq(MetricName.ok)); + } + + @Test + public void retrieveEntryShouldReturnFailureResponse() { + //given + given(httpClient.get(anyString(), any(), anyLong())).willReturn(Future.succeededFuture( + HttpClientResponse.of(500, null, "error"))); + + // when + final Future result = + target.retrieveEntry("some-key", "some-module-code", "some-app"); + + // then + assertThat(result.succeeded()).isFalse(); + + verify(metrics).updateModuleStorageCacheReadRequestTime(eq("some-module-code"), anyLong(), eq(MetricName.err)); } @SneakyThrows diff --git a/src/test/java/org/prebid/server/metric/MetricsTest.java b/src/test/java/org/prebid/server/metric/MetricsTest.java index 1ee76608ede..8a56e279d30 100644 --- a/src/test/java/org/prebid/server/metric/MetricsTest.java +++ b/src/test/java/org/prebid/server/metric/MetricsTest.java @@ -1265,6 +1265,28 @@ public void shouldIncrementVtrackWritePrebidCacheRequestTimer() { assertThat(metricRegistry.timer("account.accountId.prebid_cache.vtrack.write.err").getCount()).isOne(); } + @Test + public void shouldIncrementModuleStorageReadPrebidCacheRequestTimer() { + // when + metrics.updateModuleStorageCacheReadRequestTime("module_code", 1424L, MetricName.ok); + metrics.updateModuleStorageCacheReadRequestTime("module_code", 1424L, MetricName.err); + + // then + assertThat(metricRegistry.timer("prebid_cache.module_storage.module_code.read.ok").getCount()).isEqualTo(1); + assertThat(metricRegistry.timer("prebid_cache.module_storage.module_code.read.err").getCount()).isEqualTo(1); + } + + @Test + public void shouldIncrementModuleStorageWritePrebidCacheRequestTimer() { + // when + metrics.updateModuleStorageCacheWriteRequestTime("module_code", 1424L, MetricName.ok); + metrics.updateModuleStorageCacheWriteRequestTime("module_code", 1424L, MetricName.err); + + // then + assertThat(metricRegistry.timer("prebid_cache.module_storage.module_code.write.ok").getCount()).isEqualTo(1); + assertThat(metricRegistry.timer("prebid_cache.module_storage.module_code.write.err").getCount()).isEqualTo(1); + } + @Test public void shouldIncrementPrebidCacheCreativeSizeHistogram() { // when @@ -1303,6 +1325,25 @@ public void shouldIncrementPrebidCacheVtrackCreativeSizeHistogram() { .isEqualTo(1); } + @Test + public void shouldIncrementPrebidCacheModuleStorageCreativeSizeHistogram() { + // when + metrics.updateModuleStorageCacheEntrySize("module_code", 123, MetricName.json); + metrics.updateModuleStorageCacheEntrySize("module_code", 456, MetricName.xml); + metrics.updateModuleStorageCacheEntrySize("module_code", 789, MetricName.unknown); + metrics.updateModuleStorageCacheEntrySize("module_code", 1011, MetricName.text); + + // then + assertThat(metricRegistry.histogram("prebid_cache.module_storage.module_code.entry_size.json").getCount()) + .isEqualTo(1); + assertThat(metricRegistry.histogram("prebid_cache.module_storage.module_code.entry_size.xml").getCount()) + .isEqualTo(1); + assertThat(metricRegistry.histogram("prebid_cache.module_storage.module_code.entry_size.unknown").getCount()) + .isEqualTo(1); + assertThat(metricRegistry.histogram("prebid_cache.module_storage.module_code.entry_size.text").getCount()) + .isEqualTo(1); + } + @Test public void shouldIncrementPrebidCacheCreativeTtlHistogram() { // when @@ -1341,6 +1382,25 @@ public void shouldIncrementPrebidCacheVtrackCreativeTtlHistogram() { .isEqualTo(1); } + @Test + public void shouldIncrementPrebidCacheModuleStorageCreativeTtlHistogram() { + // when + metrics.updateModuleStorageCacheEntryTtl("module_code", 123, MetricName.json); + metrics.updateModuleStorageCacheEntryTtl("module_code", 456, MetricName.xml); + metrics.updateModuleStorageCacheEntryTtl("module_code", 789, MetricName.unknown); + metrics.updateModuleStorageCacheEntryTtl("module_code", 1011, MetricName.text); + + // then + assertThat(metricRegistry.histogram("prebid_cache.module_storage.module_code.entry_ttl.json").getCount()) + .isEqualTo(1); + assertThat(metricRegistry.histogram("prebid_cache.module_storage.module_code.entry_ttl.xml").getCount()) + .isEqualTo(1); + assertThat(metricRegistry.histogram("prebid_cache.module_storage.module_code.entry_ttl.unknown").getCount()) + .isEqualTo(1); + assertThat(metricRegistry.histogram("prebid_cache.module_storage.module_code.entry_ttl.text").getCount()) + .isEqualTo(1); + } + @Test public void shouldCreateCurrencyRatesGaugeMetric() { // when