Bidder Configuration Fixes & URL Validation

extra/modules/live-intent-omni-channel-identity/src/main/java/org/prebid/server/hooks/modules/liveintent/omni/channel/identity/v1/hooks/LiveIntentOmniChannelIdentityProcessedAuctionRequestHook.java

@@ -63,7 +63,7 @@ public LiveIntentOmniChannelIdentityProcessedAuctionRequestHook(LiveIntentOmniCh
                                                                     double logSamplingRate) {
 
         this.config = Objects.requireNonNull(config);
-        HttpUtil.validateUrlSyntax(config.getIdentityResolutionEndpoint());
+        HttpUtil.validateUrl(config.getIdentityResolutionEndpoint());
         this.mapper = Objects.requireNonNull(mapper);
         this.httpClient = Objects.requireNonNull(httpClient);
         this.logSamplingRate = logSamplingRate;

extra/modules/wurfl-devicedetection/src/main/java/org/prebid/server/hooks/modules/com/scientiamobile/wurfl/devicedetection/model/WURFLEngineUtils.java

@@ -6,8 +6,9 @@
 import com.scientiamobile.wurfl.core.cache.NullCacheProvider;
 import org.prebid.server.hooks.modules.com.scientiamobile.wurfl.devicedetection.config.WURFLDeviceDetectionConfigProperties;
 import org.prebid.server.hooks.modules.com.scientiamobile.wurfl.devicedetection.exc.WURFLDeviceDetectionException;
+import org.prebid.server.util.HttpUtil;
 
-import java.net.URI;
+import java.net.URL;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -56,8 +57,8 @@ private static String wurflFilePathFromConfig(WURFLDeviceDetectionConfigProperti
 
     public static String extractWURFLFileName(String wurflSnapshotUrl) {
         try {
-            final URI uri = new URI(wurflSnapshotUrl);
-            final String path = uri.getPath();
+            final URL url = HttpUtil.parseUrl(wurflSnapshotUrl);
+            final String path = url.getPath();
             return path.substring(path.lastIndexOf('/') + 1);
         } catch (Exception e) {
             throw new IllegalArgumentException("Invalid WURFL snapshot URL: " + wurflSnapshotUrl, e);

extra/pom.xml

@@ -39,9 +39,7 @@
         <commons.collections.version>4.4</commons.collections.version>
         <commons.compress.version>1.27.1</commons.compress.version>
         <commons-math3.version>3.6.1</commons-math3.version>
-        <commons-validator.version>1.10.0</commons-validator.version>
         <scram.version>2.1</scram.version>
-        <httpclient.version>4.5.14</httpclient.version>
         <ipaddress.version>5.5.1</ipaddress.version>
         <oshi.version>6.8.0</oshi.version>
         <json-schema-validator.version>1.5.6</json-schema-validator.version>
@@ -136,23 +134,6 @@
                 <artifactId>commons-math3</artifactId>
                 <version>${commons-math3.version}</version>
             </dependency>
-            <dependency>
-                <groupId>commons-validator</groupId>
-                <artifactId>commons-validator</artifactId>
-                <version>${commons-validator.version}</version>
-            </dependency>
-            <!-- TODO: refactor code to replace URIBuilder with something else so that this dep can be removed -->
-            <dependency>
-                <groupId>org.apache.httpcomponents</groupId>
-                <artifactId>httpclient</artifactId>
-                <version>${httpclient.version}</version>
-                <exclusions>
-                    <exclusion>
-                        <groupId>commons-logging</groupId>
-                        <artifactId>commons-logging</artifactId>
-                    </exclusion>
-                </exclusions>
-            </dependency>
             <dependency>
                 <groupId>com.github.seancfoley</groupId>
                 <artifactId>ipaddress</artifactId>

pom.xml

@@ -80,6 +80,10 @@
             <groupId>io.vertx</groupId>
             <artifactId>vertx-auth-common</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.vertx</groupId>
+            <artifactId>vertx-uri-template</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.ongres.scram</groupId>
             <artifactId>client</artifactId>
@@ -110,14 +114,6 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-math3</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.apache.httpcomponents</groupId>
-            <artifactId>httpclient</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>commons-validator</groupId>
-            <artifactId>commons-validator</artifactId>
-        </dependency>
         <dependency>
             <groupId>com.github.seancfoley</groupId>
             <artifactId>ipaddress</artifactId>

src/main/java/org/prebid/server/analytics/reporter/liveintent/LiveIntentAnalyticsReporter.java

@@ -5,8 +5,10 @@
 import com.iab.openrtb.response.BidResponse;
 import com.iab.openrtb.response.SeatBid;
 import io.vertx.core.Future;
+import io.vertx.uritemplate.UriTemplate;
+import io.vertx.uritemplate.Variables;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.http.client.utils.URIBuilder;
+import org.apache.commons.lang3.StringUtils;
 import org.prebid.server.analytics.AnalyticsReporter;
 import org.prebid.server.analytics.model.AuctionEvent;
 import org.prebid.server.analytics.model.NotificationEvent;
@@ -27,9 +29,12 @@
 import org.prebid.server.log.LoggerFactory;
 import org.prebid.server.proto.openrtb.ext.request.ExtRequest;
 import org.prebid.server.proto.openrtb.ext.request.ExtRequestPrebid;
+import org.prebid.server.util.HttpUtil;
+import org.prebid.server.util.UriTemplateUtil;
 import org.prebid.server.vertx.httpclient.HttpClient;
 
-import java.net.URISyntaxException;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
@@ -44,6 +49,7 @@ public class LiveIntentAnalyticsReporter implements AnalyticsReporter {
     private final HttpClient httpClient;
     private final LiveIntentAnalyticsProperties properties;
     private final JacksonMapper jacksonMapper;
+    private final UriTemplate uriTemplate;
 
     public LiveIntentAnalyticsReporter(
             LiveIntentAnalyticsProperties properties,
@@ -53,6 +59,21 @@ public LiveIntentAnalyticsReporter(
         this.httpClient = Objects.requireNonNull(httpClient);
         this.properties = Objects.requireNonNull(properties);
         this.jacksonMapper = Objects.requireNonNull(jacksonMapper);
+
+        final URL url;
+        try {
+            url = HttpUtil.parseUrl(properties.getAnalyticsEndpoint());
+        } catch (MalformedURLException e) {
+            throw new IllegalArgumentException(e);
+        }
+
+        final String query = url.getQuery() != null ? "?" + url.getQuery() : StringUtils.EMPTY;
+        this.uriTemplate = UriTemplateUtil.createTemplate(
+                url.getProtocol() + "://" + url.getAuthority() + "/analytic-events{/path}" + query,
+                query.contains("?"),
+                "b",
+                "bidId");
+
     }
 
     @Override
@@ -96,10 +117,7 @@ private Future<Void> processAuctionEvent(AuctionContext auctionContext) {
 
         try {
             return httpClient.post(
-                    new URIBuilder(properties.getAnalyticsEndpoint())
-                            .setPath("/analytic-events/pbsj-bids")
-                            .build()
-                            .toString(),
+                    uriTemplate.expandToString(Variables.variables().set("path", "pbsj-bids")),
                     jacksonMapper.encodeToString(pbsjBids),
                     properties.getTimeoutMs())
                     .mapEmpty();
@@ -169,18 +187,11 @@ private Optional<PbsjBid> buildPbsjBid(
     }
 
     private Future<Void> processNotificationEvent(NotificationEvent notificationEvent) {
-        try {
-            final String url = new URIBuilder(properties.getAnalyticsEndpoint())
-                    .setPath("/analytic-events/pbsj-winning-bid")
-                    .setParameter("b", notificationEvent.getBidder())
-                    .setParameter("bidId", notificationEvent.getBidId())
-                    .build()
-                    .toString();
-            return httpClient.get(url, properties.getTimeoutMs()).mapEmpty();
-        } catch (URISyntaxException e) {
-            logger.error("Error composing url for notification event: {}", e.getMessage());
-            return Future.failedFuture(e);
-        }
+        final String url = uriTemplate.expandToString(Variables.variables()
+                .set("path", "pbsj-winning-bid")
+                .set("b", notificationEvent.getBidder())
+                .set("bidId", notificationEvent.getBidId()));
+        return httpClient.get(url, properties.getTimeoutMs()).mapEmpty();
     }
 
     @Override

src/main/java/org/prebid/server/auction/IpAddressHelper.java

@@ -5,17 +5,19 @@
 import inet.ipaddr.IPAddressString;
 import inet.ipaddr.IPAddressStringParameters;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.conn.util.InetAddressUtils;
 import org.prebid.server.auction.model.IpAddress;
 import org.prebid.server.log.Logger;
 import org.prebid.server.log.LoggerFactory;
 
 import java.util.List;
+import java.util.regex.Pattern;
 
 public class IpAddressHelper {
 
     private static final Logger logger = LoggerFactory.getLogger(IpAddressHelper.class);
 
+    private static final Pattern IP_V4_PATTERN =
+            Pattern.compile("^(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\\.(?!$)|$)){4}$");
     private static final IPAddressStringParameters IP_ADDRESS_VALIDATION_OPTIONS =
             IPAddressString.DEFAULT_VALIDATION_OPTIONS.toBuilder()
                     .allowSingleSegment(false)
@@ -73,7 +75,7 @@ public IpAddress toIpAddress(String ip) {
     }
 
     public String maskIpv4(String ip) {
-        if (StringUtils.isBlank(ip) || !InetAddressUtils.isIPv4Address(ip)) {
+        if (StringUtils.isBlank(ip) || !IP_V4_PATTERN.matcher(ip).matches()) {
             return ip;
         }
 

src/main/java/org/prebid/server/bidder/adgeneration/AdgenerationBidder.java

@@ -12,9 +12,10 @@
 import com.iab.openrtb.response.Bid;
 import io.vertx.core.MultiMap;
 import io.vertx.core.http.HttpMethod;
+import io.vertx.uritemplate.UriTemplate;
+import io.vertx.uritemplate.Variables;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.URIBuilder;
 import org.prebid.server.bidder.Bidder;
 import org.prebid.server.bidder.adgeneration.model.AdgenerationResponse;
 import org.prebid.server.bidder.model.BidderBid;
@@ -31,10 +32,12 @@
 import org.prebid.server.proto.openrtb.ext.response.BidType;
 import org.prebid.server.util.HttpUtil;
 import org.prebid.server.util.ObjectUtil;
+import org.prebid.server.util.UriTemplateUtil;
 
-import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -51,11 +54,12 @@ public class AdgenerationBidder implements Bidder<Void> {
             new TypeReference<>() {
             };
 
-    private final String endpointUrl;
+    private final UriTemplate endpointUrlTemplate;
     private final JacksonMapper mapper;
 
     public AdgenerationBidder(String endpointUrl, JacksonMapper mapper) {
-        this.endpointUrl = HttpUtil.validateUrl(Objects.requireNonNull(endpointUrl));
+        HttpUtil.validateUrl(Objects.requireNonNull(endpointUrl));
+        this.endpointUrlTemplate = UriTemplateUtil.createTemplate(endpointUrl, "queryParams");
         this.mapper = Objects.requireNonNull(mapper);
     }
 
@@ -96,47 +100,41 @@ private ExtImpAdgeneration parseAndValidateImpExt(Imp imp) {
     }
 
     private String getUri(String adSize, String id, String currency, BidRequest bidRequest) {
-        final URIBuilder uriBuilder;
-        try {
-            uriBuilder = new URIBuilder(endpointUrl);
-        } catch (URISyntaxException e) {
-            throw new PreBidException("Invalid url: %s, error: %s".formatted(endpointUrl, e.getMessage()));
-        }
-
-        uriBuilder
-                .addParameter("posall", "SSPLOC")
-                .addParameter("id", id)
-                .addParameter("hb", "true")
-                .addParameter("t", "json3")
-                .addParameter("currency", currency)
-                .addParameter("sdkname", "prebidserver")
-                .addParameter("adapterver", VERSION);
-
-        addParameterIfNotEmpty(uriBuilder, "sizes", adSize);
-        addParameterIfNotEmpty(uriBuilder, "tp", ObjectUtil.getIfNotNull(bidRequest.getSite(), Site::getPage));
-        addParameterIfNotEmpty(uriBuilder, "appbundle", ObjectUtil.getIfNotNull(bidRequest.getApp(), App::getBundle));
-        addParameterIfNotEmpty(uriBuilder, "appname", ObjectUtil.getIfNotNull(bidRequest.getApp(), App::getName));
-        addParameterIfNotEmpty(
-                uriBuilder, "transactionid", ObjectUtil.getIfNotNull(bidRequest.getSource(), Source::getTid));
+        final Map<String, String> queryParams = new HashMap<>();
+
+        queryParams.put("posall", "SSPLOC");
+        queryParams.put("id", id);
+        queryParams.put("hb", "true");
+        queryParams.put("t", "json3");
+        queryParams.put("currency", currency);
+        queryParams.put("sdkname", "prebidserver");
+        queryParams.put("adapterver", VERSION);
+
+        addParameterIfNotEmpty(queryParams, "sizes", adSize);
+        addParameterIfNotEmpty(queryParams, "tp", ObjectUtil.getIfNotNull(bidRequest.getSite(), Site::getPage));
+        addParameterIfNotEmpty(queryParams, "appbundle", ObjectUtil.getIfNotNull(bidRequest.getApp(), App::getBundle));
+        addParameterIfNotEmpty(queryParams, "appname", ObjectUtil.getIfNotNull(bidRequest.getApp(), App::getName));
+        addParameterIfNotEmpty(queryParams, "transactionid",
+                ObjectUtil.getIfNotNull(bidRequest.getSource(), Source::getTid));
 
         final Device device = bidRequest.getDevice();
         final String deviceOs = device != null ? device.getOs() : null;
         if ("android".equals(deviceOs)) {
-            uriBuilder.addParameter("sdktype", "1");
-            addParameterIfNotEmpty(uriBuilder, "advertising_id", device.getIfa());
+            queryParams.put("sdktype", "1");
+            addParameterIfNotEmpty(queryParams, "advertising_id", device.getIfa());
         } else if ("ios".equals(deviceOs)) {
-            uriBuilder.addParameter("sdktype", "2");
-            addParameterIfNotEmpty(uriBuilder, "idfa", device.getIfa());
+            queryParams.put("sdktype", "2");
+            addParameterIfNotEmpty(queryParams, "idfa", device.getIfa());
         } else {
-            uriBuilder.addParameter("sdktype", "0");
+            queryParams.put("sdktype", "0");
         }
 
-        return uriBuilder.toString();
+        return endpointUrlTemplate.expandToString(Variables.variables().set("queryParams", queryParams));
     }
 
-    private static void addParameterIfNotEmpty(URIBuilder uriBuilder, String parameter, String value) {
+    private static void addParameterIfNotEmpty(Map<String, String> queryParams, String parameter, String value) {
         if (StringUtils.isNotEmpty(value)) {
-            uriBuilder.addParameter(parameter, value);
+            queryParams.put(parameter, value);
         }
     }
 

src/main/java/org/prebid/server/bidder/adkernel/AdkernelBidder.java

@@ -41,6 +41,8 @@ public class AdkernelBidder implements Bidder<BidRequest> {
             new TypeReference<>() {
             };
 
+    private static final String ZONE_ID_MACRO = "{{ZoneId}}";
+
     private static final String MF_SUFFIX = "__mf";
     private static final String MF_SUFFIX_BANNER = "b" + MF_SUFFIX;
     private static final String MF_SUFFIX_VIDEO = "v" + MF_SUFFIX;
@@ -49,11 +51,11 @@ public class AdkernelBidder implements Bidder<BidRequest> {
 
     private static final int MF_SUFFIX_LENGTH = MF_SUFFIX.length() + 1;
 
-    private final String endpointTemplate;
+    private final String endpointUrl;
     private final JacksonMapper mapper;
 
-    public AdkernelBidder(String endpointTemplate, JacksonMapper mapper) {
-        this.endpointTemplate = HttpUtil.validateUrl(Objects.requireNonNull(endpointTemplate));
+    public AdkernelBidder(String endpointUrl, JacksonMapper mapper) {
+        this.endpointUrl = HttpUtil.validateUrl(Objects.requireNonNull(endpointUrl));
         this.mapper = Objects.requireNonNull(mapper);
     }
 
@@ -183,10 +185,9 @@ private HttpRequest<BidRequest> createHttpRequest(Map.Entry<ExtImpAdkernel, List
                                                       App app) {
 
         final ExtImpAdkernel impExt = extAndImp.getKey();
-        final String uri = endpointTemplate.formatted(impExt.getZoneId());
+        final String uri = endpointUrl.replace(ZONE_ID_MACRO, HttpUtil.encodeUrl(impExt.getZoneId().toString()));
 
-        final MultiMap headers = HttpUtil.headers()
-                .add(HttpUtil.X_OPENRTB_VERSION_HEADER, "2.5");
+        final MultiMap headers = HttpUtil.headers().add(HttpUtil.X_OPENRTB_VERSION_HEADER, "2.5");
 
         final BidRequest outgoingRequest = createBidRequest(extAndImp.getValue(), requestBuilder, site, app);