From 181cb8cb745383c09f62302c6247d8e8e9c6d36b Mon Sep 17 00:00:00 2001 From: Ian Downard <54998167+ianwow@users.noreply.github.com> Date: Mon, 17 Mar 2025 15:11:09 -0700 Subject: [PATCH 1/7] add unit tests for S3SettingsConfiguration --- .../config/SettingsConfigurationTest.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 src/test/java/org/prebid/server/spring/config/SettingsConfigurationTest.java diff --git a/src/test/java/org/prebid/server/spring/config/SettingsConfigurationTest.java b/src/test/java/org/prebid/server/spring/config/SettingsConfigurationTest.java new file mode 100644 index 00000000000..8a517a1b0e9 --- /dev/null +++ b/src/test/java/org/prebid/server/spring/config/SettingsConfigurationTest.java @@ -0,0 +1,51 @@ +package org.prebid.server.spring.config; + +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.s3.S3AsyncClient; + +import java.net.URISyntaxException; + +import static org.assertj.core.api.Assertions.assertThat; + +public class SettingsConfigurationTest { + + @Test + void shouldCreateClientWhenStaticCredentialsNotUsed() throws URISyntaxException { + // given + final SettingsConfiguration.S3SettingsConfiguration.S3ConfigurationProperties s3Properties = + new SettingsConfiguration.S3SettingsConfiguration.S3ConfigurationProperties(); + s3Properties.setEndpoint("http://example.com"); + s3Properties.setForcePathStyle(true); + s3Properties.setRegion("us-east-1"); + // Not setting access key and secret key properties should + // cause the S3AsyncClient to use DefaultCredentialsProvider. + + // when + final S3AsyncClient client = new SettingsConfiguration.S3SettingsConfiguration().s3AsyncClient(s3Properties); + + // then + assertThat(client).isNotNull(); + } + + @Test + void shouldCreateClientWhenStaticCredentialsUsed() throws URISyntaxException { + // given + final SettingsConfiguration.S3SettingsConfiguration.S3ConfigurationProperties s3Properties = + new SettingsConfiguration.S3SettingsConfiguration.S3ConfigurationProperties(); + s3Properties.setEndpoint("http://localhost:4566"); + s3Properties.setForcePathStyle(true); + s3Properties.setRegion("us-east-1"); + // Setting access key and secret key properties should cause + // the S3AsyncClient to use StaticCredentialsProvider. + s3Properties.setAccessKeyId("testAccessKeyId"); + s3Properties.setSecretAccessKey("testSecretAccessKey"); + + // when + final S3AsyncClient client = + new SettingsConfiguration.S3SettingsConfiguration().s3AsyncClient(s3Properties); + + // then + assertThat(client).isNotNull(); + } + +} From 88f43656c9a8fb1cd570a5aaa033c3829c25a3f6 Mon Sep 17 00:00:00 2001 From: Ian Downard <54998167+ianwow@users.noreply.github.com> Date: Mon, 17 Mar 2025 15:12:20 -0700 Subject: [PATCH 2/7] add AWS default credentials provider --- .../spring/config/SettingsConfiguration.java | 57 +++++++++++++++---- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java index f101495eb66..0c333affca9 100644 --- a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java @@ -1,5 +1,8 @@ package org.prebid.server.spring.config; +import org.apache.commons.lang3.StringUtils; +import org.prebid.server.log.Logger; +import org.prebid.server.log.LoggerFactory; import io.vertx.core.Vertx; import io.vertx.core.file.FileSystem; import lombok.Data; @@ -40,9 +43,13 @@ import org.springframework.stereotype.Component; import org.springframework.validation.annotation.Validated; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; +import software.amazon.awssdk.core.exception.SdkClientException; import javax.validation.constraints.Min; import javax.validation.constraints.NotBlank; @@ -58,6 +65,8 @@ @UtilityClass public class SettingsConfiguration { + private static final Logger logger = LoggerFactory.getLogger(SettingsConfiguration.class); + @Configuration @ConditionalOnProperty(prefix = "settings.filesystem", name = {"settings-filename", "stored-requests-dir", "stored-imps-dir"}) @@ -233,18 +242,30 @@ static class S3SettingsConfiguration { @Component @ConfigurationProperties(prefix = "settings.s3") - @ConditionalOnProperty(prefix = "settings.s3", name = {"accessKeyId", "secretAccessKey"}) @Validated @Data @NoArgsConstructor protected static class S3ConfigurationProperties { - @NotBlank + /** + * If accessKeyId and secretAccessKey are provided in the + * configuration file then they will be used. Otherwise, the + * DefaultCredentialsProvider will look for credentials in this order: + * + * - Java System Properties + * - Environment Variables + * - Web Identity Token + * - AWS credentials file (~/.aws/credentials) + * - ECS container credentials + * - EC2 instance profile + */ private String accessKeyId; - - @NotBlank private String secretAccessKey; + private boolean useStaticCredentials() { + return StringUtils.isNotBlank(accessKeyId) && StringUtils.isNotBlank(secretAccessKey); + } + /** * If not provided AWS_GLOBAL will be used as a region */ @@ -274,20 +295,32 @@ protected static class S3ConfigurationProperties { @Bean S3AsyncClient s3AsyncClient(S3ConfigurationProperties s3ConfigurationProperties) throws URISyntaxException { - final AwsBasicCredentials credentials = AwsBasicCredentials.create( - s3ConfigurationProperties.getAccessKeyId(), - s3ConfigurationProperties.getSecretAccessKey()); + final Region awsRegion = Optional.ofNullable(s3ConfigurationProperties.getRegion()) .map(Region::of) .orElse(Region.AWS_GLOBAL); - return S3AsyncClient - .builder() - .credentialsProvider(StaticCredentialsProvider.create(credentials)) + final S3AsyncClientBuilder clientBuilder = S3AsyncClient.builder() .endpointOverride(new URI(s3ConfigurationProperties.getEndpoint())) .forcePathStyle(s3ConfigurationProperties.getForcePathStyle()) - .region(awsRegion) - .build(); + .region(awsRegion); + final AwsCredentialsProvider credentialsProvider; + if (s3ConfigurationProperties.useStaticCredentials()) { + final AwsBasicCredentials basicCredentials = AwsBasicCredentials.create( + s3ConfigurationProperties.getAccessKeyId(), + s3ConfigurationProperties.getSecretAccessKey()); + credentialsProvider = StaticCredentialsProvider.create(basicCredentials); + } else { + credentialsProvider = DefaultCredentialsProvider.create(); + } + try { + credentialsProvider.resolveCredentials(); + } catch (SdkClientException e) { + logger.error("Failed to resolve AWS credentials", e); + } + clientBuilder.credentialsProvider(credentialsProvider); + + return clientBuilder.build(); } @Bean From 5fc6200cc6d468b1e459a10c23ec70a5f3aac136 Mon Sep 17 00:00:00 2001 From: Ian Downard <54998167+ianwow@users.noreply.github.com> Date: Tue, 18 Mar 2025 13:17:54 -0700 Subject: [PATCH 3/7] Document the lookup order for AWS credentials --- docs/application-settings.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/application-settings.md b/docs/application-settings.md index 2a20e0d8342..d99607a1477 100644 --- a/docs/application-settings.md +++ b/docs/application-settings.md @@ -277,8 +277,8 @@ The general idea is that you'll place all the account-specific settings in a sep ```yaml settings: s3: - accessKeyId: - secretAccessKey: + accessKeyId: # optional + secretAccessKey: #optional endpoint: # http://s3.storage.com bucket: # prebid-application-settings region: # if not provided AWS_GLOBAL will be used. Example value: 'eu-central-1' @@ -298,6 +298,14 @@ settings: timeout: 5000 ``` +If `accessKeyId` and `secretAccessKey` are not specified in the Prebid Server configuration then AWS credentials will be looked up in this order: +- Java System Properties - `aws.accessKeyId` and `aws.secretAccessKey` +- Environment Variables - `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +- Web Identity Token credentials from system properties or environment variables +- Credential profiles file at the default location (`~/.aws/credentials`) shared by all AWS SDKs and the AWS CLI +- Credentials delivered through the Amazon EC2 container service if "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" environment variable is set and security manager has permission to access the variable, +- Instance profile credentials delivered through the Amazon EC2 metadata service + ### File format We recommend using the `json` format for your account configuration. A minimal configuration may look like this. From 8018d31ec02643b6c2fd6d18f0382d759b60cd27 Mon Sep 17 00:00:00 2001 From: Ian Downard <54998167+ianwow@users.noreply.github.com> Date: Tue, 18 Mar 2025 13:18:45 -0700 Subject: [PATCH 4/7] Document the lookup order for AWS credentials --- docs/config-app.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/config-app.md b/docs/config-app.md index 1ab92b1c929..4ed99996b9f 100644 --- a/docs/config-app.md +++ b/docs/config-app.md @@ -390,8 +390,8 @@ contain 'WHERE last_updated > ?' for MySQL and 'WHERE last_updated > $1' for Pos For S3 storage configuration - `settings.in-memory-cache.s3-update.refresh-rate` - refresh period in ms for stored request updates in S3 -- `settings.s3.access-key-id` - an access key -- `settings.s3.secret-access-key` - a secret access key +- `settings.s3.access-key-id` - an access key (optional) +- `settings.s3.secret-access-key` - a secret access key (optional) - `settings.s3.region` - a region, AWS_GLOBAL by default - `settings.s3.endpoint` - an endpoint - `settings.s3.bucket` - a bucket name @@ -401,6 +401,15 @@ For S3 storage configuration - `settings.s3.stored-requests-dir` - a directory with stored requests - `settings.s3.stored-responses-dir` - a directory with stored responses +If `settings.s3.access-key-id` and `settings.s3.secret-access-key` are not specified in the Prebid Server configuration then AWS credentials will be looked up in this order: +- Java System Properties - `aws.accessKeyId` and `aws.secretAccessKey` +- Environment Variables - `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +- Web Identity Token credentials from system properties or environment variables +- Credential profiles file at the default location (`~/.aws/credentials`) shared by all AWS SDKs and the AWS CLI +- Credentials delivered through the Amazon EC2 container service if "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" environment variable is set and security manager has permission to access the variable, +- Instance profile credentials delivered through the Amazon EC2 metadata service + + For targeting available next options: - `settings.targeting.truncate-attr-chars` - set the max length for names of targeting keywords (0 means no truncation). From 7c5f30950aabbd348bea35a604b41f3191606399 Mon Sep 17 00:00:00 2001 From: Ian Downard <54998167+ianwow@users.noreply.github.com> Date: Wed, 2 Apr 2025 15:48:21 -0700 Subject: [PATCH 5/7] Delete unit test for SettingsConfiguration.java There is no need to unit test the configuration. --- .../config/SettingsConfigurationTest.java | 51 ------------------- 1 file changed, 51 deletions(-) delete mode 100644 src/test/java/org/prebid/server/spring/config/SettingsConfigurationTest.java diff --git a/src/test/java/org/prebid/server/spring/config/SettingsConfigurationTest.java b/src/test/java/org/prebid/server/spring/config/SettingsConfigurationTest.java deleted file mode 100644 index 8a517a1b0e9..00000000000 --- a/src/test/java/org/prebid/server/spring/config/SettingsConfigurationTest.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.prebid.server.spring.config; - -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.services.s3.S3AsyncClient; - -import java.net.URISyntaxException; - -import static org.assertj.core.api.Assertions.assertThat; - -public class SettingsConfigurationTest { - - @Test - void shouldCreateClientWhenStaticCredentialsNotUsed() throws URISyntaxException { - // given - final SettingsConfiguration.S3SettingsConfiguration.S3ConfigurationProperties s3Properties = - new SettingsConfiguration.S3SettingsConfiguration.S3ConfigurationProperties(); - s3Properties.setEndpoint("http://example.com"); - s3Properties.setForcePathStyle(true); - s3Properties.setRegion("us-east-1"); - // Not setting access key and secret key properties should - // cause the S3AsyncClient to use DefaultCredentialsProvider. - - // when - final S3AsyncClient client = new SettingsConfiguration.S3SettingsConfiguration().s3AsyncClient(s3Properties); - - // then - assertThat(client).isNotNull(); - } - - @Test - void shouldCreateClientWhenStaticCredentialsUsed() throws URISyntaxException { - // given - final SettingsConfiguration.S3SettingsConfiguration.S3ConfigurationProperties s3Properties = - new SettingsConfiguration.S3SettingsConfiguration.S3ConfigurationProperties(); - s3Properties.setEndpoint("http://localhost:4566"); - s3Properties.setForcePathStyle(true); - s3Properties.setRegion("us-east-1"); - // Setting access key and secret key properties should cause - // the S3AsyncClient to use StaticCredentialsProvider. - s3Properties.setAccessKeyId("testAccessKeyId"); - s3Properties.setSecretAccessKey("testSecretAccessKey"); - - // when - final S3AsyncClient client = - new SettingsConfiguration.S3SettingsConfiguration().s3AsyncClient(s3Properties); - - // then - assertThat(client).isNotNull(); - } - -} From ed2043fd12ca5b06d67f53723e74d5535beca514 Mon Sep 17 00:00:00 2001 From: Ian Downard <54998167+ianwow@users.noreply.github.com> Date: Thu, 3 Apr 2025 13:50:11 -0700 Subject: [PATCH 6/7] noop --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9fbfe912715..f585b577aa9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ ### This is the Java version of Prebid Server. See the Prebid Server [Feature List](https://docs.prebid.org/prebid-server/features/pbs-feature-idx.html) and [FAQ entry](https://docs.prebid.org/faq/prebid-server-faq.html#why-are-there-two-versions-of-prebid-server-are-they-kept-in-sync) to understand the differences between PBS-Java and [PBS-Go](https://github.com/prebid/prebid-server). + # Prebid Server (Java) [![GitHub version](https://badge.fury.io/gh/prebid%2fprebid-server-java.svg)](http://badge.fury.io/gh/prebid%2fprebid-server-java) From ae6feaf6eacc484862f622b7dcb43b2f31ffdec3 Mon Sep 17 00:00:00 2001 From: Ian Downard <54998167+ianwow@users.noreply.github.com> Date: Fri, 4 Apr 2025 07:43:22 -0700 Subject: [PATCH 7/7] noop --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index f585b577aa9..9fbfe912715 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ ### This is the Java version of Prebid Server. See the Prebid Server [Feature List](https://docs.prebid.org/prebid-server/features/pbs-feature-idx.html) and [FAQ entry](https://docs.prebid.org/faq/prebid-server-faq.html#why-are-there-two-versions-of-prebid-server-are-they-kept-in-sync) to understand the differences between PBS-Java and [PBS-Go](https://github.com/prebid/prebid-server). - # Prebid Server (Java) [![GitHub version](https://badge.fury.io/gh/prebid%2fprebid-server-java.svg)](http://badge.fury.io/gh/prebid%2fprebid-server-java)