diff --git a/buildSrc/src/main/kotlin/ai.smoke-test.gradle.kts b/buildSrc/src/main/kotlin/ai.smoke-test.gradle.kts
index 805acb06c4..fbed49ea3b 100644
--- a/buildSrc/src/main/kotlin/ai.smoke-test.gradle.kts
+++ b/buildSrc/src/main/kotlin/ai.smoke-test.gradle.kts
@@ -8,19 +8,6 @@ plugins {
 
 val aiSmokeTest = extensions.create<AiSmokeTestExtension>("aiSmokeTest")
 
-sourceSets {
-  create("smokeTest") {
-    compileClasspath += sourceSets.main.get().output
-    runtimeClasspath += sourceSets.main.get().output
-  }
-}
-
-val smokeTestImplementation by configurations.getting {
-  extendsFrom(configurations.implementation.get())
-}
-
-configurations["smokeTestRuntimeOnly"].extendsFrom(configurations.runtimeOnly.get())
-
 // FIXME (trask) copy-pasted from ai.java-conventions.gradle
 java {
   toolchain {
@@ -66,18 +53,33 @@ dependencies {
   // FIXME (trask) copy-pasted from ai.java-conventions.gradle
   dependencyManagement(platform(project(":dependencyManagement")))
 
-  smokeTestImplementation(project(":smoke-tests:framework"))
+  agent(project(":agent:agent", configuration = "shadow"))
+
+  old3xAgent("com.microsoft.azure:applicationinsights-agent:3.2.11")
+}
 
-  smokeTestImplementation("org.junit.jupiter:junit-jupiter-api")
-  smokeTestImplementation("org.junit.jupiter:junit-jupiter-params")
-  smokeTestImplementation("org.junit.jupiter:junit-jupiter-engine")
-  smokeTestImplementation("org.junit.platform:junit-platform-launcher")
+// Configure test suites
+testing {
+  suites {
+    register<JvmTestSuite>("smokeTest") {
+      dependencies {
+        implementation(project(":smoke-tests:framework"))
 
-  smokeTestImplementation("org.assertj:assertj-core")
+        implementation("org.junit.jupiter:junit-jupiter-api")
+        implementation("org.junit.jupiter:junit-jupiter-params")
+        runtimeOnly("org.junit.jupiter:junit-jupiter-engine")
+        runtimeOnly("org.junit.platform:junit-platform-launcher")
 
-  agent(project(":agent:agent", configuration = "shadow"))
+        implementation("org.assertj:assertj-core")
+      }
+    }
+  }
+}
 
-  old3xAgent("com.microsoft.azure:applicationinsights-agent:3.2.11")
+// Make smokeTest configuration extend from main implementation configuration
+// so that project dependencies (e.g., log4j in ClassicSdkLog4j1Interop2x) are available
+configurations.named("smokeTestImplementation") {
+  extendsFrom(configurations.implementation.get())
 }
 
 configurations.all {
@@ -106,7 +108,7 @@ tasks {
     }
   }
 
-  register<Test>("smokeTest") {
+  named<Test>("smokeTest") {
     useJUnitPlatform()
 
     // this is just to force building the agent first
@@ -114,9 +116,6 @@ tasks {
 
     dependsOn(assemble)
 
-    testClassesDirs = sourceSets["smokeTest"].output.classesDirs
-    classpath = sourceSets["smokeTest"].runtimeClasspath
-
     // TODO (trask) experiment with parallelization
     // maxParallelForks = (Runtime.getRuntime().availableProcessors() / 2).takeIf { it > 0 } ?: 1
 
diff --git a/smoke-tests/apps/OtlpMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/OtlpLogAnalyticsOnAksTest.java b/smoke-tests/apps/OtlpMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/OtlpLogAnalyticsOnAksTest.java
index 246cbfb687..0b70b310ff 100644
--- a/smoke-tests/apps/OtlpMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/OtlpLogAnalyticsOnAksTest.java
+++ b/smoke-tests/apps/OtlpMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/OtlpLogAnalyticsOnAksTest.java
@@ -17,7 +17,6 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.awaitility.Awaitility.await;
 import static org.mockserver.model.HttpRequest.request;
-import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;
 
 import com.microsoft.applicationinsights.smoketest.schemav2.Data;
 import com.microsoft.applicationinsights.smoketest.schemav2.Envelope;
@@ -28,11 +27,7 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 import org.mockserver.model.HttpRequest;
-import org.springframework.boot.test.context.SpringBootTest;
 
-@SpringBootTest(
-    classes = {OtlpApplication.class},
-    webEnvironment = RANDOM_PORT)
 @UseAgent
 abstract class OtlpLogAnalyticsOnAksTest {
 
diff --git a/smoke-tests/apps/OtlpMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/OtlpTest.java b/smoke-tests/apps/OtlpMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/OtlpTest.java
index b66c1509d4..7fb3ed481a 100644
--- a/smoke-tests/apps/OtlpMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/OtlpTest.java
+++ b/smoke-tests/apps/OtlpMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/OtlpTest.java
@@ -8,7 +8,6 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.awaitility.Awaitility.await;
 import static org.mockserver.model.HttpRequest.request;
-import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;
 
 import com.microsoft.applicationinsights.smoketest.schemav2.Data;
 import com.microsoft.applicationinsights.smoketest.schemav2.Envelope;
@@ -19,11 +18,7 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 import org.mockserver.model.HttpRequest;
-import org.springframework.boot.test.context.SpringBootTest;
 
-@SpringBootTest(
-    classes = {OtlpApplication.class},
-    webEnvironment = RANDOM_PORT)
 @UseAgent
 abstract class OtlpTest {