From 4dd472c1b2df977d66f3f1b5ff4ac2f37742f28e Mon Sep 17 00:00:00 2001 From: Erick Benitez-Ramos Date: Wed, 26 Mar 2025 21:49:29 -0700 Subject: [PATCH 1/2] fix: fix flaky spark processor integ --- tests/integ/test_spark_processing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integ/test_spark_processing.py b/tests/integ/test_spark_processing.py index 25a4942d70..17e88ed3e0 100644 --- a/tests/integ/test_spark_processing.py +++ b/tests/integ/test_spark_processing.py @@ -35,7 +35,7 @@ SPARK_PATH = os.path.join(DATA_DIR, "spark") -@pytest.fixture(scope="module") +@pytest.fixture(scope="module", autouse=True) def build_jar(): jar_file_path = os.path.join(SPARK_PATH, "code", "java", "hello-java-spark") # compile java file @@ -207,11 +207,11 @@ def configuration() -> list: def test_sagemaker_pyspark_v3( - spark_v3_py_processor, spark_v3_jar_processor, sagemaker_session, configuration, build_jar + spark_v3_py_processor, spark_v3_jar_processor, sagemaker_session, configuration ): test_sagemaker_pyspark_multinode(spark_v3_py_processor, sagemaker_session, configuration) test_sagemaker_java_jar_multinode( - spark_v3_jar_processor, sagemaker_session, configuration, build_jar + spark_v3_jar_processor, sagemaker_session, configuration ) @@ -281,7 +281,7 @@ def test_sagemaker_pyspark_multinode(spark_py_processor, sagemaker_session, conf def test_sagemaker_java_jar_multinode( - spark_jar_processor, sagemaker_session, configuration, build_jar + spark_jar_processor, sagemaker_session, configuration ): """Test SparkJarProcessor using Java application jar""" bucket = spark_jar_processor.sagemaker_session.default_bucket() From 6484fb04533050b7ed62e3e0de6817cb9260b29c Mon Sep 17 00:00:00 2001 From: Erick Benitez-Ramos Date: Wed, 26 Mar 2025 21:54:09 -0700 Subject: [PATCH 2/2] format --- tests/integ/test_spark_processing.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/integ/test_spark_processing.py b/tests/integ/test_spark_processing.py index 17e88ed3e0..eeba205b3b 100644 --- a/tests/integ/test_spark_processing.py +++ b/tests/integ/test_spark_processing.py @@ -210,9 +210,7 @@ def test_sagemaker_pyspark_v3( spark_v3_py_processor, spark_v3_jar_processor, sagemaker_session, configuration ): test_sagemaker_pyspark_multinode(spark_v3_py_processor, sagemaker_session, configuration) - test_sagemaker_java_jar_multinode( - spark_v3_jar_processor, sagemaker_session, configuration - ) + test_sagemaker_java_jar_multinode(spark_v3_jar_processor, sagemaker_session, configuration) def test_sagemaker_pyspark_multinode(spark_py_processor, sagemaker_session, configuration): @@ -280,9 +278,7 @@ def test_sagemaker_pyspark_multinode(spark_py_processor, sagemaker_session, conf assert len(output_contents) != 0 -def test_sagemaker_java_jar_multinode( - spark_jar_processor, sagemaker_session, configuration -): +def test_sagemaker_java_jar_multinode(spark_jar_processor, sagemaker_session, configuration): """Test SparkJarProcessor using Java application jar""" bucket = spark_jar_processor.sagemaker_session.default_bucket() with open(os.path.join(SPARK_PATH, "files", "data.jsonl")) as data: