From 41b319e2a11943b5d516c9e822dc30c3e549b6d9 Mon Sep 17 00:00:00 2001
From: zyysurely <yingyingzhao@microsoft.com>
Date: Wed, 13 May 2026 11:22:43 -0700
Subject: [PATCH] [fix] infinite eval results when outbound limited

---
 .../ai/evaluation/_evaluate/_evaluate.py      |  9 +++
 .../tests/unittests/test_evaluate.py          | 68 +++++++++++++++++++
 2 files changed, 77 insertions(+)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
index cac9e526af3a..f3af3727ea13 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
@@ -1331,6 +1331,7 @@ def emit_eval_result_events_to_app_insights(
         LOGGER.debug("No results to log to App Insights")
         return
 
+    logger_provider = None
     try:
         # Configure OpenTelemetry logging with anonymized Resource attributes
 
@@ -1398,6 +1399,14 @@ def emit_eval_result_events_to_app_insights(
 
     except Exception as e:
         LOGGER.error(f"Failed to emit evaluation results to App Insights: {e}")
+    finally:
+        # Shut down the logger provider to stop background threads (e.g. OneSettings
+        # configuration poller) that would otherwise keep the process alive indefinitely.
+        if logger_provider is not None:
+            try:
+                logger_provider.shutdown()
+            except Exception:
+                pass
 
 
 def _preprocess_data(
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
index ee9eee566439..9564cf427a4f 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
@@ -48,6 +48,7 @@
     _adjust_for_inverse_metric,
     _is_inverse_metric,
     _create_result_object,
+    emit_eval_result_events_to_app_insights,
 )
 from azure.ai.evaluation._evaluate._utils import _convert_name_map_into_property_entries
 from azure.ai.evaluation._evaluate._utils import _apply_column_mapping, _trace_destination_from_project_scope
@@ -2433,3 +2434,70 @@ def test_non_inverse_metric_preserves_values(self):
         assert result["label"] == "pass"
         assert result["passed"] is True
         assert result["score"] == 4
+
+
+@pytest.mark.unittest
+@pytest.mark.skipif(MISSING_OPENTELEMETRY, reason="This test requires the opentelemetry package")
+class TestEmitEvalResultShutdown:
+    """Tests that emit_eval_result_events_to_app_insights shuts down the LoggerProvider."""
+
+    @patch("azure.monitor.opentelemetry.exporter.AzureMonitorLogExporter")
+    @patch("opentelemetry.sdk._logs.LoggerProvider")
+    def test_shutdown_called_on_success(self, mock_lp_cls, mock_exporter_cls):
+        """logger_provider.shutdown() must be called after successful export."""
+        mock_lp = mock_lp_cls.return_value
+        mock_lp.force_flush.return_value = True
+        results = [
+            {
+                "results": [{"metric": "coherence", "score": 4.5}],
+                "datasource_item": {},
+            }
+        ]
+        config = {"connection_string": "InstrumentationKey=fake-key"}
+
+        emit_eval_result_events_to_app_insights(config, results)
+        mock_lp.shutdown.assert_called_once()
+
+    @patch("azure.monitor.opentelemetry.exporter.AzureMonitorLogExporter")
+    @patch("opentelemetry.sdk._logs.LoggerProvider")
+    def test_shutdown_called_on_exception(self, mock_lp_cls, mock_exporter_cls):
+        """logger_provider.shutdown() must be called even when an exception occurs."""
+        mock_lp = mock_lp_cls.return_value
+        mock_lp.force_flush.side_effect = RuntimeError("boom")
+        results = [
+            {
+                "results": [{"metric": "coherence", "score": 4.5}],
+                "datasource_item": {},
+            }
+        ]
+        config = {"connection_string": "InstrumentationKey=fake-key"}
+
+        # Should not raise — the exception is caught internally
+        emit_eval_result_events_to_app_insights(config, results)
+        mock_lp.shutdown.assert_called_once()
+
+    @patch("azure.monitor.opentelemetry.exporter.AzureMonitorLogExporter")
+    @patch("opentelemetry.sdk._logs.LoggerProvider")
+    def test_shutdown_called_on_flush_timeout(self, mock_lp_cls, mock_exporter_cls):
+        """logger_provider.shutdown() must be called even when flush times out."""
+        mock_lp = mock_lp_cls.return_value
+        mock_lp.force_flush.return_value = False  # Simulates timeout
+        results = [
+            {
+                "results": [{"metric": "coherence", "score": 4.5}],
+                "datasource_item": {},
+            }
+        ]
+        config = {"connection_string": "InstrumentationKey=fake-key"}
+
+        emit_eval_result_events_to_app_insights(config, results)
+        mock_lp.shutdown.assert_called_once()
+
+    @patch("azure.monitor.opentelemetry.exporter.AzureMonitorLogExporter")
+    @patch("opentelemetry.sdk._logs.LoggerProvider")
+    def test_no_shutdown_when_results_empty(self, mock_lp_cls, mock_exporter_cls):
+        """When results list is empty, no LoggerProvider is created so no shutdown."""
+        config = {"connection_string": "InstrumentationKey=fake-key"}
+
+        emit_eval_result_events_to_app_insights(config, [])
+        mock_lp_cls.assert_not_called()