From 41dd877f2ea2e4de6de0d5104c52ce20293794b6 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 31 Aug 2026 04:08:50 +0300 Subject: [PATCH 1/2] fix(evaluation): make App Insights export best effort Keep managed-identity evaluation results when Application Insights export fails or times out, while preserving strict credential validation and failure logs. Authored-by: GitHub Copilot for VS Code Model: GitHub Copilot (copilot) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure-ai-evaluation/CHANGELOG.md | 6 ++ .../ai/evaluation/_evaluate/_evaluate.py | 11 +-- .../tests/unittests/test_evaluate.py | 78 ++++++++++--------- 3 files changed, 53 insertions(+), 42 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 0df1a42ccb98..e90b62b16205 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -1,5 +1,11 @@ # Release History +## 1.18.5 (Unreleased) + +### Bugs Fixed + +- Made Application Insights export failures best-effort for evaluations using project managed identity authentication. + ## 1.18.4 (2026-08-27) ### Bugs Fixed diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py index 28be6d5a718d..76cbb33098a6 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py @@ -1463,22 +1463,19 @@ def emit_eval_result_events_to_app_insights( # Force flush to ensure events are sent, with a timeout to prevent hanging flush_timeout_millis = 60000 # 60 seconds flush_success = logger_provider.force_flush(timeout_millis=flush_timeout_millis) - if export_result_tracker is not None and export_result_tracker.export_failed: - raise RuntimeError("Failed to export evaluation results to App Insights.") + export_failed = export_result_tracker is not None and export_result_tracker.export_failed + if export_failed: + LOGGER.error("Failed to export evaluation results to App Insights.") if not flush_success: timeout_message = ( f"App Insights force_flush timed out after {flush_timeout_millis}ms. " "Some evaluation events may not have been sent." ) - if use_entra_authentication: - raise TimeoutError(timeout_message) LOGGER.warning(timeout_message) - else: + elif not export_failed: LOGGER.info(f"Successfully logged {len(results)} evaluation results to App Insights") except Exception as ex: - if use_entra_authentication: - raise LOGGER.error("Failed to emit evaluation results to App Insights: %s", ex) finally: # Shut down the logger provider to stop background threads (e.g. OneSettings diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index edb93f9176f4..f7b5430971fe 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -3632,7 +3632,7 @@ class TestAppInsightsAuthentication: ] @patch("opentelemetry.sdk._logs.LoggerProvider") - def test_project_managed_identity_credential_and_scope_are_passed_to_exporter(self, mock_lp_cls): + def test_project_managed_identity_credential_and_scope_are_passed_to_exporter(self, mock_lp_cls, caplog): mock_lp_cls.return_value.force_flush.return_value = True credential = MagicMock(spec=TokenCredential) exporter_module = MagicMock() @@ -3642,7 +3642,9 @@ def test_project_managed_identity_credential_and_scope_are_passed_to_exporter(se "credential": credential, } - with patch.dict("sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module}): + with caplog.at_level(logging.INFO), patch.dict( + "sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module} + ): emit_eval_result_events_to_app_insights(config, self._RESULTS) exporter_options = exporter_module.AzureMonitorLogExporter.call_args.kwargs @@ -3650,6 +3652,7 @@ def test_project_managed_identity_credential_and_scope_are_passed_to_exporter(se assert exporter_options["credential_scopes"] == ["https://monitor.azure.com/.default"] assert exporter_options["credential"] is not credential assert "token" not in config + assert "Successfully logged 1 evaluation results to App Insights" in caplog.text def test_exporter_credential_refresh_uses_monitor_scope(self): credential = MagicMock(spec=TokenCredential) @@ -3682,26 +3685,28 @@ def test_exporter_credential_refresh_uses_monitor_scope(self): ] @patch("opentelemetry.sdk._logs.LoggerProvider") - def test_project_managed_identity_exporter_failure_is_surfaced(self, mock_lp_cls): + def test_project_managed_identity_exporter_failure_is_logged(self, mock_lp_cls, caplog): credential = MagicMock(spec=TokenCredential) exporter_module = MagicMock() exporter_module.AzureMonitorLogExporter.side_effect = RuntimeError("authentication failed") - with patch.dict("sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module}): - with pytest.raises(RuntimeError, match="authentication failed"): - emit_eval_result_events_to_app_insights( - { - "connection_string": "InstrumentationKey=fake-key", - "credential_type": "ProjectManagedIdentity", - "credential": credential, - }, - self._RESULTS, - ) + with caplog.at_level(logging.ERROR), patch.dict( + "sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module} + ): + emit_eval_result_events_to_app_insights( + { + "connection_string": "InstrumentationKey=fake-key", + "credential_type": "ProjectManagedIdentity", + "credential": credential, + }, + self._RESULTS, + ) + assert "Failed to emit evaluation results to App Insights: authentication failed" in caplog.text mock_lp_cls.return_value.shutdown.assert_called_once() @patch("opentelemetry.sdk._logs.LoggerProvider") - def test_project_managed_identity_batch_export_failure_is_surfaced(self, mock_lp_cls): + def test_project_managed_identity_batch_export_failure_is_logged(self, mock_lp_cls, caplog): from opentelemetry.sdk._logs.export import LogExportResult mock_lp_cls.return_value.force_flush.return_value = True @@ -3718,38 +3723,41 @@ def create_processor(tracked_exporter, **_kwargs): with patch.dict("sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module}), patch( "opentelemetry.sdk._logs.export.BatchLogRecordProcessor", side_effect=create_processor, - ): - with pytest.raises(RuntimeError, match="Failed to export evaluation results"): - emit_eval_result_events_to_app_insights( - { - "connection_string": "InstrumentationKey=fake-key", - "credential_type": "ProjectManagedIdentity", - "credential": credential, - }, - self._RESULTS, - ) + ), caplog.at_level(logging.ERROR): + emit_eval_result_events_to_app_insights( + { + "connection_string": "InstrumentationKey=fake-key", + "credential_type": "ProjectManagedIdentity", + "credential": credential, + }, + self._RESULTS, + ) + assert "Failed to export evaluation results to App Insights." in caplog.text + assert "Successfully logged" not in caplog.text exporter.export.assert_called_once_with([]) mock_lp_cls.return_value.force_flush.assert_called_once() mock_lp_cls.return_value.shutdown.assert_called_once() @patch("opentelemetry.sdk._logs.LoggerProvider") - def test_project_managed_identity_flush_timeout_is_surfaced(self, mock_lp_cls): + def test_project_managed_identity_flush_timeout_is_logged(self, mock_lp_cls, caplog): mock_lp_cls.return_value.force_flush.return_value = False credential = MagicMock(spec=TokenCredential) exporter_module = MagicMock() - with patch.dict("sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module}): - with pytest.raises(TimeoutError, match="force_flush timed out"): - emit_eval_result_events_to_app_insights( - { - "connection_string": "InstrumentationKey=fake-key", - "credential_type": "ProjectManagedIdentity", - "credential": credential, - }, - self._RESULTS, - ) + with caplog.at_level(logging.WARNING), patch.dict( + "sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module} + ): + emit_eval_result_events_to_app_insights( + { + "connection_string": "InstrumentationKey=fake-key", + "credential_type": "ProjectManagedIdentity", + "credential": credential, + }, + self._RESULTS, + ) + assert "App Insights force_flush timed out after 60000ms" in caplog.text mock_lp_cls.return_value.shutdown.assert_called_once() @patch("azure.identity.DefaultAzureCredential") From e3109b79dfbaae249d40650a24d1c0026d423776 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 31 Aug 2026 22:54:33 +0300 Subject: [PATCH 2/2] fix(evaluation): bump version to 1.18.5 to match the Unreleased changelog entry Verify ChangeLogEntries failed: _version.py still read 1.18.4, so the validator picked up the 1.18.4 entry and rejected its 2026-08-27 date for no longer being the latest in the file, now that a 1.18.5 (Unreleased) section sits above it. _version.py represents the upcoming version and has to match the Unreleased heading. This is the repository convention: of the packages currently carrying an (Unreleased) top entry, 71 have a matching _version.py and 2 do not. Authored-by: GitHub Copilot for VS Code Model: Claude Opus 5 (claude-opus-5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure-ai-evaluation/azure/ai/evaluation/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_version.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_version.py index e2832ae0ea3b..39b38ed3a13a 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_version.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_version.py @@ -3,4 +3,4 @@ # --------------------------------------------------------- # represents upcoming version -VERSION = "1.18.4" +VERSION = "1.18.5"