From 1de6b6960ab940cde725cd88d2087e6b207f869f Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Fri, 21 Nov 2025 12:04:24 -0800 Subject: [PATCH 1/5] Added integration tests --- .../azure-ai/tests/test_azure_ai_client.py | 139 +++++++++++++++++- 1 file changed, 138 insertions(+), 1 deletion(-) diff --git a/python/packages/azure-ai/tests/test_azure_ai_client.py b/python/packages/azure-ai/tests/test_azure_ai_client.py index 9aaf0b3f77..c5bae1b9e1 100644 --- a/python/packages/azure-ai/tests/test_azure_ai_client.py +++ b/python/packages/azure-ai/tests/test_azure_ai_client.py @@ -1,9 +1,16 @@ # Copyright (c) Microsoft. All rights reserved. +import os +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Annotated from unittest.mock import AsyncMock, MagicMock, patch import pytest from agent_framework import ( + AgentRunResponse, + AgentRunResponseUpdate, + ChatAgent, ChatClientProtocol, ChatMessage, ChatOptions, @@ -11,16 +18,85 @@ TextContent, ) from agent_framework.exceptions import ServiceInitializationError +from azure.ai.projects.aio import AIProjectClient from azure.ai.projects.models import ( + PromptAgentDefinition, ResponseTextFormatConfigurationJsonSchema, ) +from azure.identity.aio import AzureCliCredential from openai.types.responses.parsed_response import ParsedResponse from openai.types.responses.response import Response as OpenAIResponse -from pydantic import BaseModel, ConfigDict, ValidationError +from pydantic import BaseModel, ConfigDict, Field, ValidationError from agent_framework_azure_ai import AzureAIClient, AzureAISettings +def _integration_test_skip_reason() -> str | None: + """Return a skip reason for Azure AI integration tests or None if all prerequisites are met. + + Rules: + - RUN_INTEGRATION_TESTS must be set to true. + - Endpoint and model env vars must be provided and not equal to placeholder endpoint. + """ + run_tests = os.getenv("RUN_INTEGRATION_TESTS", "false").lower() == "true" + if not run_tests: + return "Integration tests are disabled." + + endpoint = os.getenv("AZURE_AI_PROJECT_ENDPOINT", "") + model = os.getenv("AZURE_AI_MODEL_DEPLOYMENT_NAME", "") + placeholder_endpoint = "https://test-project.cognitiveservices.azure.com/" + missing_endpoint = endpoint in ("", placeholder_endpoint) + missing_model = model == "" + + if missing_endpoint and missing_model: + return "Azure AI project endpoint and model deployment name missing; skipping integration tests." + if missing_endpoint: + return "Azure AI project endpoint missing; skipping integration tests." + if missing_model: + return "Azure AI model deployment name missing; skipping integration tests." + return None + + +_skip_reason = _integration_test_skip_reason() + + +skip_if_azure_ai_integration_tests_disabled = pytest.mark.skipif( + _skip_reason is not None, + reason=_skip_reason or "", +) + + +@asynccontextmanager +async def temporary_chat_client(agent_name: str) -> AsyncIterator[AzureAIClient]: + """Async context manager that creates an Azure AI agent and yields an `AzureAIClient`. + + The underlying agent version is cleaned up automatically after use. + Tests can construct their own `ChatAgent` instances from the yielded client. + """ + endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] + model = os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"] + async with ( + AzureCliCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential) as project_client, + ): + azure_ai_agent = await project_client.agents.create_version( + agent_name=agent_name, + definition=PromptAgentDefinition(model=model), + ) + + chat_client = AzureAIClient( + project_client=project_client, + agent_name=azure_ai_agent.name, + agent_version=azure_ai_agent.version, + ) + try: + yield chat_client + finally: + await project_client.agents.delete_version( + agent_name=azure_ai_agent.name, agent_version=azure_ai_agent.version + ) + + def create_test_azure_ai_client( mock_project_client: MagicMock, agent_name: str | None = None, @@ -751,3 +827,64 @@ def mock_project_client() -> MagicMock: mock_client.close = AsyncMock() return mock_client + + +def get_weather( + location: Annotated[str, Field(description="The location to get the weather for.")], +) -> str: + """Get the weather for a given location.""" + return f"The weather in {location} is sunny with a high of 25°C." + + +@pytest.mark.flaky +@skip_if_azure_ai_integration_tests_disabled +async def test_azure_ai_chat_client_agent_basic_run() -> None: + """Test ChatAgent basic run functionality with AzureAIClient.""" + async with ( + temporary_chat_client(agent_name="BasicRunAgent") as chat_client, + ChatAgent(chat_client=chat_client) as agent, + ): + response = await agent.run("Hello! Please respond with 'Hello World' exactly.") + + # Validate response + assert isinstance(response, AgentRunResponse) + assert response.text is not None + assert len(response.text) > 0 + assert "Hello World" in response.text + + +@pytest.mark.flaky +@skip_if_azure_ai_integration_tests_disabled +async def test_azure_ai_chat_client_agent_basic_run_streaming() -> None: + """Test ChatAgent basic streaming functionality with AzureAIClient.""" + async with ( + temporary_chat_client(agent_name="BasicRunStreamingAgent") as chat_client, + ChatAgent(chat_client=chat_client) as agent, + ): + full_message: str = "" + async for chunk in agent.run_stream("Please respond with exactly: 'This is a streaming response test.'"): + assert chunk is not None + assert isinstance(chunk, AgentRunResponseUpdate) + if chunk.text: + full_message += chunk.text + + # Validate streaming response + assert len(full_message) > 0 + assert "streaming response test" in full_message.lower() + + +@pytest.mark.flaky +@skip_if_azure_ai_integration_tests_disabled +async def test_azure_ai_chat_client_agent_with_tools() -> None: + """Test ChatAgent tools with AzureAIClient.""" + async with ( + temporary_chat_client(agent_name="RunToolsAgent") as chat_client, + ChatAgent(chat_client=chat_client, tools=[get_weather]) as agent, + ): + response = await agent.run("What's the weather like in Seattle?") + + # Validate response + assert isinstance(response, AgentRunResponse) + assert response.text is not None + assert len(response.text) > 0 + assert any(word in response.text.lower() for word in ["sunny", "25"]) From b6799bd843c6ba5f1452b908851b48f7d038193d Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Fri, 21 Nov 2025 12:13:56 -0800 Subject: [PATCH 2/5] Update python/packages/azure-ai/tests/test_azure_ai_client.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../azure-ai/tests/test_azure_ai_client.py | 39 ++++--------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/python/packages/azure-ai/tests/test_azure_ai_client.py b/python/packages/azure-ai/tests/test_azure_ai_client.py index c5bae1b9e1..88bc9f06b1 100644 --- a/python/packages/azure-ai/tests/test_azure_ai_client.py +++ b/python/packages/azure-ai/tests/test_azure_ai_client.py @@ -31,38 +31,15 @@ from agent_framework_azure_ai import AzureAIClient, AzureAISettings -def _integration_test_skip_reason() -> str | None: - """Return a skip reason for Azure AI integration tests or None if all prerequisites are met. - - Rules: - - RUN_INTEGRATION_TESTS must be set to true. - - Endpoint and model env vars must be provided and not equal to placeholder endpoint. - """ - run_tests = os.getenv("RUN_INTEGRATION_TESTS", "false").lower() == "true" - if not run_tests: - return "Integration tests are disabled." - - endpoint = os.getenv("AZURE_AI_PROJECT_ENDPOINT", "") - model = os.getenv("AZURE_AI_MODEL_DEPLOYMENT_NAME", "") - placeholder_endpoint = "https://test-project.cognitiveservices.azure.com/" - missing_endpoint = endpoint in ("", placeholder_endpoint) - missing_model = model == "" - - if missing_endpoint and missing_model: - return "Azure AI project endpoint and model deployment name missing; skipping integration tests." - if missing_endpoint: - return "Azure AI project endpoint missing; skipping integration tests." - if missing_model: - return "Azure AI model deployment name missing; skipping integration tests." - return None - - -_skip_reason = _integration_test_skip_reason() - - skip_if_azure_ai_integration_tests_disabled = pytest.mark.skipif( - _skip_reason is not None, - reason=_skip_reason or "", + os.getenv("RUN_INTEGRATION_TESTS", "false").lower() != "true" + or os.getenv("AZURE_AI_PROJECT_ENDPOINT", "") in ("", "https://test-project.cognitiveservices.azure.com/") + or os.getenv("AZURE_AI_MODEL_DEPLOYMENT_NAME", "") == "", + reason=( + "No real AZURE_AI_PROJECT_ENDPOINT or AZURE_AI_MODEL_DEPLOYMENT_NAME provided; skipping integration tests." + if os.getenv("RUN_INTEGRATION_TESTS", "false").lower() == "true" + else "Integration tests are disabled." + ), ) From 87562df10c98e4da9a69b74ac8468c5dbd1e6950 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Fri, 21 Nov 2025 12:22:31 -0800 Subject: [PATCH 3/5] Small fixes in samples --- python/samples/getting_started/agents/azure_ai/README.md | 2 -- .../azure_ai/azure_ai_with_existing_conversation.py | 4 ++-- .../getting_started/agents/azure_ai_agent/README.md | 2 ++ .../azure_ai_with_search_context_agentic.py | 9 ++++----- .../azure_ai_with_search_context_semantic.py | 9 ++++----- 5 files changed, 12 insertions(+), 14 deletions(-) rename python/samples/getting_started/agents/{azure_ai => azure_ai_agent}/azure_ai_with_search_context_agentic.py (96%) rename python/samples/getting_started/agents/{azure_ai => azure_ai_agent}/azure_ai_with_search_context_semantic.py (94%) diff --git a/python/samples/getting_started/agents/azure_ai/README.md b/python/samples/getting_started/agents/azure_ai/README.md index 7a3b2d4520..8fd883b0cb 100644 --- a/python/samples/getting_started/agents/azure_ai/README.md +++ b/python/samples/getting_started/agents/azure_ai/README.md @@ -20,8 +20,6 @@ This folder contains examples demonstrating different ways to create and use age | [`azure_ai_with_file_search.py`](azure_ai_with_file_search.py) | Shows how to use the `HostedFileSearchTool` with Azure AI agents to upload files, create vector stores, and enable agents to search through uploaded documents to answer user questions. | | [`azure_ai_with_hosted_mcp.py`](azure_ai_with_hosted_mcp.py) | Shows how to integrate hosted Model Context Protocol (MCP) tools with Azure AI Agent. | | [`azure_ai_with_response_format.py`](azure_ai_with_response_format.py) | Shows how to use structured outputs (response format) with Azure AI agents using Pydantic models to enforce specific response schemas. | -| [`azure_ai_with_search_context_agentic.py`](azure_ai_with_search_context_agentic.py) | Shows how to use AzureAISearchContextProvider with agentic mode. Uses Knowledge Bases for multi-hop reasoning across documents with query planning. Recommended for most scenarios - slightly slower with more token consumption for query planning, but more accurate results. | -| [`azure_ai_with_search_context_semantic.py`](azure_ai_with_search_context_semantic.py) | Shows how to use AzureAISearchContextProvider with semantic mode. Fast hybrid search with vector + keyword search and semantic ranking for RAG. Best for simple queries where speed is critical. | | [`azure_ai_with_sharepoint.py`](azure_ai_with_sharepoint.py) | Shows how to use SharePoint grounding with Azure AI agents to search through SharePoint content and answer user questions with proper citations. Requires a SharePoint connection configured in your Azure AI project. | | [`azure_ai_with_thread.py`](azure_ai_with_thread.py) | Demonstrates thread management with Azure AI agents, including automatic thread creation for stateless conversations and explicit thread management for maintaining conversation context across multiple interactions. | | [`azure_ai_with_image_generation.py`](azure_ai_with_image_generation.py) | Shows how to use the `ImageGenTool` with Azure AI agents to generate images based on text prompts. | diff --git a/python/samples/getting_started/agents/azure_ai/azure_ai_with_existing_conversation.py b/python/samples/getting_started/agents/azure_ai/azure_ai_with_existing_conversation.py index a268b0db0e..43019d050c 100644 --- a/python/samples/getting_started/agents/azure_ai/azure_ai_with_existing_conversation.py +++ b/python/samples/getting_started/agents/azure_ai/azure_ai_with_existing_conversation.py @@ -32,7 +32,7 @@ async def example_with_client() -> None: AIProjectClient(endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"], credential=credential) as project_client, ): # Create a conversation using OpenAI client - openai_client = await project_client.get_openai_client() + openai_client = project_client.get_openai_client() conversation = await openai_client.conversations.create() conversation_id = conversation.id print(f"Conversation ID: {conversation_id}") @@ -70,7 +70,7 @@ async def example_with_thread() -> None: ) as agent, ): # Create a conversation using OpenAI client - openai_client = await project_client.get_openai_client() + openai_client = project_client.get_openai_client() conversation = await openai_client.conversations.create() conversation_id = conversation.id print(f"Conversation ID: {conversation_id}") diff --git a/python/samples/getting_started/agents/azure_ai_agent/README.md b/python/samples/getting_started/agents/azure_ai_agent/README.md index c7337bbe82..cbc43f1d70 100644 --- a/python/samples/getting_started/agents/azure_ai_agent/README.md +++ b/python/samples/getting_started/agents/azure_ai_agent/README.md @@ -20,6 +20,8 @@ This folder contains examples demonstrating different ways to create and use age | [`azure_ai_with_local_mcp.py`](azure_ai_with_local_mcp.py) | Shows how to integrate Azure AI agents with local Model Context Protocol (MCP) servers for enhanced functionality and tool integration. Demonstrates both agent-level and run-level tool configuration. | | [`azure_ai_with_multiple_tools.py`](azure_ai_with_multiple_tools.py) | Demonstrates how to use multiple tools together with Azure AI agents, including web search, MCP servers, and function tools. Shows coordinated multi-tool interactions and approval workflows. | | [`azure_ai_with_openapi_tools.py`](azure_ai_with_openapi_tools.py) | Demonstrates how to use OpenAPI tools with Azure AI agents to integrate external REST APIs. Shows OpenAPI specification loading, anonymous authentication, thread context management, and coordinated multi-API conversations using weather and countries APIs. | +| [`azure_ai_with_search_context_agentic.py`](azure_ai_with_search_context_agentic.py) | Shows how to use AzureAISearchContextProvider with agentic mode. Uses Knowledge Bases for multi-hop reasoning across documents with query planning. Recommended for most scenarios - slightly slower with more token consumption for query planning, but more accurate results. | +| [`azure_ai_with_search_context_semantic.py`](azure_ai_with_search_context_semantic.py) | Shows how to use AzureAISearchContextProvider with semantic mode. Fast hybrid search with vector + keyword search and semantic ranking for RAG. Best for simple queries where speed is critical. | | [`azure_ai_with_thread.py`](azure_ai_with_thread.py) | Demonstrates thread management with Azure AI agents, including automatic thread creation for stateless conversations and explicit thread management for maintaining conversation context across multiple interactions. | ## Environment Variables diff --git a/python/samples/getting_started/agents/azure_ai/azure_ai_with_search_context_agentic.py b/python/samples/getting_started/agents/azure_ai_agent/azure_ai_with_search_context_agentic.py similarity index 96% rename from python/samples/getting_started/agents/azure_ai/azure_ai_with_search_context_agentic.py rename to python/samples/getting_started/agents/azure_ai_agent/azure_ai_with_search_context_agentic.py index d8f6a9dc30..1bb81565fa 100644 --- a/python/samples/getting_started/agents/azure_ai/azure_ai_with_search_context_agentic.py +++ b/python/samples/getting_started/agents/azure_ai_agent/azure_ai_with_search_context_agentic.py @@ -3,12 +3,11 @@ import asyncio import os -from dotenv import load_dotenv - from agent_framework import ChatAgent from agent_framework_aisearch import AzureAISearchContextProvider from agent_framework_azure_ai import AzureAIAgentClient -from azure.identity.aio import DefaultAzureCredential +from azure.identity.aio import AzureCliCredential +from dotenv import load_dotenv # Load environment variables from .env file load_dotenv() @@ -68,7 +67,7 @@ async def main() -> None: endpoint=search_endpoint, index_name=index_name, api_key=search_key, # Use api_key for API key auth, or credential for managed identity - credential=DefaultAzureCredential() if not search_key else None, + credential=AzureCliCredential() if not search_key else None, mode="agentic", # Advanced mode for multi-hop reasoning # Agentic mode configuration azure_ai_project_endpoint=project_endpoint, @@ -87,7 +86,7 @@ async def main() -> None: AzureAIAgentClient( project_endpoint=project_endpoint, model_deployment_name=model_deployment, - async_credential=DefaultAzureCredential(), + async_credential=AzureCliCredential(), ) as client, ChatAgent( chat_client=client, diff --git a/python/samples/getting_started/agents/azure_ai/azure_ai_with_search_context_semantic.py b/python/samples/getting_started/agents/azure_ai_agent/azure_ai_with_search_context_semantic.py similarity index 94% rename from python/samples/getting_started/agents/azure_ai/azure_ai_with_search_context_semantic.py rename to python/samples/getting_started/agents/azure_ai_agent/azure_ai_with_search_context_semantic.py index 30504354f7..f94f358127 100644 --- a/python/samples/getting_started/agents/azure_ai/azure_ai_with_search_context_semantic.py +++ b/python/samples/getting_started/agents/azure_ai_agent/azure_ai_with_search_context_semantic.py @@ -3,12 +3,11 @@ import asyncio import os -from dotenv import load_dotenv - from agent_framework import ChatAgent from agent_framework_aisearch import AzureAISearchContextProvider from agent_framework_azure_ai import AzureAIAgentClient -from azure.identity.aio import DefaultAzureCredential +from azure.identity.aio import AzureCliCredential +from dotenv import load_dotenv # Load environment variables from .env file load_dotenv() @@ -58,7 +57,7 @@ async def main() -> None: endpoint=search_endpoint, index_name=index_name, api_key=search_key, # Use api_key for API key auth, or credential for managed identity - credential=DefaultAzureCredential() if not search_key else None, + credential=AzureCliCredential() if not search_key else None, mode="semantic", # Default mode top_k=3, # Retrieve top 3 most relevant documents ) @@ -69,7 +68,7 @@ async def main() -> None: AzureAIAgentClient( project_endpoint=project_endpoint, model_deployment_name=model_deployment, - async_credential=DefaultAzureCredential(), + async_credential=AzureCliCredential(), ) as client, ChatAgent( chat_client=client, From 28c26eacdcfaf8191ef1dafc45562757bd8a5e9f Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Fri, 21 Nov 2025 12:28:42 -0800 Subject: [PATCH 4/5] Small fix --- python/packages/azure-ai/tests/test_azure_ai_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/packages/azure-ai/tests/test_azure_ai_client.py b/python/packages/azure-ai/tests/test_azure_ai_client.py index 88bc9f06b1..7930bd5273 100644 --- a/python/packages/azure-ai/tests/test_azure_ai_client.py +++ b/python/packages/azure-ai/tests/test_azure_ai_client.py @@ -30,7 +30,6 @@ from agent_framework_azure_ai import AzureAIClient, AzureAISettings - skip_if_azure_ai_integration_tests_disabled = pytest.mark.skipif( os.getenv("RUN_INTEGRATION_TESTS", "false").lower() != "true" or os.getenv("AZURE_AI_PROJECT_ENDPOINT", "") in ("", "https://test-project.cognitiveservices.azure.com/") From ba3cf905a01de56a8cbf6b7c56f1dc335140586d Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Fri, 21 Nov 2025 14:39:43 -0800 Subject: [PATCH 5/5] Small fix --- .../azure-ai/tests/test_azure_ai_client.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/python/packages/azure-ai/tests/test_azure_ai_client.py b/python/packages/azure-ai/tests/test_azure_ai_client.py index 7930bd5273..2dfeb5524b 100644 --- a/python/packages/azure-ai/tests/test_azure_ai_client.py +++ b/python/packages/azure-ai/tests/test_azure_ai_client.py @@ -20,7 +20,6 @@ from agent_framework.exceptions import ServiceInitializationError from azure.ai.projects.aio import AIProjectClient from azure.ai.projects.models import ( - PromptAgentDefinition, ResponseTextFormatConfigurationJsonSchema, ) from azure.identity.aio import AzureCliCredential @@ -50,27 +49,18 @@ async def temporary_chat_client(agent_name: str) -> AsyncIterator[AzureAIClient] Tests can construct their own `ChatAgent` instances from the yielded client. """ endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] - model = os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"] async with ( AzureCliCredential() as credential, AIProjectClient(endpoint=endpoint, credential=credential) as project_client, ): - azure_ai_agent = await project_client.agents.create_version( - agent_name=agent_name, - definition=PromptAgentDefinition(model=model), - ) - chat_client = AzureAIClient( project_client=project_client, - agent_name=azure_ai_agent.name, - agent_version=azure_ai_agent.version, + agent_name=agent_name, ) try: yield chat_client finally: - await project_client.agents.delete_version( - agent_name=azure_ai_agent.name, agent_version=azure_ai_agent.version - ) + await project_client.agents.delete(agent_name=agent_name) def create_test_azure_ai_client(