Integration Testing Framework (#228)

* basic integration utility

* Integration test suite factory implementation

* Implementing core models for integration testing

* AutoClient mockup

* Adding runner starter code

* Adding foundational classes

* Drafting AgentClient and ResponseClient implementations

* Spec test

* Filling in more implementation details

* More files

* Cleaning up implementations

* Adding expect replies sending method

* Beginning unit tests

* Adding integration decor from sample test cases

* Integration from service url tests

* _handle_conversation implementation for response_client

* AgentClient tests completed

* Creating response client tests

* Hosting server for response client

* Response client tests completed

* Beginning refactor of aiohttp runner

* Unit test updates

* Fixing issues in refactor

* Fixed TestIntegrationFromSample unit test

* Another commit

* Reorganizing files

* Completed TestIntegrationFromServiceUrl unit tests

* Reformatting with black

* Quickstart integration test beginning

* Draft of quickstart sample setup

* Environment config connection

* Renaming messaging endpoint to agent url

* Removing unnecessary files

* Fixing agent client payload sending

* First successful integration test

* Beginning foundational test cases

* TypingIndicator test

* Adding more test cases

* More foundational integration test cases

* Reorganizing testing tools into package

* Polished the testing framework package

* Adding verbose logging for results with benchmark tool

* General cleanup

* Adding README

* Revising README

* Formatting

* Addressing PR comments

* Removing unused import

* Update dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update dev/microsoft-agents-testing/pyproject.toml

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Removing unused code

* Writing draft README for testing package

* Second pass for README.md

* Removing link to PyPI

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: rodrigobr-msft

dev/README.md

@@ -1,6 +1,25 @@
-This directory contains tools to aid the developers of the Microsoft 365 Agents SDK for Python.
+# Development Tools
 
-### `benchmark`
+Development utilities for the Microsoft Agents for Python project.
 
-This folder contains benchmarking utilities built in Python to send concurrent requests
-to an agent.
+## Contents
+
+- **[`install.sh`](install.sh)** - Installs testing framework in editable mode
+- **[`benchmark/`](benchmark/)** - Performance testing and stress testing tools  
+- **[`microsoft-agents-testing/`](microsoft-agents-testing/)** - Testing framework package
+
+## Quick Setup
+
+```bash
+./install.sh
+```
+
+## Benchmarking
+
+Performance testing tools with support for concurrent workers and authentication. Requires a running agent instance and Azure Bot Service credentials.
+
+See [benchmark/README.md](benchmark/README.md) for setup and usage details.
+
+## Testing Framework
+
+Provides testing utilities and helpers for Microsoft Agents development. Installed in editable mode for active development.

dev/benchmark/__init__.py

@@ -15,5 +15,6 @@
     "id": "user-id-0",
     "aadObjectId": "00000000-0000-0000-0000-0000000000020"
   },
-  "type": "message"
+  "type": "message",
+  "text": "Hello, Bot!"
 }

dev/benchmark/payload.json

@@ -8,6 +8,7 @@
 from .executor import Executor, CoroutineExecutor, ThreadExecutor
 from .aggregated_results import AggregatedResults
 from .config import BenchmarkConfig
+from .output import output_results
 
 LOG_FORMAT = "%(asctime)s: %(message)s"
 logging.basicConfig(format=LOG_FORMAT, level=logging.INFO, datefmt="%H:%M:%S")
@@ -20,13 +21,14 @@
     "--payload_path", "-p", default="./payload.json", help="Path to the payload file."
 )
 @click.option("--num_workers", "-n", default=1, help="Number of workers to use.")
+@click.option("--verbose", "-v", is_flag=True, help="Enable verbose logging.")
 @click.option(
     "--async_mode",
     "-a",
     is_flag=True,
     help="Run coroutine workers rather than thread workers.",
 )
-def main(payload_path: str, num_workers: int, async_mode: bool):
+def main(payload_path: str, num_workers: int, verbose: bool, async_mode: bool):
     """Main function to run the benchmark."""
 
     with open(payload_path, "r", encoding="utf-8") as f:
@@ -39,6 +41,8 @@ def main(payload_path: str, num_workers: int, async_mode: bool):
     start_time = datetime.now(timezone.utc).timestamp()
     results = executor.run(func, num_workers=num_workers)
     end_time = datetime.now(timezone.utc).timestamp()
+    if verbose:
+        output_results(results)
 
     agg = AggregatedResults(results)
     agg.display(start_time, end_time)

dev/benchmark/src/main.py

@@ -0,0 +1,12 @@
+from .executor import ExecutionResult
+
+
+def output_results(results: list[ExecutionResult]) -> None:
+    """Output the results of the benchmark to the console."""
+
+    for result in results:
+        status = "Success" if result.success else "Failure"
+        print(
+            f"Execution ID: {result.exe_id}, Duration: {result.duration:.4f} seconds, Status: {status}"
+        )
+        print(result.result)

dev/benchmark/src/output.py

@@ -0,0 +1 @@
+pip install -e ./microsoft-agents-testing/ --config-settings editable_mode=compat

dev/install.sh

@@ -0,0 +1,171 @@
+# Microsoft 365 Agents SDK for Python - Testing Framework
+
+A comprehensive testing framework designed specifically for Microsoft 365 Agents SDK, providing essential utilities and abstractions to streamline integration testing, authentication, and end-to-end agent validation.
+
+## Why This Package Exists
+
+Building and testing conversational agents presents unique challenges that standard testing frameworks don't address. This package eliminates these pain points by providing useful abstractions specifically designed for agent testing scenarios.
+
+## Key Features
+
+### 🔐 Authentication Utilities
+- **OAuth2 Token Generation**: Generate access tokens using client credentials flow
+- **Configuration-Based Auth**: Load credentials from environment variables or config objects
+- **MSAL Integration**: Built-in support for Microsoft Authentication Library
+
+```python
+from microsoft_agents.testing import generate_token, generate_token_from_config
+
+# Generate token directly
+token = generate_token(
+    app_id="your-app-id",
+    app_secret="your-secret",
+    tenant_id="your-tenant"
+)
+
+# Or from SDK config
+token = generate_token_from_config(sdk_config)
+```
+
+### 🧪 Integration Test Framework
+- **Pytest Fixtures**: Pre-built fixtures for common test scenarios
+- **Environment Abstraction**: Reusable environment setup for different hosting configurations
+- **Sample Management**: Base classes for organizing test samples and configurations
+- **Application Runners**: Abstract server lifecycle management for integration tests
+
+```python
+from microsoft_agents.testing import Integration, Environment, Sample
+
+class MyAgentTests(Integration):
+    _sample_cls = MyAgentSample
+    _environment_cls = AiohttpEnvironment
+    
+    @pytest.mark.asyncio
+    async def test_conversation_flow(self, agent_client, sample):
+        # Client and sample are automatically set up via fixtures
+        response = await agent_client.send_activity("Hello")
+        assert response is not None
+```
+
+### 🤖 Agent Communication Clients
+- **AgentClient**: High-level client for sending Activities to agents
+- **ResponseClient**: Handle responses from agent services
+- **Automatic Token Management**: Clients handle authentication automatically
+- **Delivery Mode Support**: Test both standard and `ExpectReplies` delivery patterns
+
+```python
+from microsoft_agents.testing import AgentClient
+
+client = AgentClient(
+    agent_url="http://localhost:3978",
+    cid="conversation-id",
+    client_id="your-client-id",
+    tenant_id="your-tenant-id",
+    client_secret="your-secret"
+)
+
+# Send simple text message
+response = await client.send_activity("What's the weather?")
+
+# Send Activity with ExpectReplies
+replies = await client.send_expect_replies(
+    Activity(type=ActivityTypes.message, text="Hello")
+)
+```
+
+### 🛠️ Testing Utilities
+- **Activity Population**: Automatically fill default Activity properties for testing
+- **URL Parsing**: Extract host and port from service URLs
+- **Configuration Management**: Centralized SDK configuration for tests
+
+```python
+from microsoft_agents.testing import populate_activity, get_host_and_port
+
+# Populate test activity with defaults
+activity = populate_activity(
+    Activity(text="Hello"),
+    defaults={"service_url": "http://localhost", "channel_id": "test"}
+)
+
+# Parse service URLs
+host, port = get_host_and_port("http://localhost:3978/api/messages")
+```
+
+## Who Should Use This Package
+
+- **Agent Developers**: Testing agents built with `microsoft-agents-hosting-core` and related packages
+- **QA Engineers**: Writing integration and E2E tests for conversational AI systems
+- **DevOps Teams**: Automating agent validation in CI/CD pipelines
+- **Sample Authors**: Creating reproducible examples and documentation
+
+## Integration with CI/CD
+
+This package is designed for seamless integration into continuous integration pipelines:
+
+```yaml
+# Example: GitHub Actions
+- name: Run Agent Integration Tests
+  run: |
+    pip install microsoft-agents-testing pytest pytest-asyncio
+    pytest tests/integration/ -v
+  env:
+    CLIENT_ID: ${{ secrets.AGENT_CLIENT_ID }}
+    CLIENT_SECRET: ${{ secrets.AGENT_CLIENT_SECRET }}
+    TENANT_ID: ${{ secrets.TENANT_ID }}
+```
+
+## Quick Start Example
+
+```python
+import pytest
+from microsoft_agents.testing import Integration, AiohttpEnvironment, Sample
+from microsoft_agents.activity import Activity
+
+class MyAgentSample(Sample):
+    async def init_app(self):
+        # Initialize your agent application
+        self.app = create_my_agent_app(self.env)
+    
+    @classmethod
+    async def get_config(cls):
+        return {"service_url": "http://localhost:3978"}
+
+class TestMyAgent(Integration):
+    _sample_cls = MyAgentSample
+    _environment_cls = AiohttpEnvironment
+    
+    _agent_url = "http://localhost:3978"
+    _cid = "test-conversation"
+    
+    @pytest.mark.asyncio
+    async def test_greeting(self, agent_client):
+        response = await agent_client.send_activity("Hello")
+        assert "Hi there" in response
+    
+    @pytest.mark.asyncio
+    async def test_conversation(self, agent_client):
+        replies = await agent_client.send_expect_replies("What can you do?")
+        assert len(replies) > 0
+        assert replies[0].type == "message"
+```
+
+## Related Packages
+
+This package complements the Microsoft 365 Agents SDK ecosystem:
+
+- `microsoft-agents-activity`: Activity types and protocols
+- `microsoft-agents-hosting-core`: Core hosting framework
+- `microsoft-agents-hosting-aiohttp`: aiohttp hosting integration
+- `microsoft-agents-authentication-msal`: MSAL authentication
+
+## Contributing
+
+This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA). For details, visit [https://cla.opensource.microsoft.com](https://cla.opensource.microsoft.com).
+
+## License
+
+MIT
+
+## Support
+
+For issues, questions, or contributions, please visit the [GitHub repository](https://github.com/microsoft/Agents-for-python).

dev/microsoft-agents-testing/README.md

@@ -0,0 +1,30 @@
+from .sdk_config import SDKConfig
+
+from .auth import generate_token, generate_token_from_config
+
+from .utils import populate_activity, get_host_and_port
+
+from .integration import (
+    Sample,
+    Environment,
+    ApplicationRunner,
+    AgentClient,
+    ResponseClient,
+    AiohttpEnvironment,
+    Integration,
+)
+
+__all__ = [
+    "SDKConfig",
+    "generate_token",
+    "generate_token_from_config",
+    "Sample",
+    "Environment",
+    "ApplicationRunner",
+    "AgentClient",
+    "ResponseClient",
+    "AiohttpEnvironment",
+    "Integration",
+    "populate_activity",
+    "get_host_and_port",
+]

dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py

@@ -0,0 +1,3 @@
+from .generate_token import generate_token, generate_token_from_config
+
+__all__ = ["generate_token", "generate_token_from_config"]

dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py

@@ -0,0 +1,51 @@
+import requests
+
+from microsoft_agents.hosting.core import AgentAuthConfiguration
+from microsoft_agents.testing.sdk_config import SDKConfig
+
+
+def generate_token(app_id: str, app_secret: str, tenant_id: str) -> str:
+    """Generate a token using the provided app credentials.
+
+    :param app_id: Application (client) ID.
+    :param app_secret: Application client secret.
+    :param tenant_id: Directory (tenant) ID.
+    :return: Generated access token as a string.
+    """
+
+    authority_endpoint = (
+        f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"
+    )
+
+    res = requests.post(
+        authority_endpoint,
+        headers={
+            "Content-Type": "application/x-www-form-urlencoded",
+        },
+        data={
+            "grant_type": "client_credentials",
+            "client_id": app_id,
+            "client_secret": app_secret,
+            "scope": f"{app_id}/.default",
+        },
+        timeout=10,
+    )
+    return res.json().get("access_token")
+
+
+def generate_token_from_config(sdk_config: SDKConfig) -> str:
+    """Generates a token using a provided config object.
+
+    :param sdk_config: Configuration dictionary containing connection settings.
+    :return: Generated access token as a string.
+    """
+
+    settings: AgentAuthConfiguration = sdk_config.get_connection()
+
+    app_id = settings.CLIENT_ID
+    app_secret = settings.CLIENT_SECRET
+    tenant_id = settings.TENANT_ID
+
+    if not app_id or not app_secret or not tenant_id:
+        raise ValueError("Incorrect configuration provided for token generation.")
+    return generate_token(app_id, app_secret, tenant_id)