Tutorial: Testing Conversational AI with pytest and SigmaEval

import asyncio
from typing import List, Dict, Any
from litellm import acompletion

# This is the application we want to test.
# It's a simple chatbot that uses a system prompt to define its behavior.
async def app_handler(messages: List[Dict[str, str]], state: Any) -> str:
    system_prompt = {
        "role": "system",
        "content": "You are a helpful assistant for an e-commerce store. Your main functions are: tracking orders, initiating returns, answering product questions, and escalating to a human agent. Keep your answers concise."
    }
    
    # In a real app, you would manage conversation history.
    # For this simple example, we'll just prepend the system prompt to the latest user message.
    messages_with_system_prompt = [system_prompt, messages[-1]]

    # Use LiteLLM's async completion function to call the model
    response = await acompletion(
        model="gemini/gemini-2.5-flash", # Or any other model
        messages=messages_with_system_prompt
    )
    
    return response.choices[0].message.content

import pytest
from sigmaeval import (
    SigmaEval,
    ScenarioTest,
    assertions,
    metrics,
)
# Import the application handler we want to test
from app import app_handler

@pytest.mark.asyncio
async def test_bot_explains_its_capabilities():
    """
    This test evaluates if the bot correctly and efficiently explains its capabilities.
    """
    # 1. Initialize SigmaEval
    sigma_eval = SigmaEval(
        judge_model="gemini/gemini-2.5-flash",
        sample_size=10,  # A smaller sample size for a quick tutorial run
        significance_level=0.05
    )
    
    # 2. Define the ScenarioTest
    # This scenario tests two things:
    # - The bot provides the correct information (.expect_behavior)
    # - The bot responds quickly enough (.expect_metric)
    scenario = (
        ScenarioTest("Bot explains its capabilities")
        .given("A new user who has not interacted with the bot before")
        .when("The user asks a general question about the bot's capabilities, like 'what can you do?'")
        .expect_behavior(
            "Bot lists its main functions: tracking orders, initiating returns, answering product questions, and escalating to a human agent.",
            criteria=assertions.scores.proportion_gte(min_score=7, proportion=0.90)
        )
        .expect_metric(
            metrics.per_turn.response_latency,
            # We want to be confident that at least 90% of responses are faster than 2 seconds.
            criteria=assertions.metrics.proportion_lt(threshold=2.0, proportion=0.90)
        )
    )
    
    # 3. Run the evaluation
    result = await sigma_eval.evaluate(scenario, app_handler)
    
    # 4. Print the detailed summary for logs and assert the result
    print(result)
    assert result.passed, "The bot capabilities scenario failed."

pytest