Quickstart

from sigmaeval import SigmaEval, ScenarioTest, assertions
import asyncio
from typing import List, Dict, Any

# 1. Define the ScenarioTest to describe the desired behavior
scenario = (
    ScenarioTest("Simple Test")
    .given("A user interacting with a chatbot")
    .when("The user greets the bot")
    .expect_behavior(
        "The bot provides a simple and friendly greeting.",
        # We want to be confident that at least 75% of responses will score a 7/10 or higher.
        criteria=assertions.scores.proportion_gte(min_score=7, proportion=0.75)
    )
    .max_turns(1) # Only needed here since we're returning a static greeting
)
# 2. Implement the app_handler to allow SigmaEval to communicate with your app
async def app_handler(messages: List[Dict[str, str]], state: Any) -> str:
    # In a real test, you would pass messages to your app and return the response.
    # For this example, we'll return a static, friendly greeting.
    return "Hello there! Nice to meet you!"

# 3. Initialize SigmaEval and run the evaluation
async def main():
    # You can use any model that LiteLLM supports: https://docs.litellm.ai/docs/providers
    sigma_eval = SigmaEval(
        judge_model="gemini/gemini-2.5-flash",
        sample_size=20,  # The number of times to run the test
        significance_level=0.05  # Corresponds to a 95% confidence level
    )
    result = await sigma_eval.evaluate(scenario, app_handler)

    # Print the detailed summary to the console
    print(result)

    # Programmatically check the result
    if result.passed:
        print("✅ Scenario passed!")
    else:
        print("❌ Scenario failed.")

if __name__ == "__main__":
    asyncio.run(main())