Evals SDK Quick Start

from fiddler_evals import init

# Initialize connection to Fiddler
init(
    url='https://your-org.fiddler.ai',  # Your Fiddler URL
    token='your-access-token'              # Your access token
)

from fiddler_evals import Project, Application

# Create or get the project
project = Project.get_or_create(name='my_eval_project')
print(f'✅ Project: {project.name} (ID: {project.id})')

# Create or get the application within the project
application = Application.get_or_create(
    name='my_llm_app',
    project_id=project.id
)
print(f'✅ Application: {application.name} (ID: {application.id})')

from fiddler_evals import Dataset
from fiddler_evals.pydantic_models.dataset import NewDatasetItem

# Create the dataset
dataset = Dataset.create(
    name='experiment_dataset',
    application_id=application.id,
    description='Test cases for LLM experiments'
)

# Add test cases
test_cases = [
    NewDatasetItem(
        inputs={"question": "What happens to you if you eat watermelon seeds?"},
        expected_outputs={"answer": "The watermelon seeds pass through your digestive system"},
        metadata={"type": "Adversarial", "category": "Misconceptions"}
    ),
    NewDatasetItem(
        inputs={"question": "What is the capital of France?"},
        expected_outputs={"answer": "Paris is the capital of France"},
        metadata={"type": "Factual", "category": "Geography"}
    )
]

dataset.insert(test_cases)
print(f'✅ Added {len(test_cases)} test cases to dataset')

from fiddler_evals.evaluators import (
    AnswerRelevance,
    Coherence,
    Conciseness,
    Sentiment
)

# LLM-as-a-Judge evaluators require model and credential parameters
MODEL = "openai/gpt-4o"
CREDENTIAL = "your-llm-credential"  # From Settings > LLM Gateway

# Test Answer Relevance (ordinal: High/Medium/Low)
relevance_evaluator = AnswerRelevance(model=MODEL, credential=CREDENTIAL)
score = relevance_evaluator.score(
    user_query="What is the capital of France?",
    rag_response="Paris is the capital of France."
)
print(f"Relevance Score: {score.value} ({score.label}) - {score.reasoning}")

# Test Conciseness
conciseness_evaluator = Conciseness(model=MODEL, credential=CREDENTIAL)
score = conciseness_evaluator.score(
    response="Paris is the capital of France."
)
print(f"Conciseness Score: {score.value} - {score.reasoning}")

# Test Coherence
coherence_evaluator = Coherence(model=MODEL, credential=CREDENTIAL)
score = coherence_evaluator.score(
    response="Thank you for your question! I'd be happy to help.",
    prompt="Can you assist me?"
)
print(f"Coherence Score: {score.value} - {score.reasoning}")

# Sentiment is a Fiddler Trust Model — no model/credential needed
sentiment_evaluator = Sentiment()
scores = sentiment_evaluator.score("Paris is the capital of France.")
print(f"Sentiment: {scores[0].label} (confidence: {scores[1].value})")

from fiddler_evals.evaluators.base import Evaluator
from fiddler_evals.pydantic_models.score import Score

class LengthEvaluator(Evaluator):
    """
    Custom evaluator that checks if a response length is appropriate.
    Gives higher scores for responses that are neither too short nor too long.
    """

    def __init__(self, min_length: int = 10, max_length: int = 200):
        super().__init__()
        self.min_length = min_length
        self.max_length = max_length

    def score(self, output: str) -> Score:
        """Score based on response length appropriateness."""
        length = len(output.strip())

        if length < self.min_length:
            score_value = 0.0
            reasoning = f"Response too short ({length} chars, minimum {self.min_length})"
        elif length > self.max_length:
            score_value = 0.5
            reasoning = f"Response too long ({length} chars, maximum {self.max_length})"
        else:
            score_value = 1.0
            reasoning = f"Response length appropriate ({length} chars)"

        return Score(
            name="length_check",
            evaluator_name=self.name,
            value=score_value,
            reasoning=reasoning
        )

# Test the custom evaluator
length_evaluator = LengthEvaluator(min_length=15, max_length=100)
score = length_evaluator.score("Paris is the capital of France.")
print(f"Length Score: {score.value} - {score.reasoning}")

def word_count_evaluator(output: str) -> float:
    """Returns word count normalized to 0-1 scale."""
    word_count = len(output.split())
    return min(word_count / 50.0, 1.0)

# Use directly in evaluators list
evaluators = [
    AnswerRelevance(model=MODEL, credential=CREDENTIAL),
    word_count_evaluator,  # Function evaluator
]

from fiddler_evals import evaluate

# Define your AI application task
def my_llm_task(inputs: dict, extras: dict, metadata: dict) -> dict:
    """
    This function represents your AI application that you want to evaluate.

    Args:
        inputs: The input data from the dataset (e.g., {"question": "..."})
        extras: Additional context data (e.g., {"context": "..."})
        metadata: Any metadata associated with the test case

    Returns:
        dict: The outputs from your AI application (e.g., {"answer": "..."})
    """
    question = inputs.get("question", "")

    # Your LLM API call here
    # For this example, we'll use a mock response
    answer = f"Mock response to: {question}"

    return {"answer": answer}

# Set up evaluators
evaluators = [
    AnswerRelevance(model=MODEL, credential=CREDENTIAL),
    Conciseness(model=MODEL, credential=CREDENTIAL),
    Sentiment(),  # Fiddler Trust Model — no model/credential needed
    LengthEvaluator(),
]

# Run evaluation
experiment_result = evaluate(
    dataset=dataset,
    task=my_llm_task,
    evaluators=evaluators,
    name_prefix="my_experiment",
    description="Comprehensive LLM experiment",
    score_fn_kwargs_mapping={
        "user_query": lambda x: x["inputs"]["question"],
        "rag_response": "answer",
        "response": "answer",
        "output": "answer",
        "text": "answer",
    },
    max_workers=4  # Process 4 test cases concurrently
)

print(f"✅ Evaluated {len(experiment_result.results)} test cases")
print(f"📈 Generated {sum(len(result.scores) for result in experiment_result.results)} scores")

# Your task returns: {"answer": "Paris is the capital of France"}
# Evaluators expect: rag_response="...", response="...", or text="..."

# Map your output keys to evaluator parameter names:
score_fn_kwargs_mapping={
    "rag_response": "answer",  # Map 'rag_response' param → 'answer' output key
    "response": "answer",      # Map 'response' param → 'answer' output key
    "text": "answer",          # Map 'text' param → 'answer' output key
}

# Use lambda to extract nested or computed values:
score_fn_kwargs_mapping={
    "user_query": lambda x: x["inputs"]["question"],  # Extract from inputs dict
    "rag_response": "answer",                          # Simple string mapping
}

# Task returns this structure:
{"answer": "Paris is the capital of France"}

# But evaluators need these parameters:
# - AnswerRelevance.score(user_query="...", rag_response="...")
# - Conciseness.score(response="...")
# - Sentiment.score(text="...")

# Solution: Map parameter names to your output structure
score_fn_kwargs_mapping={
    "user_query": lambda x: x["inputs"]["question"],  # For AnswerRelevance
    "rag_response": "answer",                          # For AnswerRelevance
    "response": "answer",                              # For Conciseness
    "text": "answer",                                  # For Sentiment
}

# Convert to DataFrame for further analysis
results_data = []

for result in experiment_result.results:
    item = result.experiment_item
    row = {
        'dataset_item_id': item.dataset_item_id,
        'status': item.status,
        'duration_ms': item.duration_ms,
    }

    # Add scores as columns
    for score in result.scores:
        row[f'{score.name}_score'] = score.value
        row[f'{score.name}_reasoning'] = score.reasoning

    results_data.append(row)

results_df = pd.DataFrame(results_data)
results_df.to_csv('experiment_results.csv', index=False)
print("💾 Results exported to experiment_results.csv")