Azure / Azure/azure-sdk-for-python
[Question] Azure-AI-Evaluation | How to create custom aggregate metrics?
- Dominant language
- Python
- Stars
- 5.6k
- Forks
- 3.4k
- Avg merge
- 1d 21h
- Merged PRs (30d)
- 193
Description
I'm using the latest 1.2.0 release of `azure-ai-evaluation` and have defined the below code to run evals on a dataset using both provided and custom evaluators. This works and the results are visible in Azure AI Foundry, but all of the eval results are _averaged_. One of my custom evaluators `LatencyAndCostEvaluator` has total_tokens and total_cost but it's averaged and I want it summed.
How can we define the aggregation metrics using evaluate()?
```py
from azure.ai.evaluation import evaluate,CoherenceEvaluator,FluencyEvaluator,RelevanceEvaluator
from azure_ai.evals.custom_evaluators.latency_and_costs import LatencyAndCostEvaluator
from azure_ai.evals.custom_evaluators.databricks_evals import DatabricksToolEvaluator
result = evaluate(
evaluation_name="Full Eval Flow",
data=output_file,
evaluators={
"coherence": CoherenceEvaluator(model_config=model_config),
"fluency": FluencyEvaluator(model_config=model_config),
"relevance": RelevanceEvaluator(model_config=model_config),
"performance": LatencyAndCostEvaluator(model_config=model_config),
"databricks": DatabricksToolEvaluator(),
},
evaluator_config={
"coherence": {
"column_mapping": {
"query": "${data.question}",
"response": "${data.final_answer}",
}
},
"fluency": {
"column_mapping": {
"query": "${data.question}",
"response": "${data.final_answer}",
}
},
"relevance": {
"column_mapping": {
"query": "${data.question}",
"response": "${data.final_answer}",
}
},
"performance": {
"column_mapping": {
"thread_id": "${data.thread_id}",
"run_id": "${data.run_id}",
}
},
"databricks": {
"column_mapping": {
"target_tool": "ask_database",
"tool_calls": "${data.tool_calls}",
"total_tool_calls": "${data.total_tool_calls}",
"success_tool_calls": "${data.success_tool_calls}",
}
}
},
fail_on_evaluator_errors=True,
azure_ai_project=azure_ai_project,
)
print(result['studio_url'])
```
Here is the `LatencyAndCostEvaluator` for reference
```py
from openai import AsyncAzureOpenAI
from typing import Optional, TypedDict
from tokencost.constants import TOKEN_COSTS # https://github.com/AgentOps-AI/tokencost
class LatencyAndCostEvaluatorOutput(TypedDict):
run_completed_successfully: bool
latency_seconds: Optional[float]
total_tokens: Optional[int]
total_cost: Optional[float]
class AzureOpenAIModelConfiguration(TypedDict):
azure_endpoint: str
api_version: str
api_key: str
class LatencyAndCostEvaluator:
def __init__(self, model_config: dict):
self.llm = AsyncAzureOpenAI(
azure_endpoint=model_config["azure_endpoint"],
api_version=model_config["api_version"],
api_key=model_config["api_key"],
)
async def __call__(self, *, thread_id: str, run_id: str):
run = await self.llm.beta.threads.runs.retrieve(
thread_id=thread_id, run_id=run_id
)
assistant_model = run.model
# Check if the run completed successfully
run_completed_successfully = run.status == "completed"
# Check latency
latency_seconds = (
round(run.completed_at - run.created_at, 1)
if run.completed_at and run.created_at
else None
)
# Not streaming yet
# Get token usage and costs
current_token_costs = TOKEN_COSTS[assistant_model]
total_tokens = round(run.usage.prompt_tokens + run.usage.completion_tokens)
total_cost = round(
(run.usage.prompt_tokens * current_token_costs["input_cost_per_token"])
+ (
run.usage.completion_tokens
* current_token_costs["output_cost_per_token"]
),
2,
)
return LatencyAndCostEvaluatorOutput(
run_completed_successfully=run_completed_successfully,
latency_seconds=latency_seconds,
total_tokens=total_tokens,
total_cost=total_cost
)
```
Contributor guide
Assessment
This issue has not been assessed yet.