Guides
Get started with Autoblocks Evaluators in Python.
pip install autoblocks-client
from typing import Dict, Any from autoblocks.testing import BaseTestEvaluator, Evaluation class HasSubstring(BaseTestEvaluator): id = "has-substring" def evaluate_test_case(self, test_case: Dict[str, Any], output: str) -> Evaluation: score = 1 if test_case["expected_substring"] in output else 0 return Evaluation(score=score, threshold={"gte": 1})
from typing import Dict, Any from autoblocks.testing import BaseAccuracy class Accuracy(BaseAccuracy): id = "accuracy" def output_mapper(self, output: str) -> str: return output def expected_output_mapper(self, test_case: Dict[str, Any]) -> str: return test_case["expected_output"]
from typing import Dict, Any from autoblocks.testing import run_test_suite async def main(): await run_test_suite( id="my-test-suite", test_cases=[ { "input": "hello world", "expected_output": "hello world", } ], test_case_hash=["input"], fn=lambda test_case: test_case["input"], evaluators=[Accuracy()], ) if __name__ == "__main__": import asyncio asyncio.run(main())