Overview

from dataclasses import dataclass

from autoblocks.testing.evaluators import BaseHasAllSubstrings
from autoblocks.testing.models import BaseTestCase
from autoblocks.testing.models import CreateHumanReviewJob
from autoblocks.testing.run import run_test_suite
from autoblocks.testing.util import md5

@dataclass
class TestCase(BaseTestCase):
    input: str
    expected_substrings: list[str]

    def hash(self) -> str:
        return md5(self.input) # Unique identifier for a test case

class HasAllSubstrings(BaseHasAllSubstrings[TestCase, str]):
    id = "has-all-substrings"

    def test_case_mapper(self, test_case: TestCase) -> list[str]:
        return test_case.expected_substrings

    def output_mapper(self, output: str) -> str:
        return output

run_test_suite(
    id="my-test-suite",
    test_cases=[
        TestCase(
            input="hello world",
            expected_substrings=["hello", "world"],
        )
    ], # Replace with your test cases
    fn=lambda test_case: test_case.input, # Replace with your LLM call
    evaluators=[HasAllSubstrings()], # Replace with your evaluators
    human_review_job=CreateHumanReviewJob(
        assignee_email_address="example@example.com",
        name="Review for accuracy",
    )
)

import { BaseHasAllSubstrings } from '@autoblocks/client/testing';
import { runTestSuite } from '@autoblocks/client/testing/v2';

interface TestCase {
  input: string;
  expectedSubstrings: string[];
}

class HasAllSubstrings extends BaseHasAllSubstrings<TestCase, string> {
  id = 'has-all-substrings';

  outputMapper(args: { output: string }) {
    return args.output;
  }

  testCaseMapper(args: { testCase: TestCase }) {
    return args.testCase.expectedSubstrings;
  }
}

runTestSuite<TestCase, string>({
  id: 'my-test-suite',
  testCases: [
    {
      input: 'hello world',
      expectedSubstrings: ['hello', 'world'],
    },
  ], // Replace with your test cases
  testCaseHash: ['input'],
  fn: ({ testCase }) => testCase.input, // Replace with your LLM call
  evaluators: [new HasAllSubstrings()], // Replace with your evaluators
  humanReviewJob: {
    assigneeEmailAddress: 'example@example.com',
    name: 'Review for accuracy',
    rubricId: '<rubric-id>',
  },
});

from dataclasses import dataclass

from autoblocks.testing.models import BaseTestCase
from autoblocks.testing.models import HumanReviewField
from autoblocks.testing.models import HumanReviewFieldContentType
from autoblocks.testing.run import RunManager
from autoblocks.testing.util import md5


# Update with your test case type
@dataclass
class TestCase(BaseTestCase):
    input: str

    def serialize_for_human_review(self) -> list[HumanReviewField]:
        return [
            HumanReviewField(
                name="Input",
                value=self.input,
                content_type=HumanReviewFieldContentType.TEXT,
            ),
        ]

    def hash(self) -> str:
        return md5(self.input)


# Update with your output type
@dataclass
class Output:
    output: str

    def serialize_for_human_review(self) -> list[HumanReviewField]:
        return [
            HumanReviewField(
                name="Output",
                value=self.output,
                content_type=HumanReviewFieldContentType.TEXT,
            ),
        ]


run = RunManager[TestCase, Output](
    test_id="test-id",
)

run.start()
# Add results from your test suite here
run.add_result(
    test_case=TestCase(input="Hello, world!"),
    output=Output(output="Hi, world!"),
)
run.end()

run.create_human_review_job(
    assignee_email_address="${emailAddress}",
    name="Review for accuracy",
)

import {
  HumanReviewFieldContentType,
  RunManager,
} from '@autoblocks/client/testing';

// Update with your test case and output type
interface TestCase {
  input: string;
}

interface Output {
  output: string;
}

const main = async () => {
  const runManager = new RunManager<TestCase, Output>({
    testId: 'test-id',
    testCaseHash: ['input'],
    serializeTestCaseForHumanReview: (testCase) => [
      {
        type: HumanReviewFieldContentType.TEXT,
        value: testCase.input,
        name: 'input',
      },
    ],
    serializeOutputForHumanReview: ({ output }) => [
      { type: HumanReviewFieldContentType.TEXT, value: output, name: 'output' },
    ],
  });

  await runManager.start();
  // Add results from your test suite here
  await runManager.addResult({
    testCase: { input: 'Hello, world!' },
    output: { output: 'Hi, world!' },
    evaluations: [], // Add any automated evaluations
  });
  await runManager.end();

  // Create a human review job for the test run
  await runManager.createHumanReviewJob({
    assigneeEmailAddress: 'example@example.com',
    name: 'Review for accuracy',
  });
};

main();