> ## Documentation Index
> Fetch the complete documentation index at: https://docs.autoblocks.ai/llms.txt
> Use this file to discover all available pages before exploring further.

# Overview

> Human review mode is designed for humans to review, grade, and discuss test results..

## Creating a human review job programmatically

Whether you are on the free plan or a paid plan, you can create human review jobs directly in code with either the `RunManager` or in `runTestSuite`.

### run\_test\_suite / runTestSuite

<CodeGroup>
  ```python python
  from dataclasses import dataclass

  from autoblocks.testing.evaluators import BaseHasAllSubstrings
  from autoblocks.testing.models import BaseTestCase
  from autoblocks.testing.models import CreateHumanReviewJob
  from autoblocks.testing.run import run_test_suite
  from autoblocks.testing.util import md5

  @dataclass
  class TestCase(BaseTestCase):
      input: str
      expected_substrings: list[str]

      def hash(self) -> str:
          return md5(self.input) # Unique identifier for a test case

  class HasAllSubstrings(BaseHasAllSubstrings[TestCase, str]):
      id = "has-all-substrings"

      def test_case_mapper(self, test_case: TestCase) -> list[str]:
          return test_case.expected_substrings

      def output_mapper(self, output: str) -> str:
          return output

  run_test_suite(
      id="my-test-suite",
      test_cases=[
          TestCase(
              input="hello world",
              expected_substrings=["hello", "world"],
          )
      ], # Replace with your test cases
      fn=lambda test_case: test_case.input, # Replace with your LLM call
      evaluators=[HasAllSubstrings()], # Replace with your evaluators
      human_review_job=CreateHumanReviewJob(
          assignee_email_address="example@example.com",
          name="Review for accuracy",
      )
  )
  ```

  ```typescript typescript
  import { BaseHasAllSubstrings } from '@autoblocks/client/testing';
  import { runTestSuite } from '@autoblocks/client/testing/v2';

  interface TestCase {
    input: string;
    expectedSubstrings: string[];
  }

  class HasAllSubstrings extends BaseHasAllSubstrings<TestCase, string> {
    id = 'has-all-substrings';

    outputMapper(args: { output: string }) {
      return args.output;
    }

    testCaseMapper(args: { testCase: TestCase }) {
      return args.testCase.expectedSubstrings;
    }
  }

  runTestSuite<TestCase, string>({
    id: 'my-test-suite',
    testCases: [
      {
        input: 'hello world',
        expectedSubstrings: ['hello', 'world'],
      },
    ], // Replace with your test cases
    testCaseHash: ['input'],
    fn: ({ testCase }) => testCase.input, // Replace with your LLM call
    evaluators: [new HasAllSubstrings()], // Replace with your evaluators
    humanReviewJob: {
      assigneeEmailAddress: 'example@example.com',
      name: 'Review for accuracy',
      rubricId: '<rubric-id>',
    },
  });
  ```
</CodeGroup>

### Run Manager

<CodeGroup>
  ```python python
  from dataclasses import dataclass

  from autoblocks.testing.models import BaseTestCase
  from autoblocks.testing.models import HumanReviewField
  from autoblocks.testing.models import HumanReviewFieldContentType
  from autoblocks.testing.run import RunManager
  from autoblocks.testing.util import md5


  # Update with your test case type
  @dataclass
  class TestCase(BaseTestCase):
      input: str

      def serialize_for_human_review(self) -> list[HumanReviewField]:
          return [
              HumanReviewField(
                  name="Input",
                  value=self.input,
                  content_type=HumanReviewFieldContentType.TEXT,
              ),
          ]

      def hash(self) -> str:
          return md5(self.input)


  # Update with your output type
  @dataclass
  class Output:
      output: str

      def serialize_for_human_review(self) -> list[HumanReviewField]:
          return [
              HumanReviewField(
                  name="Output",
                  value=self.output,
                  content_type=HumanReviewFieldContentType.TEXT,
              ),
          ]


  run = RunManager[TestCase, Output](
      test_id="test-id",
  )

  run.start()
  # Add results from your test suite here
  run.add_result(
      test_case=TestCase(input="Hello, world!"),
      output=Output(output="Hi, world!"),
  )
  run.end()

  run.create_human_review_job(
      assignee_email_address="${emailAddress}",
      name="Review for accuracy",
  )
  ```

  ```typescript typescript
  import {
    HumanReviewFieldContentType,
    RunManager,
  } from '@autoblocks/client/testing';

  // Update with your test case and output type
  interface TestCase {
    input: string;
  }

  interface Output {
    output: string;
  }

  const main = async () => {
    const runManager = new RunManager<TestCase, Output>({
      testId: 'test-id',
      testCaseHash: ['input'],
      serializeTestCaseForHumanReview: (testCase) => [
        {
          type: HumanReviewFieldContentType.TEXT,
          value: testCase.input,
          name: 'input',
        },
      ],
      serializeOutputForHumanReview: ({ output }) => [
        { type: HumanReviewFieldContentType.TEXT, value: output, name: 'output' },
      ],
    });

    await runManager.start();
    // Add results from your test suite here
    await runManager.addResult({
      testCase: { input: 'Hello, world!' },
      output: { output: 'Hi, world!' },
      evaluations: [], // Add any automated evaluations
    });
    await runManager.end();

    // Create a human review job for the test run
    await runManager.createHumanReviewJob({
      assigneeEmailAddress: 'example@example.com',
      name: 'Review for accuracy',
    });
  };

  main();
  ```
</CodeGroup>

## Using the results

You can use the results of a human review job for a variety of purposes, such as:

* Fine tuning an evaluation model
* Few shot examples in your LLM judges
* Improving your core product based on expert feedback
* and more!