npm install @autoblocks/client

import { BaseTestEvaluator, Evaluation } from '@autoblocks/client/testing';

interface MyTestCase {
  input: string;
  expectedSubstring: string;
}

class HasSubstring extends BaseTestEvaluator<MyTestCase, string> {
  id = 'has-substring';

  evaluateTestCase(args: { testCase: MyTestCase; output: string }): Evaluation {
    const score = args.output.includes(args.testCase.expectedSubstring) ? 1 : 0;
    return {
      score,
      threshold: { gte: 1 },
    };
  }
}

import { BaseAccuracy } from '@autoblocks/client/testing';

class Accuracy extends BaseAccuracy<MyTestCase, string> {
  id = 'accuracy';

  outputMapper(args: { output: string }): string {
    return args.output;
  }

  expectedOutputMapper(args: { testCase: MyTestCase }): string {
    return args.testCase.expectedOutput;
  }
}

import { runTestSuite } from '@autoblocks/client/testing/v2';

runTestSuite<MyTestCase, string>({
  id: 'my-test-suite',
  testCases: [
    {
      input: 'hello world',
      expectedOutput: 'hello world',
    },
  ],
  testCaseHash: ['input'],
  fn: ({ testCase }) => testCase.input,
  evaluators: [new Accuracy()],
});