Quick Start

pnpm add @m4trix/evals

mkdir -p src/evals

// src/evals/smoke.dataset.ts
import { Dataset } from '@m4trix/evals';

export const smokeDataset = Dataset.define({
  name: 'smoke',
  displayName: 'Smoke Suite',
  includedTags: ['smoke'],
});

// src/evals/smoke.test-case.ts
import { S, TestCase } from '@m4trix/evals';

const inputSchema = S.Struct({ prompt: S.String });
const outputSchema = S.Struct({ expectedMinScore: S.Number });

export const greetingCase = TestCase.describe({
  name: 'greeting',
  displayName: 'Friendly greeting',
  tags: ['smoke'],
  inputSchema,
  input: { prompt: 'Write a warm one-sentence greeting.' },
  outputSchema,
  output: { expectedMinScore: 70 },
});

// src/evals/quality.evaluator.ts
import { Evaluator, S, latencyMetric, percentScore, tokenCountMetric } from '@m4trix/evals';

const inputSchema = S.Struct({ prompt: S.String });
const outputSchema = S.Struct({ expectedMinScore: S.Number });

export const qualityEvaluator = Evaluator.use({
  name: 'withModel',
  resolve: () => ({ model: 'demo-model' }),
})
  .define({
    name: 'quality',
    displayName: 'Quality',
    inputSchema,
    outputSchema,
    scoreSchema: S.Struct({ scores: S.Array(S.Unknown) }),
    tags: ['smoke'],
  })
  .evaluate(async ({ input, output }) => {
    const startedAt = Date.now();
    const value = Math.min(100, input.prompt.length * 3);

    return {
      scores: [
        percentScore.make(
          { value },
          { definePassed: (data) => data.value >= (output?.expectedMinScore ?? 50) },
        ),
      ],
      metrics: [
        tokenCountMetric.make({ input: input.prompt.length, output: 20 }),
        latencyMetric.make({ ms: Date.now() - startedAt }),
      ],
    };
  });

// src/evals/smoke.run-config.ts
import { RunConfig } from '@m4trix/evals';
import { smokeDataset } from './smoke.dataset';
import { qualityEvaluator } from './quality.evaluator';

export const smokeRun = RunConfig.define({
  name: 'smoke',
  displayName: 'Smoke evals',
  tags: ['ci'],
  runs: [
    {
      dataset: smokeDataset,
      evaluators: [qualityEvaluator],
      repetitions: 1,
    },
  ],
});

eval-agents-simple run --run-config smoke