Evals in Depth

// an eval harness in miniature
type Case = { id: string; input: string; tags: string[]; mustPass?: boolean;
              expect: { cites?: string[]; contains?: string[]; absent?: string[]; refuses?: boolean } };

async function grade(c: Case, out: Answer): Promise<{ pass: boolean; why: string[] }> {
  const why: string[] = [];
  for (const s of c.expect.cites ?? []) if (!out.citations.includes(s)) why.push(`missing citation ${s}`);
  for (const s of c.expect.contains ?? []) if (!out.text.includes(s)) why.push(`missing "${s}"`);
  for (const s of c.expect.absent ?? []) if (out.text.includes(s)) why.push(`forbidden "${s}"`);
  if (c.expect.refuses !== undefined && out.refused !== c.expect.refuses) why.push('refusal mismatch');
  return { pass: why.length === 0, why };
}

async function runEval(cases: Case[], system: System, runs = 3) {
  const results = [];
  for (const c of cases) {
    const graded = await Promise.all(Array.from({ length: runs }, async () => grade(c, await system.answer(c.input))));
    results.push({ id: c.id, mustPass: c.mustPass, pass: graded.every(g => g.pass), why: graded.flatMap(g => g.why) });
  }
  const score = results.filter(r => r.pass).length / results.length;
  const blocked = score < 0.8 || results.some(r => r.mustPass && !r.pass);
  return { score, blocked, results };       // stored with prompt, model, index and code versions
}