Run evals from code

import { Anpord, type StartEvalRequest } from "anpord";

const anpord = new Anpord();
const model = process.env.EVAL_MODEL;

if (!model) throw new Error("Set EVAL_MODEL");

const cases: StartEvalRequest["cases"] = [
  {
    name: "repairs add",
    goal: "Fix the failing add function without changing its public API",
    source: {
      kind: "repo",
      url: "https://github.com/acme/agent-fixtures",
      ref: "b17b2ac",
    },
    setup: "bun install",
    verify: "bun test src/add.test.ts",
  },
];

const { id } = await anpord.evals.start({
  cases,
  prompt: "Read the repository. Make the smallest correct change. {{goal}}",
  tasks: [{ harness: "codex", model, provider: "daytona" }],
  trials: 3,
});

console.log(`Anpord run: ${id}`);

let run = await anpord.evals.get({ id });

while (run.status === "running") {
  await new Promise((resolve) => setTimeout(resolve, 2_000));
  run = await anpord.evals.get({ id });
}

if (run.status === "failed") {
  throw new Error(run.failure ?? "Eval run failed");
}

const regressions = run.cells.filter(
  (cell) => cell.comparison?.verdict === "regressed"
);
const missingEvidence = run.cells.filter(
  (cell) => (cell.distribution?.scored ?? 0) === 0
);

for (const cell of regressions) {
  console.error(cell.caseName, cell.comparison);
}
for (const cell of missingEvidence) {
  console.error(cell.caseName, "produced no scored trials");
}

if (regressions.length || missingEvidence.length) process.exitCode = 1;

await anpord.dispose();