> ## Documentation Index
> Fetch the complete documentation index at: https://docs.anpord.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Run a suite

> Starts one run per case and variant, and returns the batch while trials continue in the background. The command harness needs a profile with a run command; other harnesses take no run. Any harness may take an install command.



## OpenAPI

````yaml /openapi.json post /v1/evals.start
openapi: 3.1.0
info:
  title: Anpord API
  version: 1.0.0
  description: >-
    Run agent evals and manage prompts. Every endpoint takes a JSON body over
    POST and authenticates with a bearer API key.
servers:
  - description: Production
    url: https://api.anpord.com
security: []
tags:
  - name: Connectors
    description: >-
      Connect the credentials an eval run needs: a harness to drive, a sandbox
      to run in, and a model to play a case's human.
  - name: Evals
    description: >-
      Run the cases of a suite on harness, model and sandbox variants, and
      compare the runs.
  - name: Prompts
    description: >-
      Read and write prompts. Content is versioned, so writes append rather than
      overwrite, and channels decide which version callers receive.
paths:
  /v1/evals.start:
    post:
      tags:
        - Evals
      summary: Run a suite
      description: >-
        Starts one run per case and variant, and returns the batch while trials
        continue in the background. The command harness needs a profile with a
        run command; other harnesses take no run. Any harness may take an
        install command.
      operationId: evals.start
      parameters: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/PublicStartBatchRequest'
        required: true
      responses:
        '200':
          description: The batch started, and the run it holds for each case and variant.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/StartedBatch'
        '400':
          description: The request did not match the expected schema
          content:
            application/json:
              schema:
                anyOf:
                  - $ref: '#/components/schemas/HttpApiDecodeError'
                  - $ref: '#/components/schemas/BadRequest'
        '401':
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Unauthorized'
        '403':
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Forbidden'
        '404':
          description: NotFound
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/NotFound'
        '409':
          description: Conflict
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Conflict'
      security:
        - bearer: []
components:
  schemas:
    PublicStartBatchRequest:
      type: object
      required:
        - cases
        - suite
        - trials
        - variants
      properties:
        cases:
          type: array
          items:
            type: object
            required:
              - id
              - name
            properties:
              cache:
                $ref: '#/components/schemas/CaseCache'
              id:
                $ref: '#/components/schemas/EvalCaseId'
              name:
                $ref: '#/components/schemas/EvalCaseName'
              user:
                anyOf:
                  - anyOf:
                      - type: object
                        required:
                          - kind
                          - goal
                          - prompt
                        properties:
                          kind:
                            type: string
                            enum:
                              - simulated
                          goal:
                            type: string
                            description: a string at most 2000 character(s) long
                            title: maxLength(2000)
                            minLength: 1
                            maxLength: 2000
                          prompt:
                            type: string
                            description: a string at most 8000 character(s) long
                            title: maxLength(8000)
                            minLength: 1
                            maxLength: 8000
                        additionalProperties: false
                      - type: object
                        required:
                          - kind
                          - replies
                        properties:
                          kind:
                            type: string
                            enum:
                              - scripted
                          replies:
                            type: array
                            items:
                              type: string
                              description: a string at most 4000 character(s) long
                              title: maxLength(4000)
                              minLength: 1
                              maxLength: 4000
                            description: an array of at most 24 item(s)
                            title: maxItems(24)
                            minItems: 1
                            maxItems: 24
                        additionalProperties: false
                  - type: 'null'
              prepare:
                anyOf:
                  - $ref: '#/components/schemas/EvalPrepare'
                  - type: 'null'
              source:
                $ref: '#/components/schemas/EvalSource'
              tags:
                $ref: '#/components/schemas/EvalCaseTags'
              variables:
                $ref: '#/components/schemas/EvalVariables'
              validator:
                anyOf:
                  - $ref: '#/components/schemas/EvalValidator'
                  - type: 'null'
              verify:
                anyOf:
                  - $ref: '#/components/schemas/EvalVerify'
                  - type: 'null'
            additionalProperties: false
          description: an array of at most 100 item(s)
          title: maxItems(100)
          minItems: 1
          maxItems: 100
        local:
          type: boolean
        suite:
          $ref: '#/components/schemas/EvalSuiteRequest'
        trials:
          $ref: '#/components/schemas/Int'
          description: a number between 1 and 10
          title: between(1, 10)
          minimum: 1
          maximum: 10
        trigger:
          anyOf:
            - $ref: '#/components/schemas/EvalTrigger'
            - type: 'null'
        variants:
          type: array
          items:
            $ref: '#/components/schemas/EvalVariantRequest'
          description: an array of at most 20 item(s)
          title: maxItems(20)
          minItems: 1
          maxItems: 20
      additionalProperties: false
      description: >-
        Run every case of a suite on every variant, each as many times as
        trials.
    StartedBatch:
      type: object
      required:
        - id
        - runs
      properties:
        id:
          type: string
        runs:
          type: array
          items:
            type: object
            required:
              - caseId
              - id
              - variantIndex
            properties:
              caseId:
                $ref: '#/components/schemas/EvalCaseId'
              id:
                type: string
              variantIndex:
                $ref: '#/components/schemas/Int'
            additionalProperties: false
      additionalProperties: false
      description: The batch started, and the run it holds for each case and variant.
    HttpApiDecodeError:
      type: object
      required:
        - issues
        - message
        - _tag
      properties:
        issues:
          type: array
          items:
            $ref: '#/components/schemas/Issue'
        message:
          type: string
        _tag:
          type: string
          enum:
            - HttpApiDecodeError
      additionalProperties: false
      description: The request did not match the expected schema
    BadRequest:
      type: object
      required:
        - message
        - _tag
      properties:
        message:
          type: string
        _tag:
          type: string
          enum:
            - BadRequest
      additionalProperties: false
    Unauthorized:
      type: object
      required:
        - message
        - _tag
      properties:
        message:
          type: string
        _tag:
          type: string
          enum:
            - Unauthorized
      additionalProperties: false
    Forbidden:
      type: object
      required:
        - message
        - _tag
      properties:
        message:
          type: string
        _tag:
          type: string
          enum:
            - Forbidden
      additionalProperties: false
    NotFound:
      type: object
      required:
        - message
        - _tag
      properties:
        message:
          type: string
        _tag:
          type: string
          enum:
            - NotFound
      additionalProperties: false
    Conflict:
      type: object
      required:
        - message
        - _tag
      properties:
        message:
          type: string
        _tag:
          type: string
          enum:
            - Conflict
      additionalProperties: false
    CaseCache:
      type: object
      required:
        - key
        - path
      properties:
        key:
          type: string
          description: a string at least 1 character(s) long
          title: minLength(1)
          minLength: 1
        path:
          type: string
          description: a string at least 1 character(s) long
          title: minLength(1)
          minLength: 1
      additionalProperties: false
    EvalCaseId:
      type: string
      description: The stable handle an author gives a case.
      title: maxLength(100)
      minLength: 1
      maxLength: 100
      pattern: ^[a-z0-9]+(?:-[a-z0-9]+)*$
    EvalCaseName:
      type: string
      description: What a case is called.
      title: maxLength(200)
      maxLength: 200
    EvalPrepare:
      type: object
      required:
        - name
        - source
      properties:
        name:
          type: string
          description: a string at most 100 character(s) long
          title: maxLength(100)
          minLength: 1
          maxLength: 100
        source:
          type: string
          description: a string at most 1000000 character(s) long
          title: maxLength(1000000)
          minLength: 1
          maxLength: 1000000
      additionalProperties: false
      description: A bundled TypeScript workspace setup and its exported function name.
    EvalSource:
      anyOf:
        - type: object
          required:
            - kind
          properties:
            kind:
              type: string
              enum:
                - empty
          additionalProperties: false
        - type: object
          required:
            - kind
            - ref
            - url
          properties:
            kind:
              type: string
              enum:
                - repo
            ref:
              anyOf:
                - type: string
                - type: 'null'
            url:
              type: string
              description: a string at least 1 character(s) long
              title: minLength(1)
              minLength: 1
          additionalProperties: false
        - type: object
          required:
            - files
            - kind
          properties:
            files:
              type: object
              required: []
              properties: {}
              additionalProperties:
                type: string
            kind:
              type: string
              enum:
                - files
          additionalProperties: false
      description: The workspace available to the harness before setup runs.
    EvalCaseTags:
      type: array
      items:
        type: string
        description: a string at most 40 character(s) long
        title: maxLength(40)
        minLength: 1
        maxLength: 40
      description: What a case is grouped under.
      title: maxItems(12)
      maxItems: 12
    EvalVariables:
      type: object
      required: []
      properties: {}
      additionalProperties:
        $ref: '#/components/schemas/EvalVariableValue'
      description: Values for the placeholders the run prompt names, such as {{task}}.
    EvalValidator:
      anyOf:
        - type: object
          required:
            - name
            - source
          properties:
            capture:
              type: boolean
            manifest:
              type: array
              items:
                type: object
                required:
                  - index
                  - name
                properties:
                  index:
                    $ref: '#/components/schemas/NonNegativeInt'
                  name:
                    type: string
                    description: a string at most 200 character(s) long
                    title: maxLength(200)
                    maxLength: 200
                additionalProperties: false
              description: an array of at most 20 item(s)
              title: maxItems(20)
              minItems: 1
              maxItems: 20
            name:
              type: string
              description: a string at most 100 character(s) long
              title: maxLength(100)
              minLength: 1
              maxLength: 100
            source:
              type: string
              description: a string at most 1000000 character(s) long
              title: maxLength(1000000)
              minLength: 1
              maxLength: 1000000
            sourceFiles:
              type: array
              items:
                type: object
                required:
                  - path
                  - content
                properties:
                  path:
                    type: string
                    description: >-
                      a string matching the pattern
                      ^(?!\.{1,2}(?:\/|$))[^/\\:\0]+(?:\/(?!\.{1,2}(?:\/|$))[^/\\:\0]+)*$
                    title: maxLength(512)
                    maxLength: 512
                    pattern: >-
                      ^(?!\.{1,2}(?:\/|$))[^/\\:\0]+(?:\/(?!\.{1,2}(?:\/|$))[^/\\:\0]+)*$
                  content:
                    type: string
                    description: a string at most 1000000 character(s) long
                    title: maxLength(1000000)
                    maxLength: 1000000
                additionalProperties: false
              description: an array of at most 100 item(s)
              title: maxItems(100)
              maxItems: 100
          additionalProperties: false
        - type: object
          required:
            - kind
            - name
            - checks
            - judges
          properties:
            kind:
              type: string
              enum:
                - judged
            capture:
              type: boolean
            name:
              type: string
              description: a string at most 100 character(s) long
              title: maxLength(100)
              minLength: 1
              maxLength: 100
            checks:
              type: array
              items:
                type: object
                required:
                  - name
                  - source
                properties:
                  capture:
                    type: boolean
                  manifest:
                    type: array
                    items:
                      type: object
                      required:
                        - index
                        - name
                      properties:
                        index:
                          $ref: '#/components/schemas/NonNegativeInt'
                        name:
                          type: string
                          description: a string at most 200 character(s) long
                          title: maxLength(200)
                          maxLength: 200
                      additionalProperties: false
                    description: an array of at most 20 item(s)
                    title: maxItems(20)
                    minItems: 1
                    maxItems: 20
                  name:
                    type: string
                    description: a string at most 100 character(s) long
                    title: maxLength(100)
                    minLength: 1
                    maxLength: 100
                  source:
                    type: string
                    description: a string at most 1000000 character(s) long
                    title: maxLength(1000000)
                    minLength: 1
                    maxLength: 1000000
                additionalProperties: false
              description: an array of at most 20 item(s)
              title: maxItems(20)
              maxItems: 20
            judges:
              type: array
              items:
                anyOf:
                  - type: object
                    required:
                      - kind
                      - name
                      - model
                      - prompt
                      - choices
                      - provider
                    properties:
                      kind:
                        type: string
                        enum:
                          - judge
                      name:
                        type: string
                        description: a string at most 100 character(s) long
                        title: maxLength(100)
                        minLength: 1
                        maxLength: 100
                      model:
                        type: string
                        description: a string at most 200 character(s) long
                        title: maxLength(200)
                        minLength: 1
                        maxLength: 200
                      prompt:
                        type: string
                        description: a string at most 32000 character(s) long
                        title: maxLength(32000)
                        minLength: 1
                        maxLength: 32000
                      expected:
                        type: string
                        description: a string at most 32000 character(s) long
                        title: maxLength(32000)
                        maxLength: 32000
                      choices:
                        type: object
                        required: []
                        properties: {}
                        patternProperties:
                          '':
                            type: number
                            description: a number between 0 and 1
                            title: between(0, 1)
                            minimum: 0
                            maximum: 1
                        propertyNames:
                          type: string
                          description: a string at least 1 character(s) long
                          title: minLength(1)
                          minLength: 1
                      threshold:
                        type: number
                        description: a number between 0 and 1
                        title: between(0, 1)
                        minimum: 0
                        maximum: 1
                      timeoutMs:
                        allOf:
                          - $ref: '#/components/schemas/Int'
                            description: a number between 1000 and 300000
                            title: between(1000, 300000)
                            minimum: 1000
                            maximum: 300000
                        title: between(1000, 300000)
                        description: a number between 1000 and 300000
                      provider:
                        type: string
                        enum:
                          - openai
                      harness:
                        $id: /schemas/never
                        not: {}
                        title: never
                    additionalProperties: false
                  - type: object
                    required:
                      - kind
                      - name
                      - model
                      - prompt
                      - choices
                      - harness
                    properties:
                      kind:
                        type: string
                        enum:
                          - judge
                      name:
                        type: string
                        description: a string at most 100 character(s) long
                        title: maxLength(100)
                        minLength: 1
                        maxLength: 100
                      model:
                        type: string
                        description: a string at most 200 character(s) long
                        title: maxLength(200)
                        minLength: 1
                        maxLength: 200
                      prompt:
                        type: string
                        description: a string at most 32000 character(s) long
                        title: maxLength(32000)
                        minLength: 1
                        maxLength: 32000
                      expected:
                        type: string
                        description: a string at most 32000 character(s) long
                        title: maxLength(32000)
                        maxLength: 32000
                      choices:
                        type: object
                        required: []
                        properties: {}
                        patternProperties:
                          '':
                            type: number
                            description: a number between 0 and 1
                            title: between(0, 1)
                            minimum: 0
                            maximum: 1
                        propertyNames:
                          type: string
                          description: a string at least 1 character(s) long
                          title: minLength(1)
                          minLength: 1
                      threshold:
                        type: number
                        description: a number between 0 and 1
                        title: between(0, 1)
                        minimum: 0
                        maximum: 1
                      timeoutMs:
                        allOf:
                          - $ref: '#/components/schemas/Int'
                            description: a number between 1000 and 300000
                            title: between(1000, 300000)
                            minimum: 1000
                            maximum: 300000
                        title: between(1000, 300000)
                        description: a number between 1000 and 300000
                      provider:
                        $id: /schemas/never
                        not: {}
                        title: never
                      harness:
                        type: string
                        enum:
                          - codex
                          - opencode
                          - pi
                          - fx
                          - claude
                          - gemini
                          - qwen
                          - cursor
                    additionalProperties: false
              description: an array of at most 20 item(s)
              title: maxItems(20)
              minItems: 1
              maxItems: 20
            sourceFiles:
              type: array
              items:
                type: object
                required:
                  - path
                  - content
                properties:
                  path:
                    type: string
                    description: >-
                      a string matching the pattern
                      ^(?!\.{1,2}(?:\/|$))[^/\\:\0]+(?:\/(?!\.{1,2}(?:\/|$))[^/\\:\0]+)*$
                    title: maxLength(512)
                    maxLength: 512
                    pattern: >-
                      ^(?!\.{1,2}(?:\/|$))[^/\\:\0]+(?:\/(?!\.{1,2}(?:\/|$))[^/\\:\0]+)*$
                  content:
                    type: string
                    description: a string at most 1000000 character(s) long
                    title: maxLength(1000000)
                    maxLength: 1000000
                additionalProperties: false
              description: an array of at most 100 item(s)
              title: maxItems(100)
              maxItems: 100
          additionalProperties: false
      description: A bundled TypeScript validator and its exported function name.
    EvalVerify:
      type: string
      description: The shell command that decides whether a trial passed.
      title: maxLength(8192)
      maxLength: 8192
    EvalSuiteRequest:
      type: object
      required:
        - id
        - name
        - prompt
      properties:
        id:
          $ref: '#/components/schemas/EvalSuiteId'
        name:
          $ref: '#/components/schemas/EvalSuiteName'
        prompt:
          $ref: '#/components/schemas/EvalPrompt'
      additionalProperties: false
      description: The suite the cases belong to, and the prompt they share.
    Int:
      type: integer
      description: an integer
      title: int
    EvalTrigger:
      type: object
      required:
        - source
      properties:
        source:
          type: string
          enum:
            - dashboard
            - api
            - cli
            - ci
            - mcp
        url:
          type: string
          description: a string at most 2048 character(s) long
          title: maxLength(2048)
          maxLength: 2048
      additionalProperties: false
      description: What started this run. Client-reported context, not authorization.
    EvalVariantRequest:
      type: object
      required:
        - harness
        - model
      properties:
        credentials:
          type: object
          required: []
          properties:
            harnessConnectionId:
              type: string
            sandboxConnectionId:
              type: string
          additionalProperties: false
        harness:
          type: string
          enum:
            - codex
            - opencode
            - pi
            - fx
            - claude
            - gemini
            - qwen
            - cursor
            - command
        model:
          type: string
          description: a string at least 1 character(s) long
          title: minLength(1)
          minLength: 1
        profile:
          $ref: '#/components/schemas/HarnessProfile'
        sandbox:
          type: string
          enum:
            - daytona
            - e2b
            - upstash
            - modal
            - cloudflare
            - vercel
            - local
      additionalProperties: false
    Issue:
      type: object
      required:
        - _tag
        - path
        - message
      properties:
        _tag:
          type: string
          enum:
            - Pointer
            - Unexpected
            - Missing
            - Composite
            - Refinement
            - Transformation
            - Type
            - Forbidden
          description: The tag identifying the type of parse issue
        path:
          type: array
          items:
            $ref: '#/components/schemas/PropertyKey'
          description: The path to the property where the issue occurred
        message:
          type: string
          description: A descriptive message explaining the issue
      additionalProperties: false
      description: >-
        Represents an error encountered while parsing a value to match the
        schema
    EvalVariableValue:
      type: string
      description: a string at most 2048 character(s) long
      title: maxLength(2048)
      maxLength: 2048
    NonNegativeInt:
      type: integer
      description: an integer
      title: int
      minimum: 0
    EvalSuiteId:
      type: string
      description: The stable handle an author gives a suite.
      title: maxLength(100)
      minLength: 1
      maxLength: 100
      pattern: ^[a-z0-9]+(?:-[a-z0-9]+)*$
    EvalSuiteName:
      type: string
      description: What a suite is called.
      title: maxLength(200)
      minLength: 1
      maxLength: 200
    EvalPrompt:
      type: string
      description: What the agent is asked to do.
      title: maxLength(16384)
      maxLength: 16384
    HarnessProfile:
      type: object
      required:
        - files
        - name
      properties:
        env:
          type: object
          required: []
          properties: {}
          patternProperties:
            '':
              type: string
          propertyNames:
            $ref: '#/components/schemas/EnvName'
        files:
          type: object
          required: []
          properties: {}
          patternProperties:
            '':
              type: string
              description: a string at most 96000 character(s) long
              title: maxLength(96000)
              maxLength: 96000
          propertyNames:
            $ref: '#/components/schemas/ProfilePath'
        install:
          type: string
        name:
          $ref: '#/components/schemas/ProfileName'
        run:
          type: string
        systemPrompt:
          type: string
      additionalProperties: false
      description: >-
        Configuration layered on a harness: files written under the sandbox home
        and workspace, a system prompt, environment, an install command run
        before the harness, and for the command harness the run command.
    PropertyKey:
      anyOf:
        - type: string
        - type: number
        - type: object
          required:
            - _tag
            - key
          properties:
            _tag:
              type: string
              enum:
                - symbol
            key:
              type: string
          additionalProperties: false
          description: an object to be decoded into a globally shared symbol
    ProfileName:
      type: string
      description: >-
        Lowercase letters, digits and hyphens, at most 64 characters, starting
        with a letter or digit.
      pattern: ^[a-z0-9][a-z0-9-]{0,63}$
  securitySchemes:
    bearer:
      type: http
      scheme: bearer

````