Create Eval

Creates a reusable simulation definition with one or more task prompts and judges. Optional agents, persona, CLI installations, MCP servers, setup commands, cleanup commands, grounding policy, repeat count, and tags are saved with the eval. search_grounding supports disabled, enabled, and default; default preserves normal platform behavior. Creating the eval does not start a run and does not accept secret environment values.

OpenAPI

openapi: 3.1.0
info:
  title: Lightsage Public API
  description: Public API for Lightsage CLI and external integrations.
  version: 1.0.0
servers:
  - url: https://api.lightsage.com
    description: Production
security: []
tags:
  - name: Status
    description: Check API availability and validate the calling API key.
  - name: Visibility
    description: >-
      Measure visibility, citations, competitors, and prompt performance across
      answer platforms.
  - name: Prompts
    description: Manage tracked prompts and topics, and inspect their stored runs.
  - name: Opportunities
    description: Review ranked opportunities computed from prompt visibility results.
  - name: Tasks
    description: Manage the persisted action queue and task lifecycle.
  - name: Site Audits
    description: Review website audit results and page-level findings.
  - name: Agent Analytics
    description: >-
      See which coding agents, AI bots, and people visit your tracked sites,
      which pages they read, which pages return errors, and where the visits
      come from.
  - name: Evals
    description: Define eval tasks, judges, subjects, and saved configurations.
  - name: Eval Templates
    description: >-
      Select reusable judges, targets, schedules, metadata, and runtime settings
      with `POST /v2/eval-templates`, then copy only those settings into
      prompt-specific evals with `POST /v2/evals/batch`. Templates never contain
      prompts and remain independent from evals created from them.
  - name: Eval Runs
    description: Start eval runs and inspect their lifecycle state.
  - name: Eval Results
    description: Review completed eval outcomes and diagnostic details.
  - name: Agent Preference
    description: >-
      Compare your company head-to-head with a tracked competitor. Create a
      comparison with `POST /v2/agent-preference/evals`, run it with `POST
      /v2/eval-runs`, then read which company agents recommended.
  - name: Agents
    description: List runnable coding-agent harness and model combinations.
  - name: Repositories
    description: Manage reusable Git repository workspaces.
  - name: Environments
    description: List reusable runtime environments without exposing secret values.
  - name: MCP Servers
    description: List reusable MCP server integrations without exposing credentials.
  - name: CLIs
    description: List reusable command-line interfaces without exposing setup commands.
  - name: Skills
    description: List reusable skills available to workflows and evals.
paths:
  /v2/evals:
    post:
      tags:
        - Evals
      summary: Create Eval
      description: >-
        Creates a reusable simulation definition with one or more task prompts
        and judges. Optional agents, persona, CLI installations, MCP servers,
        setup commands, cleanup commands, grounding policy, repeat count, and
        tags are saved with the eval. `search_grounding` supports `disabled`,
        `enabled`, and `default`; `default` preserves normal platform behavior.
        Creating the eval does not start a run and does not accept secret
        environment values.
      operationId: v2_evals_create
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateV2EvalRequest'
        required: true
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/V2Eval'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      security:
        - LightsageApiKeyAuth: []
components:
  schemas:
    CreateV2EvalRequest:
      properties:
        name:
          type: string
          maxLength: 160
          minLength: 1
          title: Name
        prompts:
          items:
            type: string
          type: array
          maxItems: 20
          minItems: 1
          title: Prompts
          description: Task prompts. CLI clients expose this as repeatable `--prompt`.
        judges:
          items:
            type: string
          type: array
          maxItems: 20
          minItems: 1
          title: Judges
          description: >-
            Plain-text success criteria. CLI clients expose this as repeatable
            `--judge`.
        agent:
          items:
            type: string
          type: array
          maxItems: 10
          title: Agent
          description: >-
            Runnable agents from `GET /v2/agents`. An empty array saves the eval
            without default agents. CLI clients expose repeatable `--agent`.
        persona:
          type: string
          maxLength: 160
          title: Persona
          description: >-
            Optional perspective applied to the whole eval. It is stored as
            metadata and does not change prompts or multiply attempts.
        clis:
          items:
            type: string
          type: array
          maxItems: 20
          title: Clis
          description: >-
            Saved CLI IDs or shell installation commands. UUID values resolve
            saved CLIs from `GET /v2/clis`; every other value is run as an
            installation command. CLI clients expose repeatable `--cli`.
        mcps:
          items:
            anyOf:
              - $ref: '#/components/schemas/V2SavedMcpReference'
              - $ref: '#/components/schemas/V2InlineMcpDefinition'
          type: array
          maxItems: 20
          title: Mcps
          description: Saved MCP references or inline portable MCP definitions.
        setup_commands:
          items:
            type: string
          type: array
          maxItems: 20
          title: Setup Commands
          description: >-
            Shell commands run sequentially in the prepared repository before
            the agent starts. CLI clients expose repeatable `--setup-command`.
        cleanup_commands:
          items:
            type: string
          type: array
          maxItems: 20
          title: Cleanup Commands
          description: >-
            Best-effort shell commands run sequentially after agent execution
            and result capture, before sandbox deletion. CLI clients expose
            repeatable `--cleanup-command`.
        search_grounding:
          $ref: '#/components/schemas/SearchGroundingMode'
          description: >-
            Web-retrieval policy for every run. `disabled` hides native search
            and fetch tools and requests network isolation; `enabled` exposes
            retrieval tools, requires live retrieval in the runtime prompt, and
            qualifies the run only when retrieval is observed; `default` exposes
            the platform's normal tools without requiring retrieval.
          default: default
        runs:
          type: integer
          maximum: 20
          minimum: 1
          title: Runs
          description: Independent attempts for every prompt-agent pair.
          default: 1
        tags:
          items:
            type: string
          type: array
          maxItems: 20
          title: Tags
          description: >-
            Optional labels for organizing the eval. CLI clients may expose this
            string array as a repeatable `--tag` flag.
      additionalProperties: false
      type: object
      required:
        - name
        - prompts
        - judges
      title: CreateV2EvalRequest
      examples:
        - agent:
            - codex:gpt-5-4
          cleanup_commands:
            - npm run db:cleanup
          clis:
            - brew install lightsagehq/tools/lightsage
          judges:
            - The SDK is installed and the request succeeds.
          name: SDK installation
          prompts:
            - Install the SDK and send a successful request.
          runs: 2
          search_grounding: disabled
          setup_commands:
            - npm install
            - npm run db:migrate
          tags:
            - sdk
            - installation
    V2Eval:
      properties:
        id:
          type: string
          title: Id
        name:
          type: string
          title: Name
        prompts:
          items:
            type: string
          type: array
          title: Prompts
        judges:
          items:
            type: string
          type: array
          title: Judges
        agent:
          items:
            type: string
          type: array
          title: Agent
        persona:
          anyOf:
            - type: string
            - type: 'null'
          title: Persona
        clis:
          items:
            anyOf:
              - $ref: '#/components/schemas/V2SavedCliReference'
              - $ref: '#/components/schemas/V2InlineCliReference'
          type: array
          title: Clis
        mcps:
          items:
            anyOf:
              - $ref: '#/components/schemas/V2SavedMcpReference'
              - $ref: '#/components/schemas/V2InlineMcpDefinition'
          type: array
          title: Mcps
        setup_commands:
          items:
            type: string
          type: array
          title: Setup Commands
        cleanup_commands:
          items:
            type: string
          type: array
          title: Cleanup Commands
        search_grounding:
          $ref: '#/components/schemas/SearchGroundingMode'
          description: 'Saved web-retrieval policy: disabled, enabled, or default.'
          default: default
        runs:
          type: integer
          maximum: 20
          minimum: 1
          title: Runs
          default: 1
        tags:
          items:
            type: string
          type: array
          title: Tags
        created_at:
          anyOf:
            - type: string
            - type: 'null'
          title: Created At
        updated_at:
          anyOf:
            - type: string
            - type: 'null'
          title: Updated At
      type: object
      required:
        - id
        - name
      title: V2Eval
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    V2SavedMcpReference:
      properties:
        id:
          type: string
          minLength: 1
          title: Id
          description: Saved MCP server ID from `GET /v2/mcp-servers`.
      additionalProperties: false
      type: object
      required:
        - id
      title: V2SavedMcpReference
    V2InlineMcpDefinition:
      properties:
        name:
          type: string
          maxLength: 100
          minLength: 1
          title: Name
          description: Stable server name used in the selected agent's MCP config.
        type:
          type: string
          enum:
            - http
            - sse
            - stdio
          title: Type
        url:
          anyOf:
            - type: string
            - type: 'null'
          title: Url
          description: HTTPS endpoint for an `http` or `sse` MCP server.
        headers:
          additionalProperties:
            type: string
          type: object
          maxProperties: 50
          title: Headers
          description: >-
            HTTP headers. Reference run secrets with `${VARIABLE_NAME}` instead
            of embedding credentials.
        command:
          anyOf:
            - type: string
            - type: 'null'
          title: Command
          description: Executable command for a `stdio` MCP server.
        args:
          items:
            type: string
          type: array
          maxItems: 100
          title: Args
        env:
          additionalProperties:
            type: string
          type: object
          maxProperties: 100
          title: Env
          description: >-
            Environment mapping for a `stdio` MCP server. Values may reference
            the run environment's variables with `${VARIABLE_NAME}`; secrets
            (names ending in KEY, TOKEN, SECRET or PASSWORD) must.
      additionalProperties: false
      type: object
      required:
        - name
        - type
      title: V2InlineMcpDefinition
    SearchGroundingMode:
      type: string
      enum:
        - disabled
        - enabled
        - default
      title: SearchGroundingMode
      description: Requested web-retrieval behavior for an eval run.
    V2SavedCliReference:
      properties:
        id:
          type: string
          minLength: 1
          title: Id
          description: Saved CLI ID from `GET /v2/clis`.
      additionalProperties: false
      type: object
      required:
        - id
      title: V2SavedCliReference
    V2InlineCliReference:
      properties:
        install_command:
          type: string
          maxLength: 4000
          minLength: 1
          title: Install Command
          description: Shell command to install this CLI before the agent starts.
      additionalProperties: false
      type: object
      required:
        - install_command
      title: V2InlineCliReference
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
              - type: string
              - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
        input:
          title: Input
        ctx:
          type: object
          title: Context
      type: object
      required:
        - loc
        - msg
        - type
      title: ValidationError
  securitySchemes:
    LightsageApiKeyAuth:
      type: apiKey
      in: header
      name: X-Lightsage-Api-Key