{"openapi":"3.1.0","info":{"title":"Dharma Evaluation Runs API","version":"1.1.0","description":"Organization-scoped preflight, launch, provider-preserving observation, polling, and evidence retrieval for Evaluation Runs. This API is separate from Agent Fabric enrollment, relay, task, and rollout operations."},"servers":[{"url":"https://www.dharma-ai.io"}],"security":[{"bearerAuth":[]}],"paths":{"/api/orgs/{orgId}/managed-evals/preflight":{"post":{"operationId":"preflightEvaluationCampaign","summary":"Validate a campaign and estimate its maximum credit authorization","parameters":[{"$ref":"#/components/parameters/OrganizationId"}],"requestBody":{"$ref":"#/components/requestBodies/CampaignRequest"},"responses":{"200":{"description":"Validated without model execution or credit debit","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PreflightResponse"}}}},"400":{"$ref":"#/components/responses/Error"},"401":{"$ref":"#/components/responses/Error"},"403":{"$ref":"#/components/responses/Error"}}}},"/api/orgs/{orgId}/managed-evals/campaigns":{"get":{"operationId":"listEvaluationCampaigns","summary":"List campaigns or retrieve one campaign and its sessions, tasks, and results","parameters":[{"$ref":"#/components/parameters/OrganizationId"},{"name":"campaignId","in":"query","schema":{"type":"string"}},{"name":"limit","in":"query","schema":{"type":"integer","minimum":1,"maximum":100,"default":20}}],"responses":{"200":{"description":"Evaluation evidence","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CampaignListResponse"}}}},"401":{"$ref":"#/components/responses/Error"},"403":{"$ref":"#/components/responses/Error"}}},"post":{"operationId":"launchEvaluationCampaign","summary":"Launch a preflight-reviewed campaign","parameters":[{"$ref":"#/components/parameters/OrganizationId"},{"$ref":"#/components/parameters/IdempotencyKey"}],"requestBody":{"$ref":"#/components/requestBodies/CampaignRequest"},"responses":{"201":{"description":"Campaign accepted for asynchronous execution","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CampaignAcceptedResponse"}}}},"400":{"$ref":"#/components/responses/Error"},"401":{"$ref":"#/components/responses/Error"},"402":{"$ref":"#/components/responses/Error"},"403":{"$ref":"#/components/responses/Error"},"409":{"$ref":"#/components/responses/Error"},"429":{"$ref":"#/components/responses/Error"}}}},"/api/orgs/{orgId}/evaluation-sidecar/events":{"post":{"operationId":"enqueueGeminiEvaluationObservation","summary":"Enqueue a Gemini-compatible production observation after the customer response path","description":"This endpoint never invokes the customer provider. It atomically records one bounded observation and queues scoring out of band.","parameters":[{"$ref":"#/components/parameters/OrganizationId"},{"$ref":"#/components/parameters/IdempotencyKey"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GeminiSidecarEvent"}}}},"responses":{"202":{"description":"Observation durably accepted or identified as an idempotent duplicate","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SidecarAcceptedResponse"}}}},"400":{"$ref":"#/components/responses/Error"},"401":{"$ref":"#/components/responses/Error"},"402":{"$ref":"#/components/responses/Error"},"403":{"$ref":"#/components/responses/Error"},"413":{"$ref":"#/components/responses/Error"},"503":{"$ref":"#/components/responses/Error"}}}},"/api/orgs/{orgId}/evaluation-sidecar/events/{eventId}":{"get":{"operationId":"getEvaluationObservation","summary":"Read deterministic and governed semantic scoring state","parameters":[{"$ref":"#/components/parameters/OrganizationId"},{"name":"eventId","in":"path","required":true,"schema":{"type":"string","format":"uuid"}}],"responses":{"200":{"description":"Observation status and bounded scoring output","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SidecarStatusResponse"}}}},"401":{"$ref":"#/components/responses/Error"},"403":{"$ref":"#/components/responses/Error"},"404":{"$ref":"#/components/responses/Error"}}}}},"components":{"securitySchemes":{"bearerAuth":{"type":"http","scheme":"bearer","bearerFormat":"Dharma organization token"}},"parameters":{"OrganizationId":{"name":"orgId","in":"path","required":true,"schema":{"type":"string","minLength":3}},"IdempotencyKey":{"name":"Idempotency-Key","in":"header","required":true,"schema":{"type":"string","minLength":8,"maxLength":200}}},"requestBodies":{"CampaignRequest":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CampaignRequest"}}}}},"schemas":{"IntegrationMode":{"type":"string","enum":["observe_existing_runtime","migrate_to_dharma_managed"],"default":"observe_existing_runtime","description":"Provider-preserving evaluation is the default. Managed migration is an explicit architecture change."},"EvaluationArm":{"type":"string","enum":["direct_baseline","stateful_dharma_runtime","held_out_backtest"]},"GeminiSidecarEvent":{"type":"object","x-dharma-image-limits":{"maxImages":8,"maxImageBytes":20971520,"maxInlineAggregateBytes":3145728,"maxBodyBytes":4456448},"additionalProperties":false,"required":["schema","provider","model","providerRequestId","traceId","agentRevisionId","contentPolicy","request","response","scoring"],"properties":{"schema":{"const":"dharma.evaluation-sidecar.gemini/v1"},"provider":{"const":"gemini"},"model":{"type":"string","pattern":"^gemini-"},"providerRequestId":{"type":"string","maxLength":200},"traceId":{"type":"string","maxLength":200},"agentRevisionId":{"type":"string","maxLength":200},"contentPolicy":{"type":"string","enum":["metadata_only","customer_authorized_content"],"default":"metadata_only"},"request":{"type":"object","description":"Gemini generateContent subset. Parts accept text, HTTPS fileData references for deterministic capture, or inlineData {mimeType,data} for native image-aware judging. At most eight images total; inline images are JPEG, PNG or WebP, 3 MiB decoded aggregate; non-image content remains bounded to 256 KiB. Semantic image scoring requires inline bytes for every image, explicit customer_authorized_content and semanticMaxCredits. Reference URLs are not fetched; credentials are removed. Metadata-only mode retains image hashes, not bytes.","additionalProperties":true},"response":{"type":"object","description":"Completed Gemini response subset and usage metadata.","additionalProperties":true},"scoring":{"type":"object","additionalProperties":false,"required":["deterministic","semantic"],"properties":{"deterministic":{"const":true},"semantic":{"type":"boolean","default":false},"semanticMaxCredits":{"type":"integer","minimum":1,"maximum":50000}}},"tags":{"type":"object","maxProperties":20,"additionalProperties":{"type":["string","number","boolean"]}}}},"SidecarAcceptedResponse":{"type":"object","required":["ok","productScope","schema","eventId","duplicate","status","statusUrl"],"properties":{"ok":{"const":true},"productScope":{"const":"evaluation_runs"},"schema":{"const":"dharma.evaluation-sidecar.gemini/v1"},"eventId":{"type":"string","format":"uuid"},"duplicate":{"type":"boolean"},"status":{"type":"string"},"statusUrl":{"type":"string"}}},"SidecarStatusResponse":{"type":"object","required":["ok","productScope","event"],"properties":{"ok":{"const":true},"productScope":{"const":"evaluation_runs"},"event":{"type":"object","additionalProperties":true}}},"CampaignRequest":{"type":"object","required":["name","arms","tasks"],"properties":{"name":{"type":"string","minLength":3,"maxLength":160},"description":{"type":"string","maxLength":2000},"agentId":{"type":"string"},"integrationMode":{"$ref":"#/components/schemas/IntegrationMode"},"arms":{"type":"array","minItems":1,"maxItems":3,"uniqueItems":true,"items":{"$ref":"#/components/schemas/EvaluationArm"}},"policyCandidateId":{"type":"string","description":"Required when the only arm is held_out_backtest."},"automationMode":{"type":"string","enum":["continuous_100"]},"evaluationContract":{"type":"object","additionalProperties":true},"tasks":{"type":"array","minItems":1,"maxItems":100,"items":{"$ref":"#/components/schemas/EvaluationTask"}}},"additionalProperties":false},"EvaluationTask":{"type":"object","required":["task_id","scenario"],"properties":{"task_id":{"type":"string","minLength":1,"maxLength":200},"scenario":{"type":"object","required":["user_message"],"additionalProperties":true},"hidden_ground_truth":{"type":"object","additionalProperties":true},"expected_state_gate":{"type":"object","additionalProperties":true},"tool_expectations":{"type":"object","additionalProperties":true},"success_criteria":{"type":"array","items":{"type":"string"}},"failure_labels_to_detect":{"type":"array","items":{"type":"string"}},"remediation_requirements":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["flag","rule"],"properties":{"flag":{"type":"string"},"rule":{"type":"string"}}}}},"additionalProperties":true},"PreflightResponse":{"type":"object","required":["ok","productScope","integrationMode","preflight"],"properties":{"ok":{"const":true},"productScope":{"const":"evaluation_runs"},"integrationMode":{"$ref":"#/components/schemas/IntegrationMode"},"preflight":{"type":"object","additionalProperties":true}}},"CampaignAcceptedResponse":{"type":"object","required":["ok","productScope","integrationMode","campaignId","status"],"properties":{"ok":{"const":true},"productScope":{"const":"evaluation_runs"},"integrationMode":{"$ref":"#/components/schemas/IntegrationMode"},"campaignId":{"type":"string"},"status":{"type":"string"},"runtimeMode":{"type":"string"},"tasks":{"type":"integer"},"results":{"type":"integer"},"arms":{"type":"array","items":{"$ref":"#/components/schemas/EvaluationArm"}}}},"CampaignListResponse":{"type":"object","required":["ok","productScope","campaigns","sessions","tasks","results"],"properties":{"ok":{"const":true},"productScope":{"const":"evaluation_runs"},"campaigns":{"type":"array","items":{"type":"object","additionalProperties":true}},"sessions":{"type":"array","items":{"type":"object","additionalProperties":true}},"tasks":{"type":"array","items":{"type":"object","additionalProperties":true}},"results":{"type":"array","items":{"type":"object","additionalProperties":true}}}},"Error":{"type":"object","required":["error","message"],"properties":{"error":{"type":"string"},"message":{"type":"string"},"correlationId":{"type":"string"}},"additionalProperties":true}},"responses":{"Error":{"description":"Organization-scoped error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}}