# eval/promptfooconfig.yaml
# Judge route A: promptfoo scores the answers that the application produced.
#
# export_run.py calls the application's own search and answer code and writes
# runs/candidate_promptfoo.jsonl, one test per line:
#   {"vars": {"query": ..., "context": ..., "response": ...}}
# promptfoo does not generate anything here. The "echo" provider returns the prompt text as the
# output, so the assertions run on the application's real answer. A pull request that changes the
# prompt, the model settings or retrieval therefore changes what is judged.
#
# THRESHOLD
# The promptfoo default for context-faithfulness is 0.5. The 0.9 below is a TEAM DEFAULT.
# Measure the run-to-run variance in your sandbox before you lock it.
#
# JUDGE (the grader)
# Replace ${JUDGE_DEPLOYMENT} with the name of your Azure OpenAI judge deployment. Provider ids
# are literal, so this one is edited here. The host comes from the AZURE_OPENAI_HOST variable,
# for example yourresource.openai.azure.com (a host, not a URL).
# Sign in with the Azure CLI session. In the pipeline this is the AzureCLI@2 task.
# Judge runs spend tokens. Check the current price before you approve the budget.
#
# VERSION
# promptfoo is a Node.js package. Pin its version in the PROMPTFOO_VERSION pipeline variable.

description: RAG evaluation gate, context-faithfulness on the application's answers

prompts:
  - '{{response}}'

providers:
  - echo

defaultTest:
  options:
    provider:
      text:
        id: azure:chat:${JUDGE_DEPLOYMENT}
        config:
          apiHost: '{{ env.AZURE_OPENAI_HOST }}'
          temperature: 0
  assert:
    - type: context-faithfulness
      threshold: 0.9

# Rows marked "unanswerable": true in the golden set also get a refusal assertion (llm-rubric),
# which export_run.py adds to that row.
tests: file://../runs/candidate_promptfoo.jsonl