# promptfoo regression tests for grounding and attribution (PB-RAG-06-05).
# Replace each ${PLACEHOLDER} before running. Add the golden set (FND-04) as extra tests,
# for example: tests: file://../../../../../data/golden/<set>/promptfoo-tests.csv
# TODO(author): point tests at the golden set export, including unanswerable rows.
# verify: promptfoo azure provider fields (apiHost) and judge override (defaultTest.options.provider)
#   | https://www.promptfoo.dev/docs/providers/azure/
description: Grounding regression tests for prompt and model changes
prompts:
  - file://../prompts/system_prompt.txt

providers:
  - id: azure:chat:${CHAT_MODEL_DEPLOYMENT}
    config:
      apiHost: ${AZURE_OPENAI_SERVICE}.openai.azure.com
      temperature: 0.0

defaultTest:
  options:
    # Judge for model-graded assertions such as context-faithfulness.
    provider: azure:chat:${JUDGE_MODEL_DEPLOYMENT}
  assert:
    - type: latency
      threshold: 5000 # team default, milliseconds

tests:
  - description: Answerable query with high context faithfulness
    vars:
      query: What is the expense report submission deadline?
      context: >-
        All employees must submit business expense claims within 30 calendar days of the transaction date.
        Late submissions beyond 60 days require explicit vice president approval and risk forfeiture.
    assert:
      - type: is-json
      - type: context-faithfulness
        threshold: 0.95 # team default
      - type: icontains
        value: 30 calendar days

  - description: Factual query requiring verbatim attribution
    vars:
      query: Who must approve expense reports submitted after 60 days?
      context: Claims submitted after 60 days require Vice President sign-off and may be rejected.
    assert:
      - type: is-json
      - type: context-faithfulness
        threshold: 0.95 # team default
      - type: icontains
        value: Vice President

  - description: Unanswerable query must return answerable false with no claims
    vars:
      query: What is the company policy on pet insurance reimbursement?
      context: Corporate Travel and Expense Policy covers airfare, rail, lodging, and meals.
    assert:
      - type: is-json
      - type: javascript
        value: JSON.parse(output).answerable === false && JSON.parse(output).claims.length === 0