# azure-pipelines-rag-eval.yml
# RAG evaluation gate for Azure DevOps (Azure Repos).
#
# HOW TO WIRE IT UP
# 1. Copy this file into your repository and create a pipeline from it.
# 2. Project Settings > Repositories > Policies > the main branch > Build Validation:
#    add a policy that points to this pipeline, set it to Required, and set the path filter to
#    your prompt, retrieval, index and config folders (see the help text next to the field).
# 3. Optionally add Automatically included reviewers for baselines/ and eval/.
#
# The Build Validation policy is what runs this pipeline on a pull request, so this file has no
# pr: trigger.
#
# WHAT IT DOES
# eval/export_run.py calls your application's own search and answer code, so a pull request that
# changes retrieval, the prompt or the model settings changes what is scored. Both gates run, so
# a reviewer sees every failure. The step fails if either gate fails.

trigger: none

pool:
  vmImage: ubuntu-latest

variables:
  # Define these in the pipeline settings or in a variable group. Do not hard-code values here.
  #   SERVICE_CONNECTION  Azure service connection that signs in with Microsoft Entra ID
  #   SEARCH_FUNCTION     your search function, for example retrieval.search:search
  #   ANSWER_FUNCTION     your answer function, for example app.answer:answer
  #   AZURE_OPENAI_HOST   host of the judge resource, for example yourresource.openai.azure.com
  #   JUDGE_DEPLOYMENT    name of the judge deployment (route B reads it; edit the same name in
  #                       eval/promptfooconfig.yaml for route A)
  #   PROMPTFOO_VERSION   pinned promptfoo version, for example 0.124.1
  # Also define the settings that your application reads, so that it queries the CI test index.
  PYTHON_VERSION: "3.11"

steps:
  - checkout: self

  - task: UsePythonVersion@0
    displayName: Set the Python version
    inputs:
      versionSpec: $(PYTHON_VERSION)

  - script: |
      python -m pip install --upgrade pip
      pip install -r eval/requirements.txt
    displayName: Install the pinned Python packages

  - task: AzureCLI@2
    displayName: Run the evaluation gate
    inputs:
      azureSubscription: $(SERVICE_CONNECTION)
      scriptType: bash
      scriptLocation: inlineScript
      inlineScript: |
        set -uo pipefail
        mkdir -p runs reports
        python eval/export_run.py --search "$(SEARCH_FUNCTION)" --answer "$(ANSWER_FUNCTION)" --out runs/candidate.json
        status=0
        # --max-p95-increase 0.25 is a team default: fail if p95 search latency rises by more than 25%.
        python eval/eval_retrieval.py \
          --candidate runs/candidate.json --baseline baselines/baseline.json \
          --candidate-meta runs/candidate.meta.json --baseline-meta baselines/baseline.meta.json \
          --max-p95-increase 0.25 || status=1
        # Judge route A: promptfoo. For route B (Azure AI Evaluation SDK) use instead:
        #   python eval/eval_azure_ai.py --output reports/azure-eval-report.json || status=1
        npx promptfoo@$(PROMPTFOO_VERSION) eval \
          --config eval/promptfooconfig.yaml --output reports/promptfoo-report.json || status=1
        exit $status

  - task: PublishPipelineArtifact@1
    displayName: Publish the evaluation reports
    condition: always()
    inputs:
      targetPath: reports
      artifact: eval-reports

  - task: PublishPipelineArtifact@1
    displayName: Publish the candidate runs
    condition: always()
    inputs:
      targetPath: runs
      artifact: eval-runs