# .github/workflows/rag-eval.yml
# RAG evaluation gate for GitHub Actions. The Azure DevOps version is the main route for this team.
#
# HOW TO WIRE IT UP
# 1. Copy this file to .github/workflows/ in your repository.
# 2. Settings > Branches > the rule for main: require status checks to pass, and add "evaluate".
# 3. Add a CODEOWNERS entry so that changes to baselines/ and eval/ need a RAG Lead review:
#      baselines/  @your-org/rag-leads
#      eval/       @your-org/rag-leads
#
# AUTHENTICATION
# azure/login uses OpenID Connect (OIDC), so no long-lived key is stored. Pull requests from
# forks cannot get an OIDC token or the repository variables, so the job runs only for branches in
# this repository. Do not use pull_request_target to work around that: it would run
# untrusted code with the repository's permissions. Ask a maintainer to push a fork's change to
# a branch in this repository instead.
#
# Repository variables used: AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID,
# SEARCH_FUNCTION, ANSWER_FUNCTION, AZURE_OPENAI_HOST, JUDGE_DEPLOYMENT, PROMPTFOO_VERSION.
# Also define the settings that your application reads, so that it queries the CI test index.

name: rag-eval-gate

on:
  pull_request:
    branches:
      - main
    paths:
      - "prompts/**"
      - "retrieval/**"
      - "index/**"
      - "config/**"

permissions:
  id-token: write
  contents: read

jobs:
  evaluate:
    name: evaluate
    if: github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    env:
      AZURE_OPENAI_HOST: ${{ vars.AZURE_OPENAI_HOST }}
      JUDGE_DEPLOYMENT: ${{ vars.JUDGE_DEPLOYMENT }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install the pinned Python packages
        run: |
          python -m pip install --upgrade pip
          pip install -r eval/requirements.txt

      - name: Azure login with OIDC, no API key
        uses: azure/login@v2
        with:
          client-id: ${{ vars.AZURE_CLIENT_ID }}
          tenant-id: ${{ vars.AZURE_TENANT_ID }}
          subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }}

      - name: Run the evaluation gate
        run: |
          set -uo pipefail
          mkdir -p runs reports
          python eval/export_run.py --search "${{ vars.SEARCH_FUNCTION }}" --answer "${{ vars.ANSWER_FUNCTION }}" --out runs/candidate.json
          status=0
          # --max-p95-increase 0.25 is a team default: fail if p95 search latency rises by more than 25%.
          python eval/eval_retrieval.py \
            --candidate runs/candidate.json --baseline baselines/baseline.json \
            --candidate-meta runs/candidate.meta.json --baseline-meta baselines/baseline.meta.json \
            --max-p95-increase 0.25 || status=1
          # Judge route A: promptfoo. For route B (Azure AI Evaluation SDK) use instead:
          #   python eval/eval_azure_ai.py --output reports/azure-eval-report.json || status=1
          npx promptfoo@${{ vars.PROMPTFOO_VERSION }} eval \
            --config eval/promptfooconfig.yaml --output reports/promptfoo-report.json || status=1
          exit $status

      - name: Upload the reports
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: eval-reports
          path: |
            reports/
            runs/