run-chat-eval-pf-pipeline.yml

name: Test and Evaulate Contoso-Chat with Promptflow

on:
  workflow_dispatch:

env: 
  GROUP: ${{secrets.GROUP}}
  WORKSPACE: ${{secrets.WORKSPACE}}
  SUBSCRIPTION: ${{secrets.SUBSCRIPTION}}
  RUN_NAME: contoso_chat
  EVAL_RUN_NAME: contoso_chat_eval

jobs:
  login-runpf-evalpf-assertpf-registermodel:
    runs-on: ubuntu-latest 
    steps:
    - name: Check out repo
      uses: actions/checkout@v2
    - name: Install az ml extension
      run: az extension add -n ml -y
    - name: Azure login
      uses: azure/login@v1
      with:
        creds: ${{secrets.AZURE_CREDENTIALS}}
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: '3.11.4'
    - name: List current directory
      run: ls
    - name: Install promptflow
      run: pip install -r contoso-chat/requirements.txt
    - name: Run promptflow
      run: |
        pfazure run create -f contoso-chat/run.yml --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} --stream > run_info.txt
        cat run_info.txt
    - name: List current directory
      run: ls
    - name: Set run name
      run: |
        echo "RUN_NAME=$(python deployment/llmops-helper/parse_run_output.py run_info.txt)" >> "$GITHUB_ENV"
    - name: Show the current run name
      run: echo "Run name is:" ${{env.RUN_NAME}}
    - name: Show promptflow results
      run: pfazure run show-details --name ${{env.RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
    - name: Run promptflow evaluations
      run: |
        pfazure run create -f contoso-chat/run_evaluation_multi.yml --run ${{env.RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} --stream > eval_info.txt 
        cat eval_info.txt
    - name: Get eval run name
      run: echo "EVAL_RUN_NAME=$(python deployment/llmops-helper/parse_run_output.py eval_info.txt)" >> "$GITHUB_ENV"
    - name: Show the current eval run name
      run: echo "Eval run name is:" ${{env.EVAL_RUN_NAME}} 
    - name: Show promptflow details
      run: pfazure run show-details --name ${{env.EVAL_RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
    - name: Show promptflow metrics
      run: |
        pfazure run show-metrics --name ${{env.EVAL_RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} > eval_result.json 
        cat eval_result.json
    - name: List current directory
      run: ls
    - name: Get assert eval results
      id: jobMetricAssert
      run: |
            # NOTE The number after the file is the threshold score to pass the assertion.
            export ASSERT=$(python deployment/llmops-helper/assert.py eval_result.json 3) # NOTE <file>.json is the file name and decimal is the threshold for the assertion
            echo "::debug::Assert has returned the following value: $ASSERT"
            # assert.py will return True or False, but bash expects lowercase.
            if ${ASSERT,,} ; then
              echo "::debug::Prompt flow run met the quality bar and can be deployed."
              echo "::set-output name=result::true"
            else
              echo "::warning::Prompt flow run didn't meet quality bar."
              echo "::set-output name=result::false"
            fi
    - name: Show the assert result
      run: echo "Assert result is:" ${{ steps.jobMetricAssert.outputs.result }}
    - name: Register promptflow model
      if: ${{ steps.jobMetricAssert.outputs.result == 'true' }}
      run: az ml model create --file deployment/chat-model.yaml  -g ${{env.GROUP}} -w ${{env.WORKSPACE}}