Notebook Tests #460
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # These should only be run on main, because they access secrets | |
| # Not part of the regular CI run, since notebook tests seem | |
| # particularly flaky | |
| name: Notebook Tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| commit_id: | |
| description: 'Branch or Commit ID (optional)' | |
| required: false | |
| type: string | |
| schedule: | |
| # * is a special character in YAML so we quote this string | |
| # Run at 10:00 UTC every day | |
| - cron: '00 10 * * *' | |
| jobs: | |
| notebook_tests: | |
| runs-on: "Large_Linux" | |
| environment: test | |
| strategy: | |
| fail-fast: false # Don't cancel all on first failure | |
| matrix: | |
| python-version: ["3.9", "3.10", "3.11", "3.12"] | |
| permissions: | |
| id-token: write # for Azure CLI login | |
| steps: | |
| - name: Checkout repo at ${{ github.event_name == 'workflow_dispatch' && inputs.commit_id || github.sha }} | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event_name == 'workflow_dispatch' && inputs.commit_id || github.sha }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Rust | |
| shell: bash | |
| run: | | |
| curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.75.0 | |
| echo "$HOME/.cargo/bin" >> $GITHUB_PATH | |
| - name: Install guidance | |
| shell: bash | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install -e .[all,llamacpp,test] | |
| - name: Azure login | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - name: 'Run Azure CLI commands' | |
| shell: bash | |
| run: | | |
| az account show | |
| az group list | |
| - name: Notebook tests | |
| shell: bash | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| # Configure endpoints for Azure OpenAI | |
| AZUREAI_CHAT_ENDPOINT: ${{ vars.AZUREAI_CHAT_ENDPOINT }} | |
| AZUREAI_CHAT_MODEL: ${{ vars.AZUREAI_CHAT_MODEL }} | |
| AZUREAI_COMPLETION_ENDPOINT: ${{ vars.AZUREAI_COMPLETION_ENDPOINT }} | |
| AZUREAI_COMPLETION_MODEL: ${{ vars.AZUREAI_COMPLETION_MODEL }} | |
| # Configure endpoints for Azure AI Studio | |
| AZURE_AI_STUDIO_PHI3_ENDPOINT: ${{ vars.AZURE_AI_STUDIO_PHI3_ENDPOINT }} | |
| AZURE_AI_STUDIO_PHI3_DEPLOYMENT: ${{ vars.AZURE_AI_STUDIO_PHI3_DEPLOYMENT }} | |
| AZURE_AI_STUDIO_PHI3_KEY: ${{ secrets.AZURE_AI_STUDIO_PHI3_KEY }} | |
| AZURE_AI_STUDIO_MISTRAL_CHAT_ENDPOINT: ${{ vars.AZURE_AI_STUDIO_MISTRAL_CHAT_ENDPOINT }} | |
| AZURE_AI_STUDIO_MISTRAL_CHAT_DEPLOYMENT: ${{ vars.AZURE_AI_STUDIO_MISTRAL_CHAT_DEPLOYMENT }} | |
| AZURE_AI_STUDIO_MISTRAL_CHAT_KEY: ${{ secrets.AZURE_AI_STUDIO_MISTRAL_CHAT_KEY }} | |
| AZURE_AI_STUDIO_LLAMA3_CHAT_ENDPOINT: ${{ vars.AZURE_AI_STUDIO_LLAMA3_CHAT_ENDPOINT }} | |
| AZURE_AI_STUDIO_LLAMA3_CHAT_DEPLOYMENT: ${{ vars.AZURE_AI_STUDIO_LLAMA3_CHAT_DEPLOYMENT }} | |
| AZURE_AI_STUDIO_LLAMA3_CHAT_KEY: ${{ secrets.AZURE_AI_STUDIO_LLAMA3_CHAT_KEY }} | |
| run: | | |
| pytest -vv --cov=guidance --cov-report=xml --cov-report=term-missing \ | |
| ./tests/notebooks | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} |