python-integration-tests.yml

#
# This workflow will run all python integrations tests.
#

name: Python Integration Tests

on:
  workflow_dispatch:
  pull_request:
    branches: ["main"]
  merge_group:
    branches: ["main"]
  schedule:
    - cron: "0 0 * * *" # Run at midnight UTC daily

permissions:
  contents: read

jobs:
  paths-filter:
    runs-on: ubuntu-latest
    outputs:
      pythonChanges: ${{ steps.filter.outputs.python}}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            python:
              - 'python/**'
      # run only if 'python' files were changed
      - name: python tests
        if: steps.filter.outputs.python == 'true'
        run: echo "Python file"
      # run only if not 'python' files were changed
      - name: not python tests
        if: steps.filter.outputs.python != 'true'
        run: echo "NOT python file"

  python-merge-gate:
    name: Python Pre-Merge Integration Tests
    needs: paths-filter
    if: github.event_name != 'pull_request' && github.event_name != 'schedule' && needs.paths-filter.outputs.pythonChanges == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      max-parallel: 1
      fail-fast: false
      matrix:
        python-version: ["3.11"]
        os: [ubuntu-latest]
    steps:
      - uses: actions/checkout@v4
      - name: Install poetry
        run: pipx install poetry
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: "poetry"
      - name: Install dependencies with hnswlib native disabled
        if: matrix.os == 'macos-latest' && matrix.python-version == '3.11'
        run: |
          export HNSWLIB_NO_NATIVE=1
          python -m pip install --upgrade pip setuptools wheel
          cd python && poetry install --with tests
      - name: Install dependencies with hnswlib native enabled
        if: matrix.os != 'macos-latest' || matrix.python-version != '3.11'
        run: |
          python -m pip install --upgrade pip setuptools wheel
          cd python
          poetry install --with tests
      - name: Install Ollama
        if: matrix.os == 'ubuntu-latest'
        run: |
          if ${{ vars.OLLAMA_MODEL != '' }}; then
            curl -fsSL https://ollama.com/install.sh | sh
            ollama serve &
            sleep 5
          fi
      - name: Pull model in Ollama
        if: matrix.os == 'ubuntu-latest'
        run: |
          if ${{ vars.OLLAMA_MODEL != '' }}; then
            ollama pull ${{ vars.OLLAMA_MODEL }}
            ollama list
          fi
      - name: Google auth
        uses: google-github-actions/auth@v2
        with:
          project_id: ${{ vars.VERTEX_AI_PROJECT_ID }}
          credentials_json: ${{ secrets.VERTEX_AI_SERVICE_ACCOUNT_KEY }}
      - name: Set up gcloud
        uses: google-github-actions/setup-gcloud@v2
      - name: Setup Redis Stack Server
        if: matrix.os == 'ubuntu-latest'
        run: docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest
      - name: Run Integration Tests
        id: run_tests
        shell: bash
        env: # Set Azure credentials secret as an input
          HNSWLIB_NO_NATIVE: 1
          Python_Integration_Tests: Python_Integration_Tests
          AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }} # azure-text-embedding-ada-002
          AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
          AZURE_OPENAI_TEXT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_TEXT_DEPLOYMENT_NAME }}
          AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }}
          AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
          BING_API_KEY: ${{ secrets.BING_API_KEY }}
          OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_ID }}
          OPENAI_TEXT_MODEL_ID: ${{ vars.OPENAI_TEXT_MODEL_ID }}
          OPENAI_EMBEDDING_MODEL_ID: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          PINECONE_API_KEY: ${{ secrets.PINECONE__APIKEY }}
          POSTGRES_CONNECTION_STRING: ${{secrets.POSTGRES__CONNECTIONSTR}}
          AZURE_AI_SEARCH_API_KEY: ${{secrets.AZURE_AI_SEARCH_API_KEY}}
          AZURE_AI_SEARCH_ENDPOINT: ${{secrets.AZURE_AI_SEARCH_ENDPOINT}}
          MONGODB_ATLAS_CONNECTION_STRING: ${{secrets.MONGODB_ATLAS_CONNECTION_STRING}}
          AZURE_KEY_VAULT_ENDPOINT: ${{secrets.AZURE_KEY_VAULT_ENDPOINT}}
          AZURE_KEY_VAULT_CLIENT_ID: ${{secrets.AZURE_KEY_VAULT_CLIENT_ID}}
          AZURE_KEY_VAULT_CLIENT_SECRET: ${{secrets.AZURE_KEY_VAULT_CLIENT_SECRET}}
          ACA_POOL_MANAGEMENT_ENDPOINT: ${{secrets.ACA_POOL_MANAGEMENT_ENDPOINT}}
          MISTRALAI_API_KEY: ${{secrets.MISTRALAI_API_KEY}}
          MISTRALAI_CHAT_MODEL_ID: ${{ vars.MISTRALAI_CHAT_MODEL_ID }}
          MISTRALAI_EMBEDDING_MODEL_ID: ${{ vars.MISTRALAI_EMBEDDING_MODEL_ID }}
          OLLAMA_MODEL: "${{ matrix.os == 'ubuntu-latest' && vars.OLLAMA_MODEL || '' }}" # phi3
          GOOGLE_AI_GEMINI_MODEL_ID: ${{ vars.GOOGLE_AI_GEMINI_MODEL_ID }}
          GOOGLE_AI_EMBEDDING_MODEL_ID: ${{ vars.GOOGLE_AI_EMBEDDING_MODEL_ID }}
          GOOGLE_AI_API_KEY: ${{ secrets.GOOGLE_AI_API_KEY }}
          VERTEX_AI_PROJECT_ID: ${{ vars.VERTEX_AI_PROJECT_ID }}
          VERTEX_AI_GEMINI_MODEL_ID: ${{ vars.VERTEX_AI_GEMINI_MODEL_ID }}
          VERTEX_AI_EMBEDDING_MODEL_ID: ${{ vars.VERTEX_AI_EMBEDDING_MODEL_ID }}
          REDIS_CONNECTION_STRING: ${{ vars.REDIS_CONNECTION_STRING }}
        run: |
          cd python
          poetry run pytest ./tests/integration ./tests/samples -v --junitxml=pytest.xml
      - name: Surface failing tests
        if: always()
        uses: pmeier/pytest-results-action@main
        with:
          # A list of JUnit XML files, directories containing the former, and wildcard
          # patterns to process.
          # See @actions/glob for supported patterns.
          path: python/pytest.xml
          # (Optional) Add a summary of the results at the top of the report
          summary: true
          # (Optional) Select which results should be included in the report.
          # Follows the same syntax as `pytest -r`
          display-options: fEX
          # (Optional) Fail the workflow if no JUnit XML was found.
          fail-on-empty: true
          # (Optional) Title of the test results section in the workflow summary
          title: Test results

  python-integration-tests:
    needs: paths-filter
    if: (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && needs.paths-filter.outputs.pythonChanges == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      max-parallel: 1
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11", "3.12"]
        os: [ubuntu-latest, windows-latest, macos-latest]
    steps:
      - uses: actions/checkout@v4
      - name: Install poetry
        run: pipx install poetry
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: "poetry"
      - name: Install dependencies with hnswlib native disabled
        if: matrix.os == 'macos-latest' && matrix.python-version == '3.11'
        run: |
          export HNSWLIB_NO_NATIVE=1
          python -m pip install --upgrade pip setuptools wheel
          cd python && poetry install --with tests

      - name: Install dependencies with hnswlib native enabled
        if: matrix.os != 'macos-latest' || matrix.python-version != '3.11'
        run: |
          python -m pip install --upgrade pip setuptools wheel
          cd python && poetry install --with tests

      - name: Install Ollama
        if: matrix.os == 'ubuntu-latest'
        run: |
          curl -fsSL https://ollama.com/install.sh | sh
          ollama serve &
          sleep 5

      - name: Pull model in Ollama
        if: matrix.os == 'ubuntu-latest'
        run: |
          ollama pull ${{ vars.OLLAMA_MODEL }}
          ollama list

      - name: Google auth
        uses: google-github-actions/auth@v2
        with:
          project_id: ${{ vars.VERTEX_AI_PROJECT_ID }}
          credentials_json: ${{ secrets.VERTEX_AI_SERVICE_ACCOUNT_KEY }}

      - name: Set up gcloud
        uses: google-github-actions/setup-gcloud@v2

      - name: Run Integration Tests
        id: run_tests
        shell: bash
        env: # Set Azure credentials secret as an input
          HNSWLIB_NO_NATIVE: 1
          Python_Integration_Tests: Python_Integration_Tests
          AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }} # azure-text-embedding-ada-002
          AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
          AZURE_OPENAI_TEXT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_TEXT_DEPLOYMENT_NAME }}
          AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }}
          AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
          BING_API_KEY: ${{ secrets.BING_API_KEY }}
          OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_ID }}
          OPENAI_TEXT_MODEL_ID: ${{ vars.OPENAI_TEXT_MODEL_ID }}
          OPENAI_EMBEDDING_MODEL_ID: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          PINECONE_API_KEY: ${{ secrets.PINECONE__APIKEY }}
          POSTGRES_CONNECTION_STRING: ${{secrets.POSTGRES__CONNECTIONSTR}}
          AZURE_AI_SEARCH_API_KEY: ${{secrets.AZURE_AI_SEARCH_API_KEY}}
          AZURE_AI_SEARCH_ENDPOINT: ${{secrets.AZURE_AI_SEARCH_ENDPOINT}}
          MONGODB_ATLAS_CONNECTION_STRING: ${{secrets.MONGODB_ATLAS_CONNECTION_STRING}}
          AZURE_KEY_VAULT_ENDPOINT: ${{secrets.AZURE_KEY_VAULT_ENDPOINT}}
          AZURE_KEY_VAULT_CLIENT_ID: ${{secrets.AZURE_KEY_VAULT_CLIENT_ID}}
          AZURE_KEY_VAULT_CLIENT_SECRET: ${{secrets.AZURE_KEY_VAULT_CLIENT_SECRET}}
          ACA_POOL_MANAGEMENT_ENDPOINT: ${{secrets.ACA_POOL_MANAGEMENT_ENDPOINT}}
          MISTRALAI_API_KEY: ${{secrets.MISTRALAI_API_KEY}}
          MISTRALAI_CHAT_MODEL_ID: ${{ vars.MISTRALAI_CHAT_MODEL_ID }}
          MISTRALAI_EMBEDDING_MODEL_ID: ${{ vars.MISTRALAI_EMBEDDING_MODEL_ID }}
          OLLAMA_MODEL: "${{ matrix.os == 'ubuntu-latest' && vars.OLLAMA_MODEL || '' }}" # phi3
          GOOGLE_AI_GEMINI_MODEL_ID: ${{ vars.GOOGLE_AI_GEMINI_MODEL_ID }}
          GOOGLE_AI_EMBEDDING_MODEL_ID: ${{ vars.GOOGLE_AI_EMBEDDING_MODEL_ID }}
          GOOGLE_AI_API_KEY: ${{ secrets.GOOGLE_AI_API_KEY }}
          VERTEX_AI_PROJECT_ID: ${{ vars.VERTEX_AI_PROJECT_ID }}
          VERTEX_AI_GEMINI_MODEL_ID: ${{ vars.VERTEX_AI_GEMINI_MODEL_ID }}
          VERTEX_AI_EMBEDDING_MODEL_ID: ${{ vars.VERTEX_AI_EMBEDDING_MODEL_ID }}
          REDIS_CONNECTION_STRING: ${{ vars.REDIS_CONNECTION_STRING }}
        run: |
          if ${{ matrix.os == 'ubuntu-latest' }}; then
            docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest
          fi

          cd python
          poetry run pytest ./tests/integration -v
          poetry run pytest ./tests/samples -v

  # This final job is required to satisfy the merge queue. It must only run (or succeed) if no tests failed
  python-integration-tests-check:
    if: always()
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 1
      fail-fast: false
    needs: [python-merge-gate, python-integration-tests]
    steps:
      - name: Get Date
        shell: bash
        run: |
          echo "date=$(date +'%m/%d/%Y %H:%M:%S')" >> "$GITHUB_ENV"

      - name: Run Type is Daily
        if: ${{ github.event_name == 'schedule' }}
        shell: bash
        run: |
          echo "run_type=Daily" >> "$GITHUB_ENV"

      - name: Run Type is Manual
        if: ${{ github.event_name == 'workflow_dispatch' }}
        shell: bash
        run: |
          echo "run_type=Manual" >> "$GITHUB_ENV"

      - name: Run Type is ${{ github.event_name }}
        if: ${{ github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'}}
        shell: bash
        run: |
          echo "run_type=${{ github.event_name }}" >> "$GITHUB_ENV"

      - name: Fail workflow if tests failed
        id: check_tests_failed
        if: contains(join(needs.*.result, ','), 'failure')
        uses: actions/github-script@v6
        with:
          script: core.setFailed('Integration Tests Failed!')

      - name: Fail workflow if tests cancelled
        id: check_tests_cancelled
        if: contains(join(needs.*.result, ','), 'cancelled')
        uses: actions/github-script@v6
        with:
          script: core.setFailed('Integration Tests Cancelled!')

      - name: Microsoft Teams Notification
        uses: skitionek/notify-microsoft-teams@master
        if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
        with:
          webhook_url: ${{ secrets.MSTEAMS_WEBHOOK }}
          dry_run: ${{ env.run_type != 'Daily' && env.run_type != 'Manual'}}
          job: ${{ toJson(job) }}
          steps: ${{ toJson(steps) }}
          overwrite: "{title: ` ${{ env.run_type }}: ${{ env.date }} `, text: ` ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`}"

      - name: Microsoft Teams Notification (Dry Run)
        uses: skitionek/notify-microsoft-teams@master
        if: github.ref != 'refs/heads/main'
        with:
          webhook_url: NONE
          dry_run: ${{ env.run_type != 'Daily' && env.run_type != 'Manual'}}
          job: ${{ toJson(job) }}
          steps: ${{ toJson(steps) }}
          overwrite: "{title: ` ${{ env.run_type }}: ${{ env.date }} `, text: ` ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`}"