Turn agent_eval into a proper module to include it in Python package #5491

Workflow file for this run

.github/workflows/contrib-tests.yml at 4222570

	# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
	# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

	name: ContribTests

	on:
	pull_request:
	branches: ["main"]
	paths:
	- "autogen/**"
	- "test/agentchat/contrib/**"
	- "test/test_browser_utils.py"
	- "test/test_retrieve_utils.py"
	- ".github/workflows/contrib-tests.yml"
	- "setup.py"

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
	cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
	permissions:
	{}
	# actions: read
	# checks: read
	# contents: read
	# deployments: read
	jobs:
	RetrieveChatTest:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [macos-latest, windows-2019]
	python-version: ["3.9", "3.10", "3.11"]
	exclude:
	- os: macos-latest
	python-version: "3.9"
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest-cov>=5
	- name: Install qdrant_client when python-version is 3.10
	if: matrix.python-version == '3.10'
	run: \|
	pip install -e .[retrievechat-qdrant]
	- name: Install packages and dependencies for RetrieveChat
	run: \|
	pip install -e .[retrievechat]
	- name: Set AUTOGEN_USE_DOCKER based on OS
	shell: bash
	run: \|
	if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
	echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
	fi
	- name: Coverage
	run: \|
	pytest test/test_retrieve_utils.py test/agentchat/contrib/retrievechat/test_retrievechat.py test/agentchat/contrib/retrievechat/test_qdrant_retrievechat.py test/agentchat/contrib/vectordb --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests

	RetrieveChatTest-Ubuntu:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.9", "3.10", "3.11"]
	services:
	pgvector:
	image: ankane/pgvector
	env:
	POSTGRES_DB: postgres
	POSTGRES_USER: postgres
	POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
	POSTGRES_HOST_AUTH_METHOD: trust
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	ports:
	- 5432:5432
	mongodb:
	image: mongodb/mongodb-atlas-local:latest
	ports:
	- 27017:27017
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest
	- name: Install qdrant_client when python-version is 3.10
	if: matrix.python-version == '3.10'
	run: \|
	pip install -e .[retrievechat-qdrant]
	- name: Install pgvector when on linux
	run: \|
	pip install -e .[retrievechat-pgvector]
	- name: Install mongodb when on linux
	run: \|
	pip install -e .[retrievechat-mongodb]
	- name: Install unstructured when python-version is 3.9 and on linux
	if: matrix.python-version == '3.9'
	run: \|
	sudo apt-get update
	sudo apt-get install -y tesseract-ocr poppler-utils
	pip install --no-cache-dir unstructured[all-docs]==0.13.0
	- name: Install packages and dependencies for RetrieveChat
	run: \|
	pip install -e .[retrievechat]
	- name: Set AUTOGEN_USE_DOCKER based on OS
	shell: bash
	run: \|
	echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
	- name: Coverage
	run: \|
	pip install pytest-cov>=5
	pytest test/test_retrieve_utils.py test/agentchat/contrib/retrievechat test/agentchat/contrib/vectordb --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests

	AgentEvalTest:
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest]
	python-version: ["3.10"]
	runs-on: ${{ matrix.os }}
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest-cov>=5
	- name: Install packages and dependencies for AgentEval
	run: \|
	pip install -e .
	- name: Coverage
	run: \|
	pytest test/agentchat/contrib/agent_eval/ --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests

	GPTAssistantAgent:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, macos-latest, windows-2019]
	python-version: ["3.10"]
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest-cov>=5
	- name: Install packages and dependencies for GPTAssistantAgent
	run: \|
	pip install -e .
	- name: Set AUTOGEN_USE_DOCKER based on OS
	shell: bash
	run: \|
	if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
	echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
	fi
	- name: Coverage
	run: \|
	pytest test/agentchat/contrib/test_gpt_assistant.py --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests

	TeachableAgent:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, macos-latest, windows-2019]
	python-version: ["3.11"]
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest-cov>=5
	- name: Install packages and dependencies for Teachability
	run: \|
	pip install -e .[teachable]
	- name: Set AUTOGEN_USE_DOCKER based on OS
	shell: bash
	run: \|
	if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
	echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
	fi
	- name: Coverage
	run: \|
	pytest test/agentchat/contrib/capabilities/test_teachable_agent.py --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests

	WebSurfer:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, macos-latest, windows-2019]
	python-version: ["3.12"]
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest-cov>=5
	- name: Install packages and dependencies for WebSurfer
	run: \|
	pip install -e .[websurfer]
	- name: Set AUTOGEN_USE_DOCKER based on OS
	shell: bash
	run: \|
	if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
	echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
	fi
	- name: Coverage
	run: \|
	pytest test/test_browser_utils.py test/agentchat/contrib/test_web_surfer.py --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests

	LMMTest:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, macos-latest, windows-2019]
	python-version: ["3.12"]
	steps:
	- uses: actions/checkout@v4
	with:
	lfs: true
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest-cov>=5
	- name: Install packages and dependencies for LMM
	run: \|
	pip install -e .[lmm]
	- name: Set AUTOGEN_USE_DOCKER based on OS
	shell: bash
	run: \|
	if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
	echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
	fi
	- name: Coverage
	run: \|
	pytest test/agentchat/contrib/test_img_utils.py test/agentchat/contrib/test_lmm.py test/agentchat/contrib/test_llava.py test/agentchat/contrib/capabilities/test_vision_capability.py --skip-openai
	- name: Image Gen Coverage
	if: ${{ matrix.os != 'windows-2019' && matrix.python-version != '3.12' }}
	run: \|
	pytest test/agentchat/contrib/capabilities/test_image_generation_capability.py --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests

	GeminiTest:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, macos-latest, windows-2019]
	python-version: ["3.9", "3.10", "3.11", "3.12"]
	exclude:
	- os: macos-latest
	python-version: "3.9"
	steps:
	- uses: actions/checkout@v4
	with:
	lfs: true
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest-cov>=5
	- name: Install packages and dependencies for Gemini
	run: \|
	pip install -e .[gemini,test]
	- name: Set AUTOGEN_USE_DOCKER based on OS
	shell: bash
	run: \|
	if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
	echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
	fi
	- name: Coverage
	run: \|
	pytest test/oai/test_gemini.py --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests

	TransformMessages:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, macos-latest, windows-2019]
	python-version: ["3.11"]
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest-cov>=5
	- name: Install packages and dependencies for Transform Messages
	run: \|
	pip install -e '.[long-context]'
	- name: Set AUTOGEN_USE_DOCKER based on OS
	shell: bash
	run: \|
	if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
	echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
	fi
	- name: Coverage
	run: \|
	pytest test/agentchat/contrib/capabilities/test_transform_messages.py --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittest

	LlamaIndexAgent:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, macos-latest, windows-2019]
	python-version: ["3.11"]
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest-cov>=5
	- name: Install packages and dependencies for LlamaIndexConverableAgent
	run: \|
	pip install -e .
	pip install llama-index
	pip install llama-index-llms-openai
	- name: Coverage
	run: \|
	pytest test/agentchat/contrib/test_llamaindex_conversable_agent.py --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests

	AnthropicTest:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: ["ubuntu-latest", "windows-latest", "macos-latest"]
	python-version: ["3.9", "3.10", "3.11", "3.12"]

	steps:
	- uses: actions/checkout@v4
	with:
	lfs: true
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest-cov>=5

	- name: Install packages and dependencies for Anthropic
	run: \|
	pip install -e .[test]
	pip install -e .[anthropic]

	- name: Set AUTOGEN_USE_DOCKER based on OS
	shell: bash
	run: \|
	if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
	echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
	fi

	- name: Coverage
	run: \|
	pytest test/oai/test_anthropic.py --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests

	MistralTest:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, macos-latest, windows-2019]
	python-version: ["3.9", "3.10", "3.11", "3.12"]
	exclude:
	- os: macos-latest
	python-version: "3.9"
	steps:
	- uses: actions/checkout@v4
	with:
	lfs: true
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest-cov>=5
	- name: Install packages and dependencies for Mistral
	run: \|
	pip install -e .[mistral,test]
	- name: Set AUTOGEN_USE_DOCKER based on OS
	shell: bash
	run: \|
	if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
	echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
	fi
	- name: Coverage
	run: \|
	pytest test/oai/test_mistral.py --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests

	TogetherTest:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, macos-latest, windows-2019]
	python-version: ["3.9", "3.10", "3.11", "3.12"]
	exclude:
	- os: macos-latest
	python-version: "3.9"
	steps:
	- uses: actions/checkout@v4
	with:
	lfs: true
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest-cov>=5
	- name: Install packages and dependencies for Together
	run: \|
	pip install -e .[together,test]
	- name: Set AUTOGEN_USE_DOCKER based on OS
	shell: bash
	run: \|
	if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
	echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
	fi
	- name: Coverage
	run: \|
	pytest test/oai/test_together.py --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests

	GroqTest:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, macos-latest, windows-2019]
	python-version: ["3.9", "3.10", "3.11", "3.12"]
	exclude:
	- os: macos-latest
	python-version: "3.9"
	steps:
	- uses: actions/checkout@v4
	with:
	lfs: true
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest-cov>=5
	- name: Install packages and dependencies for Groq
	run: \|
	pip install -e .[groq,test]
	- name: Set AUTOGEN_USE_DOCKER based on OS
	shell: bash
	run: \|
	if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
	echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
	fi
	- name: Coverage
	run: \|
	pytest test/oai/test_groq.py --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests

	CohereTest:
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os: [ubuntu-latest, macos-latest, windows-latest]
	python-version: ["3.9", "3.10", "3.11", "3.12"]
	steps:
	- uses: actions/checkout@v4
	with:
	lfs: true
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest-cov>=5
	- name: Install packages and dependencies for Cohere
	run: \|
	pip install -e .[cohere,test]
	- name: Set AUTOGEN_USE_DOCKER based on OS
	shell: bash
	run: \|
	if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
	echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
	fi
	- name: Coverage
	run: \|
	pytest test/oai/test_cohere.py --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests

	BedrockTest:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, macos-latest, windows-2019]
	python-version: ["3.9", "3.10", "3.11", "3.12"]
	exclude:
	- os: macos-latest
	python-version: "3.9"
	steps:
	- uses: actions/checkout@v4
	with:
	lfs: true
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- name: Install packages and dependencies for all tests
	run: \|
	python -m pip install --upgrade pip wheel
	pip install pytest-cov>=5
	- name: Install packages and dependencies for Amazon Bedrock
	run: \|
	pip install -e .[boto3,test]
	- name: Set AUTOGEN_USE_DOCKER based on OS
	shell: bash
	run: \|
	if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
	echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
	fi
	- name: Coverage
	run: \|
	pytest test/oai/test_bedrock.py --skip-openai
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Turn agent_eval into a proper module to include it in Python package #5491

Workflow file

Turn agent_eval into a proper module to include it in Python package #5491

Jobs

Run details

Workflow file for this run