diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4a9bf555f..85b898b4d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,9 @@ jobs: - name: Upgrade pip, setuptools run: python -m pip install --upgrade pip setuptools - name: Install capa with build requirements - run: pip install -e .[build] + run: | + pip install -r requirements.txt + pip install -e .[build] - name: Build standalone executable run: pyinstaller --log-level DEBUG .github/pyinstaller/pyinstaller.spec - name: Does it run (PE)? diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cb2a00f97..4a591d778 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,6 +25,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + pip install -r requirements.txt pip install -e .[build] - name: build package run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 812528564..5553ceae5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,7 +35,9 @@ jobs: with: python-version: "3.11" - name: Install dependencies - run: pip install -e .[dev] + run: | + pip install -r requirements.txt + pip install -e .[dev] - name: Lint with ruff run: pre-commit run ruff - name: Lint with isort @@ -61,7 +63,9 @@ jobs: with: python-version: "3.11" - name: Install capa - run: pip install -e .[dev] + run: | + pip install -r requirements.txt + pip install -e .[dev] - name: Run rule linter run: python scripts/lint.py rules/ @@ -96,7 +100,9 @@ jobs: if: matrix.os == 'ubuntu-20.04' run: sudo apt-get install -y libyaml-dev - name: Install capa - run: pip install -e .[dev] + run: | + pip install -r requirements.txt + pip install -e .[dev] - name: Run tests (fast) # this set of tests runs about 80% of the cases in 20% of the time, # and should catch most errors quickly. @@ -131,7 +137,9 @@ jobs: run: sudo apt-get install -y libyaml-dev - name: Install capa if: ${{ env.BN_SERIAL != 0 }} - run: pip install -e .[dev] + run: | + pip install -r requirements.txt + pip install -e .[dev] - name: install Binary Ninja if: ${{ env.BN_SERIAL != 0 }} run: | @@ -188,7 +196,9 @@ jobs: - name: Install pyyaml run: sudo apt-get install -y libyaml-dev - name: Install capa - run: pip install -e .[dev] + run: | + pip install -r requirements.txt + pip install -e .[dev] - name: Run tests run: | mkdir ./.github/ghidra/project diff --git a/CHANGELOG.md b/CHANGELOG.md index beaa48caa..826fc78c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - replace Halo spinner with Rich #2086 @s-ff - optimize rule matching #2080 @williballenthin - add aarch64 as a valid architecture #2144 mehunhoff@google.com @williballenthin +- relax dependency version requirements for the capa library #2053 @williballenthin ### Breaking Changes diff --git a/doc/installation.md b/doc/installation.md index 57c939c2b..93df732c2 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -91,6 +91,12 @@ For more details about creating and using virtual environments, check out the [v ##### Install development dependencies +When developing capa, please use the pinned dependencies found in `requirements.txt`. +This ensures that everyone has the exact same, reproducible environment. +Please install these dependencies before install capa (from source or from PyPI): + +`$ pip install -r requirements.txt` + We use the following tools to ensure consistent code style and formatting: - [black](https://github.com/psf/black) code formatter - [isort](https://pypi.org/project/isort/) code formatter diff --git a/pyproject.toml b/pyproject.toml index 714a567b3..268950764 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,25 +32,78 @@ classifiers = [ "Topic :: Security", ] dependencies = [ - "tqdm==4.66.4", - "pyyaml==6.0.1", - "tabulate==0.9.0", - "colorama==0.4.6", - "termcolor==2.4.0", - "wcwidth==0.2.13", - "ida-settings==2.1.0", - "viv-utils[flirt]==0.7.9", - "networkx==3.1", - "ruamel.yaml==0.18.6", - "vivisect==1.1.1", - "pefile==2023.2.7", - "pyelftools==0.31", - "dnfile==0.14.1", - "dncil==1.0.2", - "pydantic==2.7.1", - "rich==13.7.1", - "humanize==4.9.0", - "protobuf==5.27.0", + # --------------------------------------- + # As a library, capa uses lower version bounds + # when specifying its dependencies. This lets + # other programs that use capa (and other libraries) + # to find a compatible set of dependency versions. + # + # We can optionally pin to specific versions or + # limit the upper bound when there's a good reason; + # but the default is to assume all greater versions + # probably work with capa until proven otherwise. + # + # The following link provides good background: + # https://iscinumpy.dev/post/bound-version-constraints/ + # + # When we develop capa, and when we distribute it as + # a standalone binary, we'll use specific versions + # that are pinned in requirements.txt. + # But the requirements for a library are specified here + # and are looser. + # + # Related discussions: + # + # - https://github.com/mandiant/capa/issues/2053 + # - https://github.com/mandiant/capa/pull/2059 + # - https://github.com/mandiant/capa/pull/2079 + # + # --------------------------------------- + # The following dependency versions were imported + # during June 2024 by truncating specific versions to + # their major-most version (major version when possible, + # or minor otherwise). + # As specific constraints are identified, please provide + # comments and context. + "tqdm>=4", + "pyyaml>=6", + "tabulate>=0.9", + "colorama>=0.4", + "termcolor>=2", + "wcwidth>=0.2", + "ida-settings>=2", + "ruamel.yaml>=0.18", + "pefile>=2023.2.7", + "pyelftools>=0.31", + "pydantic>=2", + "rich>=13", + "humanize>=4", + "protobuf>=5", + + # --------------------------------------- + # Dependencies that we develop + # + # These dependencies are often actively influenced by capa, + # so we provide a minimum patch version that includes the + # latest bug fixes we need here. + "viv-utils[flirt]>=0.7.9", + "vivisect>=1.1.1", + "dncil>=1.0.2", + + # --------------------------------------- + # Dependencies with version caps + # + # These dependencies must not exceed the version cap, + # typically due to dropping support for python releases + # we still support. + + # TODO(williballenthin): networkx 3.2 doesn't support python 3.8 while capa does. + # https://github.com/mandiant/capa/issues/1966 + "networkx>=3,<3.2", + + # TODO(williballenthin): dnfile 0.15 changes UserString API and we havent updated yet. + # https://github.com/mandiant/capa/pull/2037 + "dnfile>=0.14.1,<0.15", ] dynamic = ["version"] @@ -63,6 +116,10 @@ namespaces = false [project.optional-dependencies] dev = [ + # Dev and build dependencies are not relaxed because + # we want all developer environments to be consistent. + # These dependencies are not used in production environments + # and should not conflict with other libraries/tooling. "pre-commit==3.5.0", "pytest==8.0.0", "pytest-sugar==1.0.0", @@ -99,6 +156,10 @@ dev = [ "deptry==0.16.1" ] build = [ + # Dev and build dependencies are not relaxed because + # we want all developer environments to be consistent. + # These dependencies are not used in production environments + # and should not conflict with other libraries/tooling. "pyinstaller==6.7.0", "setuptools==69.5.1", "build==1.2.1" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..004f98725 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,46 @@ +# Dependencies with specific version constraints +# used during development and building the standalone executables. +# For these environments, use `pip install -r requirements.txt` +# before installing capa from source/pypi. This will ensure +# the following specific versions are used. +# +# Initially generated via: pip freeze | grep -v -- "-e" +# Kept up to date by dependabot. +annotated-types==0.7.0 +colorama==0.4.6 +cxxfilt==0.2.2 +dncil==1.0.2 +dnfile==0.15.0 +funcy==2.0 +humanize==4.9.0 +ida-netnode==3.0 +ida-settings==2.1.0 +intervaltree==3.1.0 +markdown-it-py==3.0.0 +mdurl==0.1.2 +msgpack==1.0.8 +networkx==3.1 +pefile==2023.2.7 +pip==24.0 +protobuf==5.27.1 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycparser==2.22 +pydantic==2.7.3 +pydantic-core==2.18.4 +pyelftools==0.31 +pygments==2.18.0 +python-flirt==0.8.6 +pyyaml==6.0.1 +rich==13.7.1 +ruamel-yaml==0.18.6 +ruamel-yaml-clib==0.2.8 +setuptools==65.5.0 +six==1.16.0 +sortedcontainers==2.4.0 +tabulate==0.9.0 +termcolor==2.4.0 +tqdm==4.66.4 +viv-utils==0.7.9 +vivisect==1.1.1 +wcwidth==0.2.13