Merge pull request #2849 from ckirchhoff2021/npu-dev

Pick commits from 1.x branch
open-mmlab · Jun 29, 2023 · 3f6318b · 3f6318b
2 parents ee74e28 + cb4a484
commit 3f6318b
Show file tree

Hide file tree

Showing 77 changed files with 1,974 additions and 11,314 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -48,9 +48,9 @@ jobs:
             pyenv install << parameters.python >>
             pyenv global << parameters.python >>
       - run:
-          name: Upgrade pip
+          name: Upgrade pip and wheel
           command: |
-            python -m pip install pip --upgrade
+            python -m pip install pip wheel --upgrade
       - run:
           name: Install PyTorch
           command: python -m pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html
@@ -62,7 +62,7 @@ jobs:
           command: |
             rm -rf .eggs
             python setup.py check -m -s
-            python -m pip install -e .
+            python -m pip install -e . -v
           no_output_timeout: 20m
           environment:
             MMCV_WITH_OPS: 1
@@ -116,9 +116,9 @@ jobs:
             pyenv install 3.7.0
             pyenv global 3.7.0
       - run:
-          name: Upgrade pip
+          name: Upgrade pip and wheel
           command: |
-            python -m pip install pip --upgrade
+            python -m pip install pip wheel --upgrade
       - run:
           name: Install PyTorch
           command: python -m pip install torch==1.8.1+cu102 torchvision==0.9.1+cu102 -f https://download.pytorch.org/whl/torch_stable.html
@@ -139,7 +139,7 @@ jobs:
           command: |
             rm -rf .eggs
             python setup.py check -m -s
-            python -m pip install -e .
+            python -m pip install -e . -v
           environment:
             MMCV_WITH_OPS: 1
             MMCV_WITH_ORT: 1

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -28,7 +28,7 @@ env:
 
 jobs:
   build_without_torch:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         python-version: [3.7]
@@ -38,6 +38,8 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Upgrade pip and wheel
+        run: pip install pip wheel --upgrade
       - name: Install system dependencies
         run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg
       - name: Build and install
@@ -65,7 +67,7 @@ jobs:
               --ignore=tests/test_utils/test_torch_ops.py
 
   build_without_ops:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     env:
       MMCV_WITH_OPS: 0
     strategy:
@@ -85,12 +87,14 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Upgrade pip and wheel
+        run: pip install pip wheel --upgrade
       - name: Install system dependencies
         run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg
       - name: Install PyTorch
         run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
       - name: Build and install
-        run: rm -rf .eggs && pip install -e .
+        run: rm -rf .eggs && pip install -e . -v
       - name: Validate the installation
         run: python -c "import mmcv"
       - name: Run unittests
@@ -99,7 +103,7 @@ jobs:
           pytest tests/ --ignore=tests/test_ops
 
   build_cpu:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         python-version: [3.7]
@@ -121,13 +125,14 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Upgrade pip and wheel
+        run: pip install pip wheel --upgrade
       - name: Install system dependencies
         run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg
       - name: Install PyTorch
         run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
-      # pstuil is an optional package to detect the number of CPU for compiling mmcv
-      - name: Install psutil
-        run: pip install psutil
+      - name: Install ninja to speed the compilation
+        run: pip install ninja psutil
       - name: Create sdist and untar
         run: |
           MMCV_WITH_OPS=1 python setup.py sdist
@@ -136,7 +141,7 @@ jobs:
       - name: Build and install from sdist
         run: |
           pushd /tmp/mmcv-full*
-          pip install -e .
+          pip install -e . -v
           popd
       - name: Validate the installation
         run: python -c "import mmcv"
@@ -148,7 +153,7 @@ jobs:
           coverage report -m
 
   build_cu101:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     container:
       image: pytorch/pytorch:1.6.0-cuda10.1-cudnn7-devel
     env:
@@ -182,36 +187,32 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Upgrade pip and wheel
+        run: pip install pip wheel --upgrade
       - name: Fetch GPG keys
         run: |
           apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
           apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
       - name: Install python-dev
         run: apt-get update && apt-get install -y python${{matrix.python-version}}-dev
-        if: ${{matrix.python-version != '3.9'}}
-      - name: Install Pillow
-        run: python -m pip install Pillow==6.2.2
-        if: ${{matrix.torchvision == '0.4.2'}}
-      # When we use a third-party container, we need to add python -m to call
-      # the user-installed pip when we use the pip command, otherwise it will
-      # call the system pip
+        if: ${{matrix.python-version == '3.6'}}
       - name: Install PyTorch
-        run: python -m pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
+        run: pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
       - name: Install system dependencies
         run: apt-get update && apt-get install -y ffmpeg libturbojpeg ninja-build
       - name: Install dependencies for compiling onnx when python=3.9
-        run: python -m pip install protobuf && apt-get -y install libprotobuf-dev protobuf-compiler cmake
+        run: pip install protobuf && apt-get -y install libprotobuf-dev protobuf-compiler cmake
         if: ${{matrix.python-version == '3.9'}}
-      # pstuil is an optional package to detect the number of CPU for compiling mmcv
-      - name: Install psutil
-        run: python -m pip install psutil
+      - name: Install ninja to speed the compilation
+        run: pip install ninja psutil
+      - name: Install pre-built opencv-python for python3.6
+        run: pip install opencv-python --prefer-binary
+        if: ${{matrix.python-version == '3.6'}}
       - name: Build and install
-        run: rm -rf .eggs && python -m pip install -e .
-      - name: Validate the installation
-        run: python -c "import mmcv"
+        run: rm -rf .eggs && pip install -e . -v
       - name: Run unittests and generate coverage report
         run: |
-          python -m pip install -r requirements/test.txt
+          pip install -r requirements/test.txt
           coverage run --branch --source=mmcv -m pytest tests/
           coverage xml
           coverage report -m
@@ -227,7 +228,7 @@ jobs:
           fail_ci_if_error: false
 
   build_cu102:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     container:
       image: pytorch/pytorch:1.9.0-cuda10.2-cudnn7-devel
     env:
@@ -260,47 +261,46 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Upgrade pip and wheel
+        run: pip install pip wheel --upgrade
       - name: Fetch GPG keys
         run: |
           apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
           apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
-      - name: Add PPA
-        run: |
-          apt-get update && apt-get install -y software-properties-common
-          add-apt-repository -y ppa:deadsnakes/ppa
       - name: Install python-dev
-        run: apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python${{matrix.python-version}}-dev
-      - name: python -m Install PyTorch
-        run: python -m pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
+        run: apt-get update && apt-get install -y python${{matrix.python-version}}-dev
+        if: ${{matrix.python-version == '3.6'}}
+      - name: Install PyTorch
+        run: pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
       - name: Install system dependencies
         run: apt-get update && apt-get install -y ffmpeg libturbojpeg ninja-build
-      # pstuil is an optional package to detect the number of CPU for compiling mmcv
-      - name: Install psutil
-        run: python -m pip install psutil
+      - name: Install ninja to speed the compilation
+        run: pip install ninja psutil
       # the directory for header files for the Python C-API could be wrong since setuptools>=65.2.0
       - name: Install specified version for setuptools when python==3.10
-        run: python -m pip install 'setuptools<=65.1.0'
+        run: pip install 'setuptools<=65.1.0'
         if: ${{matrix.python-version == '3.10'}}
+      - name: Install pre-built opencv-python for python3.6
+        run: pip install opencv-python --prefer-binary
+        if: ${{matrix.python-version == '3.6'}}
       - name: Build and install
-        run: rm -rf .eggs && python -m pip install -e .
-      - name: Validate the installation
-        run: python -c "import mmcv"
+        run: rm -rf .eggs && pip install -e . -v
       - name: Run unittests and generate coverage report
         run: |
-          python -m pip install -r requirements/test.txt
+          pip install -r requirements/test.txt
           coverage run --branch --source=mmcv -m pytest tests/
           coverage xml
         if: ${{matrix.python-version != '3.10'}}
       # special treatment for python3.10 because onnx and onnxruntime don't provide python3.10 pre-built packages
       - name: Run unittests and generate coverage report for python3.10
         run: |
-          python -m pip install -r requirements/test.txt
+          pip install -r requirements/test.txt
           coverage run --branch --source=mmcv -m pytest tests/ --ignore=tests/test_ops/test_onnx.py --ignore=tests/test_ops/test_tensorrt.py --ignore=tests/test_ops/test_tensorrt_preprocess.py
           coverage xml
         if: ${{matrix.python-version == '3.10'}}
 
   build_cu116:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     container:
       image: pytorch/pytorch:1.13.0-cuda11.6-cudnn8-devel
     env:
@@ -319,34 +319,27 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Upgrade pip and wheel
+        run: pip install pip wheel --upgrade
       - name: Fetch GPG keys
         run: |
           apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
           apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
-      - name: Add PPA
-        run: |
-          apt-get update && apt-get install -y software-properties-common
-          add-apt-repository -y ppa:deadsnakes/ppa
-      - name: Install python-dev
-        run: apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python${{matrix.python-version}}-dev
-      - name: python -m Install PyTorch
-        run: python -m pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
+      - name: Install PyTorch
+        run: pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
       - name: Install system dependencies
         run: apt-get update && apt-get install -y ffmpeg libturbojpeg ninja-build
-      # pstuil is an optional package to detect the number of CPU for compiling mmcv
-      - name: Install psutil
-        run: python -m pip install psutil
+      - name: Install ninja to speed the compilation
+        run: pip install ninja psutil
       # the directory for header files for the Python C-API could be wrong since setuptools>=65.2.0
       - name: Install specified version for setuptools when python==3.10
-        run: python -m pip install 'setuptools<=65.1.0'
+        run: pip install 'setuptools<=65.1.0'
         if: ${{matrix.python-version == '3.10'}}
       - name: Build and install
-        run: rm -rf .eggs && python -m pip install -e .
-      - name: Validate the installation
-        run: python -c "import mmcv"
+        run: rm -rf .eggs && pip install -e . -v
       - name: Run unittests and generate coverage report
         run: |
-          python -m pip install -r requirements/test.txt
+          pip install -r requirements/test.txt
           coverage run --branch --source=mmcv -m pytest tests/ --ignore=tests/test_ops/test_onnx.py --ignore=tests/test_ops/test_tensorrt.py --ignore=tests/test_ops/test_tensorrt_preprocess.py
           coverage xml
 
@@ -370,10 +363,12 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.7
+      - name: Upgrade pip and wheel
+        run: python -m pip install pip wheel --upgrade
       - name: Install PyTorch
         run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu --no-cache-dir -f https://download.pytorch.org/whl/torch_stable.html
       - name: Build and install
-        run: pip install -e .
+        run: pip install -e . -v
       - name: Validate the installation
         run: python -c "import mmcv"
       - name: Run unittests
@@ -399,10 +394,12 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.7
+      - name: Upgrade pip and wheel
+        run: python -m pip install pip wheel --upgrade
       - name: Install PyTorch
         run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu --no-cache-dir -f https://download.pytorch.org/whl/torch_stable.html
       - name: Build and install
-        run: pip install -e .
+        run: pip install -e . -v
       - name: Validate the installation
         run: python -c "import mmcv"
       - name: Run unittests
@@ -434,6 +431,8 @@ jobs:
           python-version: 3.7
       - name: Install system dependencies
         run: brew install ffmpeg jpeg-turbo
+      - name: Upgrade pip and wheel
+        run: pip install pip wheel --upgrade
       - name: Install utils
         run: pip install psutil
       - name: Install Pillow
@@ -444,7 +443,7 @@ jobs:
       - name: Build and install
         run: |
           rm -rf .eggs
-          CC=clang CXX=clang++ CFLAGS='-stdlib=libc++' pip install -e .
+          CC=clang CXX=clang++ CFLAGS='-stdlib=libc++' pip install -e . -v
       - name: Validate the installation
         run: python -c "import mmcv"
       - name: Run unittests

diff --git a/.github/workflows/build_pat.yml b/.github/workflows/build_pat.yml
@@ -18,7 +18,7 @@ env:
 
 jobs:
   build_parrots:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     container:
       image: ghcr.io/cokedong/parrots:pat0.21.0a0_cuda11
       credentials:

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -8,7 +8,7 @@ concurrency:
 
 jobs:
   lint:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.7

diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml
@@ -8,7 +8,7 @@ concurrency:
 
 jobs:
   build-n-publish:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     if: startsWith(github.event.ref, 'refs/tags')
     steps:
       - uses: actions/checkout@v2
@@ -26,7 +26,7 @@ jobs:
           twine upload dist/* -u __token__ -p ${{ secrets.pypi_password }}
 
   build-n-publish_with_ops:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     if: startsWith(github.event.ref, 'refs/tags')
     steps:
       - uses: actions/checkout@v2

diff --git a/.gitignore b/.gitignore
@@ -27,6 +27,8 @@ wheels/
 .installed.cfg
 *.egg
 MANIFEST
+mlu-ops/
+mlu-ops.*
 
 # PyInstaller
 #  Usually these files are written by a python script from a template