From fbbdfd463f56ad0da1b45853f2f8136d9e841eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 19 Dec 2022 12:14:59 +0100 Subject: [PATCH 01/71] start doc --- recipe/HOWTO.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 recipe/HOWTO.md diff --git a/recipe/HOWTO.md b/recipe/HOWTO.md new file mode 100644 index 00000000..212e8237 --- /dev/null +++ b/recipe/HOWTO.md @@ -0,0 +1,4 @@ +# How to use the GPU queue + +> Work in progress + From 5dc139e3065dee0208f7a42e18f9ef03017b4402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 19 Dec 2022 18:51:51 +0100 Subject: [PATCH 02/71] rerender with self-hosted cirun support MNT: Re-rendered with conda-build 3.21.7+119.g1b221ef0, conda-smithy 3.22.1.post.dev3, and conda-forge-pinning 2022.12.19.14.36.50 --- .azure-pipelines/azure-pipelines-linux.yml | 76 ------------- .github/workflows/conda-build.yml | 119 +++++++++++++++++++++ .scripts/build_steps.sh | 4 +- .scripts/create_conda_build_artifacts.sh | 113 ------------------- README.md | 25 ----- azure-pipelines.yml | 6 -- cirun.yml | 8 ++ conda-forge.yml | 10 +- recipe/HOWTO.md | 15 +++ 9 files changed, 151 insertions(+), 225 deletions(-) delete mode 100755 .azure-pipelines/azure-pipelines-linux.yml create mode 100644 .github/workflows/conda-build.yml delete mode 100755 .scripts/create_conda_build_artifacts.sh delete mode 100644 azure-pipelines.yml create mode 100755 cirun.yml diff --git a/.azure-pipelines/azure-pipelines-linux.yml b/.azure-pipelines/azure-pipelines-linux.yml deleted file mode 100755 index d13d56b1..00000000 --- a/.azure-pipelines/azure-pipelines-linux.yml +++ /dev/null @@ -1,76 +0,0 @@ -# This file was generated automatically from conda-smithy. To update this configuration, -# update the conda-forge.yml and/or the recipe/meta.yaml. -# -*- mode: yaml -*- - -jobs: -- job: linux - pool: - vmImage: ubuntu-latest - strategy: - matrix: - linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10numpy1.20python3.9.____cpython: - CONFIG: linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10numpy1.20python3.9.____cpython - UPLOAD_PACKAGES: 'True' - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.2 - SHORT_CONFIG: linux_64_c_compiler_version10cuda_c_h2a2c77454b - timeoutInMinutes: 360 - - steps: - - script: | - rm -rf /opt/ghc - df -h - displayName: Manage disk space - - # configure qemu binfmt-misc running. This allows us to run docker containers - # embedded qemu-static - - script: | - docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes - ls /proc/sys/fs/binfmt_misc/ - condition: not(startsWith(variables['CONFIG'], 'linux_64')) - displayName: Configure binfmt_misc - - - script: | - export CI=azure - export GIT_BRANCH=$BUILD_SOURCEBRANCHNAME - export FEEDSTOCK_NAME=$(basename ${BUILD_REPOSITORY_NAME}) - export UPLOAD_ON_BRANCH="main" - if [[ "${BUILD_REASON:-}" == "PullRequest" ]]; then - export IS_PR_BUILD="True" - else - export IS_PR_BUILD="False" - fi - .scripts/run_docker_build.sh - displayName: Run docker build - env: - BINSTAR_TOKEN: $(BINSTAR_TOKEN) - FEEDSTOCK_TOKEN: $(FEEDSTOCK_TOKEN) - STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN) - - script: | - export CI=azure - export CI_RUN_ID=$(build.BuildNumber).$(system.JobAttempt) - export FEEDSTOCK_NAME=$(basename ${BUILD_REPOSITORY_NAME}) - export CONDA_BLD_DIR=build_artifacts - export ARTIFACT_STAGING_DIR="$(Build.ArtifactStagingDirectory)" - # Archive everything in CONDA_BLD_DIR except environments - export BLD_ARTIFACT_PREFIX=conda_artifacts - if [[ "$AGENT_JOBSTATUS" == "Failed" ]]; then - # Archive the CONDA_BLD_DIR environments only when the job fails - export ENV_ARTIFACT_PREFIX=conda_envs - fi - ./.scripts/create_conda_build_artifacts.sh - displayName: Prepare conda build artifacts - condition: succeededOrFailed() - - - task: PublishPipelineArtifact@1 - displayName: Store conda build artifacts - condition: not(eq(variables.BLD_ARTIFACT_PATH, '')) - inputs: - targetPath: $(BLD_ARTIFACT_PATH) - artifactName: $(BLD_ARTIFACT_NAME) - - - task: PublishPipelineArtifact@1 - displayName: Store conda build environment artifacts - condition: not(eq(variables.ENV_ARTIFACT_PATH, '')) - inputs: - targetPath: $(ENV_ARTIFACT_PATH) - artifactName: $(ENV_ARTIFACT_NAME) \ No newline at end of file diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml new file mode 100644 index 00000000..880d6bfe --- /dev/null +++ b/.github/workflows/conda-build.yml @@ -0,0 +1,119 @@ +# This file was generated automatically from conda-smithy. To update this configuration, +# update the conda-forge.yml and/or the recipe/meta.yaml. +# -*- mode: yaml -*- + +name: Build conda package +on: ['push', 'pull_request'] + +jobs: + build: + name: ${{ matrix.CONFIG }} + runs-on: ${{ matrix.labels }} + strategy: + fail-fast: false + matrix: + include: + - CONFIG: linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10numpy1.20python3.9.____cpython + UPLOAD_PACKAGES: True + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.2 + os: ubuntu + labels: + - linux + - x64 + - self-hosted + - 'cirun-openstack-gpu--${{ github.run_id }}linux_64_c_compiler_version10cuda_c_h2a2c77454b' + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Build on Linux + if: matrix.os == 'ubuntu' + env: + CONFIG: ${{ matrix.CONFIG }} + UPLOAD_PACKAGES: ${{ matrix.UPLOAD_PACKAGES }} + DOCKER_IMAGE: ${{ matrix.DOCKER_IMAGE }} + CI: github_actions + UPLOAD_ON_BRANCH: main + BINSTAR_TOKEN: ${{ secrets.BINSTAR_TOKEN }} + FEEDSTOCK_TOKEN: ${{ secrets.FEEDSTOCK_TOKEN }} + STAGING_BINSTAR_TOKEN: ${{ secrets.STAGING_BINSTAR_TOKEN }} + shell: bash + run: | + echo "::group::Configure binfmt_misc" + docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes + export FEEDSTOCK_NAME="$(basename $GITHUB_REPOSITORY)" + export GIT_BRANCH="$(basename $GITHUB_REF)" + if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then + export IS_PR_BUILD="True" + else + export IS_PR_BUILD="False" + fi + echo "::endgroup::" + ./.scripts/run_docker_build.sh + + - name: Build on macOS + if: matrix.os == 'macos' + env: + CONFIG: ${{ matrix.CONFIG }} + UPLOAD_PACKAGES: ${{ matrix.UPLOAD_PACKAGES }} + CI: github_actions + UPLOAD_ON_BRANCH: main + BINSTAR_TOKEN: ${{ secrets.BINSTAR_TOKEN }} + FEEDSTOCK_TOKEN: ${{ secrets.FEEDSTOCK_TOKEN }} + STAGING_BINSTAR_TOKEN: ${{ secrets.STAGING_BINSTAR_TOKEN }} + shell: bash + run: | + export FEEDSTOCK_NAME="$(basename $GITHUB_REPOSITORY)" + export GIT_BRANCH="$(basename $GITHUB_REF)" + if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then + export IS_PR_BUILD="True" + else + export IS_PR_BUILD="False" + fi + ./.scripts/run_osx_build.sh + + - name: Install Miniconda for windows + uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-version: latest + miniforge-variant: Mambaforge + if: matrix.os == 'windows' + - name: Build on windows + shell: cmd + run: | + call activate base + mamba.exe install -c conda-forge 'python=3.9' conda-build conda pip boa conda-forge-ci-setup=3 + if errorlevel 1 exit 1 + setup_conda_rc .\ ".\recipe" .\.ci_support\%CONFIG%.yaml + if errorlevel 1 exit 1 + + + if EXIST LICENSE.txt ( + copy LICENSE.txt "recipe\\recipe-scripts-license.txt" + ) + conda.exe mambabuild "recipe" -m .ci_support\%CONFIG%.yaml + if errorlevel 1 exit 1 + set "FEEDSTOCK_NAME=%GITHUB_REPOSITORY:*/=%" + set "GIT_BRANCH=%GITHUB_REF:refs/heads/=%" + if /i "%GITHUB_EVENT_NAME%" == "pull_request" ( + set "IS_PR_BUILD=True" + ) else ( + set "IS_PR_BUILD=False" + ) + validate_recipe_outputs "%FEEDSTOCK_NAME%" + if errorlevel 1 exit 1 + if /i "%UPLOAD_PACKAGES%" == "true" ( + if /i "%IS_PR_BUILD%" == "false" ( + upload_package --validate --feedstock-name="%FEEDSTOCK_NAME%" .\ ".\recipe" .ci_support\%CONFIG%.yaml + ) + ) + env: + PYTHONUNBUFFERED: 1 + CONFIG: ${{ matrix.CONFIG }} + CI: github_actions + UPLOAD_PACKAGES: ${{ matrix.UPLOAD_PACKAGES }} + UPLOAD_ON_BRANCH: main + BINSTAR_TOKEN: ${{ secrets.BINSTAR_TOKEN }} + FEEDSTOCK_TOKEN: ${{ secrets.FEEDSTOCK_TOKEN }} + STAGING_BINSTAR_TOKEN: ${{ secrets.STAGING_BINSTAR_TOKEN }} + if: matrix.os == 'windows' \ No newline at end of file diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 71a4242a..595f8b5e 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -33,9 +33,9 @@ CONDARC mamba install --update-specs --yes --quiet --channel conda-forge \ - conda-build pip boa conda-forge-ci-setup=3 "py-lief<0.12" + conda-build pip boa conda-forge-ci-setup=3 mamba update --update-specs --yes --quiet --channel conda-forge \ - conda-build pip boa conda-forge-ci-setup=3 "py-lief<0.12" + conda-build pip boa conda-forge-ci-setup=3 # set up the condarc setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" diff --git a/.scripts/create_conda_build_artifacts.sh b/.scripts/create_conda_build_artifacts.sh deleted file mode 100755 index cba0faee..00000000 --- a/.scripts/create_conda_build_artifacts.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env bash - -# INPUTS (environment variables that need to be set before calling this script): -# -# CI (azure/github_actions/UNSET) -# CI_RUN_ID (unique identifier for the CI job run) -# FEEDSTOCK_NAME -# CONFIG (build matrix configuration string) -# SHORT_CONFIG (uniquely-shortened configuration string) -# CONDA_BLD_DIR (path to the conda-bld directory) -# ARTIFACT_STAGING_DIR (use working directory if unset) -# BLD_ARTIFACT_PREFIX (prefix for the conda build artifact name, skip if unset) -# ENV_ARTIFACT_PREFIX (prefix for the conda build environments artifact name, skip if unset) - -# OUTPUTS -# -# BLD_ARTIFACT_NAME -# BLD_ARTIFACT_PATH -# ENV_ARTIFACT_NAME -# ENV_ARTIFACT_PATH - -source .scripts/logging_utils.sh - -# DON'T do set -x, because it results in double echo-ing pipeline commands -# and that might end up inserting extraneous quotation marks in output variables -set -e - -# Check that the conda-build directory exists -if [ ! -d "$CONDA_BLD_DIR" ]; then - echo "conda-build directory does not exist" - exit 1 -fi - -# Set staging dir to the working dir, in Windows style if applicable -if [[ -z "${ARTIFACT_STAGING_DIR}" ]]; then - if pwd -W; then - ARTIFACT_STAGING_DIR=$(pwd -W) - else - ARTIFACT_STAGING_DIR=$PWD - fi -fi -echo "ARTIFACT_STAGING_DIR: $ARTIFACT_STAGING_DIR" - -FEEDSTOCK_ROOT=$(cd "$(dirname "$0")/.."; pwd;) -if [ -z ${FEEDSTOCK_NAME} ]; then - export FEEDSTOCK_NAME=$(basename ${FEEDSTOCK_ROOT}) -fi - -# Set a unique ID for the artifact(s), specialized for this particular job run -ARTIFACT_UNIQUE_ID="${CI_RUN_ID}_${CONFIG}" -if [[ ${#ARTIFACT_UNIQUE_ID} -gt 80 ]]; then - ARTIFACT_UNIQUE_ID="${CI_RUN_ID}_${SHORT_CONFIG}" -fi -echo "ARTIFACT_UNIQUE_ID: $ARTIFACT_UNIQUE_ID" - -# Set a descriptive ID for the archive(s), specialized for this particular job run -ARCHIVE_UNIQUE_ID="${CI_RUN_ID}_${CONFIG}" - -# Make the build artifact zip -if [[ ! -z "$BLD_ARTIFACT_PREFIX" ]]; then - export BLD_ARTIFACT_NAME="${BLD_ARTIFACT_PREFIX}_${ARTIFACT_UNIQUE_ID}" - export BLD_ARTIFACT_PATH="${ARTIFACT_STAGING_DIR}/${FEEDSTOCK_NAME}_${BLD_ARTIFACT_PREFIX}_${ARCHIVE_UNIQUE_ID}.zip" - - ( startgroup "Archive conda build directory" ) 2> /dev/null - - # Try 7z and fall back to zip if it fails (for cross-platform use) - if ! 7z a "$BLD_ARTIFACT_PATH" "$CONDA_BLD_DIR" '-xr!.git/' '-xr!_*_env*/' '-xr!*_cache/' -bb; then - pushd "$CONDA_BLD_DIR" - zip -r -y -T "$BLD_ARTIFACT_PATH" . -x '*.git/*' '*_*_env*/*' '*_cache/*' - popd - fi - - ( endgroup "Archive conda build directory" ) 2> /dev/null - - echo "BLD_ARTIFACT_NAME: $BLD_ARTIFACT_NAME" - echo "BLD_ARTIFACT_PATH: $BLD_ARTIFACT_PATH" - - if [[ "$CI" == "azure" ]]; then - echo "##vso[task.setVariable variable=BLD_ARTIFACT_NAME]$BLD_ARTIFACT_NAME" - echo "##vso[task.setVariable variable=BLD_ARTIFACT_PATH]$BLD_ARTIFACT_PATH" - elif [[ "$CI" == "github_actions" ]]; then - echo "::set-output name=BLD_ARTIFACT_NAME::$BLD_ARTIFACT_NAME" - echo "::set-output name=BLD_ARTIFACT_PATH::$BLD_ARTIFACT_PATH" - fi -fi - -# Make the environments artifact zip -if [[ ! -z "$ENV_ARTIFACT_PREFIX" ]]; then - export ENV_ARTIFACT_NAME="${ENV_ARTIFACT_PREFIX}_${ARTIFACT_UNIQUE_ID}" - export ENV_ARTIFACT_PATH="${ARTIFACT_STAGING_DIR}/${FEEDSTOCK_NAME}_${ENV_ARTIFACT_PREFIX}_${ARCHIVE_UNIQUE_ID}.zip" - - ( startgroup "Archive conda build environments" ) 2> /dev/null - - # Try 7z and fall back to zip if it fails (for cross-platform use) - if ! 7z a "$ENV_ARTIFACT_PATH" -r "$CONDA_BLD_DIR"/'_*_env*/' -bb; then - pushd "$CONDA_BLD_DIR" - zip -r -y -T "$ENV_ARTIFACT_PATH" . -i '*_*_env*/*' - popd - fi - - ( endgroup "Archive conda build environments" ) 2> /dev/null - - echo "ENV_ARTIFACT_NAME: $ENV_ARTIFACT_NAME" - echo "ENV_ARTIFACT_PATH: $ENV_ARTIFACT_PATH" - - if [[ "$CI" == "azure" ]]; then - echo "##vso[task.setVariable variable=ENV_ARTIFACT_NAME]$ENV_ARTIFACT_NAME" - echo "##vso[task.setVariable variable=ENV_ARTIFACT_PATH]$ENV_ARTIFACT_PATH" - elif [[ "$CI" == "github_actions" ]]; then - echo "::set-output name=ENV_ARTIFACT_NAME::$ENV_ARTIFACT_NAME" - echo "::set-output name=ENV_ARTIFACT_PATH::$ENV_ARTIFACT_PATH" - fi -fi \ No newline at end of file diff --git a/README.md b/README.md index 5cdfd24b..2fa43041 100644 --- a/README.md +++ b/README.md @@ -27,31 +27,6 @@ Current build status - - - - -
Azure -
- - - - - - - - - - - - -
VariantStatus
linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10numpy1.20python3.9.____cpython - - variant - -
-
-
Current release info diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index ad85a2cc..00000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,6 +0,0 @@ -# This file was generated automatically from conda-smithy. To update this configuration, -# update the conda-forge.yml and/or the recipe/meta.yaml. -# -*- mode: yaml -*- - -jobs: - - template: ./.azure-pipelines/azure-pipelines-linux.yml \ No newline at end of file diff --git a/cirun.yml b/cirun.yml new file mode 100755 index 00000000..3ca1e095 --- /dev/null +++ b/cirun.yml @@ -0,0 +1,8 @@ +runners: + - # Custom runner 1 + name: cirun-openstack-gpu + labels: ['cirun-openstack-gpu'] + cloud: openstack + instance_type: gpu_tiny + machine_image: ubuntu-focal-nvidia-14112022 + region: RegionOne \ No newline at end of file diff --git a/conda-forge.yml b/conda-forge.yml index 8377488f..6e348b43 100644 --- a/conda-forge.yml +++ b/conda-forge.yml @@ -5,9 +5,13 @@ github: branch_name: main tooling_branch_name: main provider: - linux_ppc64le: travis -azure: - store_build_artifacts: true + linux: + - github_actions conda_build: pkg_format: 2 upload_on_branch: main +github_actions: + self_hosted: true + self_hosted_triggers: ["push", "pull_request"] + self_hosted_labels: + - cirun-openstack-gpu diff --git a/recipe/HOWTO.md b/recipe/HOWTO.md index 212e8237..1c0a1a43 100644 --- a/recipe/HOWTO.md +++ b/recipe/HOWTO.md @@ -2,3 +2,18 @@ > Work in progress +On `conda-forge.yml`, add: + +```yaml +provider: + linux: + - azure # if you want to keep azure builds, not needed + - github_actions # REQUIRED +github_actions: + self_hosted: true + self_hosted_labels: + - cirun-openstack-gpu +``` + +And rerender! + From c86bbeda5f02335fe32d0b560089daa31aec5fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Tue, 27 Dec 2022 11:52:35 +0100 Subject: [PATCH 03/71] rename cirun temporarily --- cirun.yml => .cirun.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cirun.yml => .cirun.yml (100%) diff --git a/cirun.yml b/.cirun.yml similarity index 100% rename from cirun.yml rename to .cirun.yml From e7604c1eeb38978671d4214aace25027a88ffdcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Tue, 27 Dec 2022 11:55:00 +0100 Subject: [PATCH 04/71] use extended yaml syntax --- .cirun.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.cirun.yml b/.cirun.yml index 3ca1e095..7791f60d 100755 --- a/.cirun.yml +++ b/.cirun.yml @@ -1,8 +1,9 @@ runners: - # Custom runner 1 name: cirun-openstack-gpu - labels: ['cirun-openstack-gpu'] + labels: + - cirun-openstack-gpu cloud: openstack instance_type: gpu_tiny machine_image: ubuntu-focal-nvidia-14112022 - region: RegionOne \ No newline at end of file + region: RegionOne From 55807e90dc018112dd0db39b4e5956f2e5491d8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Tue, 27 Dec 2022 12:03:32 +0100 Subject: [PATCH 05/71] try to match labels --- .cirun.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.cirun.yml b/.cirun.yml index 7791f60d..931ec950 100755 --- a/.cirun.yml +++ b/.cirun.yml @@ -2,6 +2,9 @@ runners: - # Custom runner 1 name: cirun-openstack-gpu labels: + - linux + - x64 + - self-hosted - cirun-openstack-gpu cloud: openstack instance_type: gpu_tiny From 45acbf27472518de1bff9e9fcd3ed7238f997866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Tue, 27 Dec 2022 13:12:27 +0100 Subject: [PATCH 06/71] Some more docs [ci skip] --- recipe/HOWTO.md | 86 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/recipe/HOWTO.md b/recipe/HOWTO.md index 1c0a1a43..e9a312a8 100644 --- a/recipe/HOWTO.md +++ b/recipe/HOWTO.md @@ -2,6 +2,76 @@ > Work in progress +# How to register the cloud provider (only once) + +Cirun needs the OpenStack GPU provider added. +We can use the web dashboard manually, or the [`cirun-py`](https://github.com/AktechLabs/cirun-py) tool: + +``` +$ conda create -n cirun -c conda-forge cirun +$ conda activate cirun +$ cirun --help +``` + +Then, get an API token from https://cirun.io/admin/api +and export it: + +``` +export CIRUN_API_KEY= +``` + +Now you can use this command to register the cloud provider: + +``` +$ cirun cloud connect openstack \ + --username XXXXXX \ + --password XXXXXX \ + --auth-url XXXXXX \ + --project-id XXXXXXX \ + --domain-id XXXXXXX \ + --network XXXXXX +``` + +This only needs to be done once for the entire organization. +Credentials are in the vault. + +## Giving access to repos + +First, the given repo must be given access to the Cirun app via +this [API endpoint](https://docs.github.com/en/rest/apps/installations?apiVersion=2022-11-28#add-a-repository-to-an-app-installation +): + +``` +PUT +/user/installations/{installation_id}/repositories/{repository_id} +``` + +`installation_id` is 18453316. It can be obtained from the [Organization settings> GitHub Apps](https://github.com/organizations/conda-forge/settings/installations) panel. Click on "Cirun Application" and get the identifier from the URL. + + +Then, on the Cirun end, the repo nees to be enabled. `cirun-py` tool makes this easy: + +``` +$ cirun repo add conda-forge/cf-autotick-bot-test-package-feedstock +``` + +### Removing access + +Very similar! On the Github side: + +``` +DELETE +/user/installations/{installation_id}/repositories/{repository_id} +``` + +On the Cirun side: + +``` +$ cirun repo remove conda-forge/cf-autotick-bot-test-package-feedstock +``` + +## Configure the feedstock + On `conda-forge.yml`, add: ```yaml @@ -12,8 +82,20 @@ provider: github_actions: self_hosted: true self_hosted_labels: - - cirun-openstack-gpu + - cirun-openstack-gpu # having a cirun-* label will add the Cirun configs! ``` -And rerender! +And rerender! This is what should happen after you open the new PR: + +1. A new Github Actions entry is visible in the CI, waiting for a self-hosted runner for around two minutes while Cirun sets things up. +2. Cirun finishes provisioning and a new CI entry pops up. Details are available in Cirun.io. +3. At this point, the self-hosted runner is available and will run the workflow steps. The OpenStack UI should also report a new instance running. + +## Troubleshooting + +- Cirun never reported anything (CI entry didn't appear). There is a problem with the configuration. Check `.cirun.yml` and the GHA workflow and make sure the labels match. + +## Notes +- Only one conda-forge account should govern the Cirun configuration. We are using `conda-forge-daemon` to avoid issues with double access and other problems with multi-account configurations. This is a current limitation. +- VM Images are built at https://github.com/aktech/nvidia-openstack-image/ From 41a41548a6540c7a752e1ea765204c2f845cc55f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Wed, 4 Jan 2023 13:55:37 +0100 Subject: [PATCH 07/71] retrigger From 4b8e905e54a4d852fc3bffbada9c4367e446445c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Fri, 6 Jan 2023 12:09:46 +0100 Subject: [PATCH 08/71] use latest vm --- .cirun.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirun.yml b/.cirun.yml index 931ec950..1bb97e64 100755 --- a/.cirun.yml +++ b/.cirun.yml @@ -8,5 +8,5 @@ runners: - cirun-openstack-gpu cloud: openstack instance_type: gpu_tiny - machine_image: ubuntu-focal-nvidia-14112022 + machine_image: ubuntu-2204-nvidia-docker-20221229 region: RegionOne From 7901e9524e46b915be266cc181232621ccd6c3ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Fri, 6 Jan 2023 12:26:52 +0100 Subject: [PATCH 09/71] use gpu_medium --- .cirun.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirun.yml b/.cirun.yml index 1bb97e64..2e9cac0f 100755 --- a/.cirun.yml +++ b/.cirun.yml @@ -7,6 +7,6 @@ runners: - self-hosted - cirun-openstack-gpu cloud: openstack - instance_type: gpu_tiny + instance_type: gpu_medium machine_image: ubuntu-2204-nvidia-docker-20221229 region: RegionOne From a7dff98fe93e5c6c9d8a4575daf8e0f1e5a98518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 9 Jan 2023 11:18:10 +0100 Subject: [PATCH 10/71] rerender with smithy-dev 5f96e5ef --- .cirun.yml | 13 ++++++++----- .github/workflows/conda-build.yml | 8 +++++--- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.cirun.yml b/.cirun.yml index 2e9cac0f..c056827b 100755 --- a/.cirun.yml +++ b/.cirun.yml @@ -1,12 +1,15 @@ runners: - # Custom runner 1 name: cirun-openstack-gpu - labels: - - linux - - x64 - - self-hosted - - cirun-openstack-gpu + labels: ['linux', 'x64', 'self-hosted', 'cirun-openstack-gpu'] cloud: openstack instance_type: gpu_medium machine_image: ubuntu-2204-nvidia-docker-20221229 region: RegionOne + - # Custom runner 2 + name: cirun-openstack-cpu + labels: ['linux', 'x64', 'self-hosted', 'cirun-openstack-cpu'] + cloud: openstack + instance_type: ci_medium + machine_image: ubuntu-2204-cloud-jammy-20221104 + region: RegionOne \ No newline at end of file diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index 880d6bfe..a0b5d369 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -15,13 +15,14 @@ jobs: include: - CONFIG: linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10numpy1.20python3.9.____cpython UPLOAD_PACKAGES: True - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.2 os: ubuntu labels: + - self-hosted - linux - x64 - - self-hosted - - 'cirun-openstack-gpu--${{ github.run_id }}linux_64_c_compiler_version10cuda_c_h2a2c77454b' + - cirun-openstack-gpu--${{ github.run_id }}-linux_64_c_compiler_version10cuda_c_h2a2c77454b + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.2 + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" steps: - name: Checkout code uses: actions/checkout@v3 @@ -34,6 +35,7 @@ jobs: DOCKER_IMAGE: ${{ matrix.DOCKER_IMAGE }} CI: github_actions UPLOAD_ON_BRANCH: main + CONDA_FORGE_DOCKER_RUN_ARGS: "${{ matrix.CONDA_FORGE_DOCKER_RUN_ARGS }}" BINSTAR_TOKEN: ${{ secrets.BINSTAR_TOKEN }} FEEDSTOCK_TOKEN: ${{ secrets.FEEDSTOCK_TOKEN }} STAGING_BINSTAR_TOKEN: ${{ secrets.STAGING_BINSTAR_TOKEN }} From 7929ebc0996d2d085ffb30dcca5f5a8abce6253f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Tue, 10 Jan 2023 18:50:33 +0100 Subject: [PATCH 11/71] Update .cirun.yml Co-authored-by: Amit Kumar --- .cirun.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirun.yml b/.cirun.yml index c056827b..bbf0ff7f 100755 --- a/.cirun.yml +++ b/.cirun.yml @@ -4,7 +4,7 @@ runners: labels: ['linux', 'x64', 'self-hosted', 'cirun-openstack-gpu'] cloud: openstack instance_type: gpu_medium - machine_image: ubuntu-2204-nvidia-docker-20221229 + machine_image: ubuntu-2204-nvidia-container-runtime-20230107 region: RegionOne - # Custom runner 2 name: cirun-openstack-cpu From 04868823fcda62c023a11b7276e7986434284e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Tue, 10 Jan 2023 18:54:40 +0100 Subject: [PATCH 12/71] retrigger From 86e64981147fbd6a13d28389043cf5f4744fd25c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Tue, 10 Jan 2023 20:35:27 +0100 Subject: [PATCH 13/71] patch mtu? --- .scripts/run_docker_build.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index 92362398..fa3b8362 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -21,6 +21,22 @@ if [ -z ${FEEDSTOCK_NAME} ]; then export FEEDSTOCK_NAME=$(basename ${FEEDSTOCK_ROOT}) fi +# MTU PATCH +python3 < Date: Tue, 10 Jan 2023 20:52:21 +0100 Subject: [PATCH 14/71] sudo it --- .scripts/run_docker_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index fa3b8362..56a90854 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -22,7 +22,7 @@ if [ -z ${FEEDSTOCK_NAME} ]; then fi # MTU PATCH -python3 < Date: Tue, 10 Jan 2023 21:04:47 +0100 Subject: [PATCH 15/71] use gpu_large? --- .cirun.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirun.yml b/.cirun.yml index bbf0ff7f..d4d9e851 100755 --- a/.cirun.yml +++ b/.cirun.yml @@ -3,7 +3,7 @@ runners: name: cirun-openstack-gpu labels: ['linux', 'x64', 'self-hosted', 'cirun-openstack-gpu'] cloud: openstack - instance_type: gpu_medium + instance_type: gpu_large machine_image: ubuntu-2204-nvidia-container-runtime-20230107 region: RegionOne - # Custom runner 2 From e3e27ea5ff7ecad950cd6f60b812a5d9112b8621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 23 Jan 2023 16:41:56 +0100 Subject: [PATCH 16/71] try cupy + skip most + rerender --- ...piler_version10python3.9.____cpython.yaml} | 13 +- .cirun.yml | 2 +- .github/CODEOWNERS | 2 +- .github/workflows/conda-build.yml | 4 +- .scripts/run_docker_build.sh | 16 -- README.md | 70 +++--- recipe/HOWTO.md | 101 --------- recipe/activate.bat | 10 + recipe/activate.sh | 8 + recipe/build_openmm.sh | 85 ------- recipe/conda_build_config.yaml | 40 ++++ recipe/deactivate.bat | 11 + recipe/deactivate.sh | 9 + recipe/meta.yaml | 207 ++++++++++-------- .../linux64_cuda10.2_wheel.json | 7 + .../linux64_cuda11.0_wheel.json | 7 + .../linux64_cuda11.1_wheel.json | 7 + .../linux64_cuda11.2_wheel.json | 7 + .../preload_config/linux64_cuda11_wheel.json | 7 + .../preload_config/win64_cuda10.2_wheel.json | 6 + .../preload_config/win64_cuda11.0_wheel.json | 6 + .../preload_config/win64_cuda11.1_wheel.json | 6 + .../preload_config/win64_cuda11.2_wheel.json | 6 + recipe/preload_config/win64_cuda11_wheel.json | 6 + recipe/run_test.py | 57 +++++ recipe/test_openmm.sh | 91 -------- 26 files changed, 354 insertions(+), 437 deletions(-) rename .ci_support/{linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10numpy1.20python3.9.____cpython.yaml => linux_64_c_compiler_version10cuda_compiler_version11.2cudnn8cxx_compiler_version10python3.9.____cpython.yaml} (91%) delete mode 100644 recipe/HOWTO.md create mode 100755 recipe/activate.bat create mode 100755 recipe/activate.sh delete mode 100644 recipe/build_openmm.sh create mode 100644 recipe/conda_build_config.yaml create mode 100755 recipe/deactivate.bat create mode 100755 recipe/deactivate.sh mode change 100644 => 100755 recipe/meta.yaml create mode 100644 recipe/preload_config/linux64_cuda10.2_wheel.json create mode 100644 recipe/preload_config/linux64_cuda11.0_wheel.json create mode 100644 recipe/preload_config/linux64_cuda11.1_wheel.json create mode 100644 recipe/preload_config/linux64_cuda11.2_wheel.json create mode 100644 recipe/preload_config/linux64_cuda11_wheel.json create mode 100644 recipe/preload_config/win64_cuda10.2_wheel.json create mode 100644 recipe/preload_config/win64_cuda11.0_wheel.json create mode 100644 recipe/preload_config/win64_cuda11.1_wheel.json create mode 100644 recipe/preload_config/win64_cuda11.2_wheel.json create mode 100644 recipe/preload_config/win64_cuda11_wheel.json create mode 100644 recipe/run_test.py delete mode 100644 recipe/test_openmm.sh diff --git a/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10numpy1.20python3.9.____cpython.yaml b/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cudnn8cxx_compiler_version10python3.9.____cpython.yaml similarity index 91% rename from .ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10numpy1.20python3.9.____cpython.yaml rename to .ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cudnn8cxx_compiler_version10python3.9.____cpython.yaml index d0649369..6b73dc59 100644 --- a/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10numpy1.20python3.9.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cudnn8cxx_compiler_version10python3.9.____cpython.yaml @@ -12,16 +12,18 @@ cuda_compiler: - nvcc cuda_compiler_version: - '11.2' +cudnn: +- '8' +cutensor: +- '1' cxx_compiler: - gxx cxx_compiler_version: - '10' docker_image: - quay.io/condaforge/linux-anvil-cuda:11.2 -fftw: -- '3' -numpy: -- '1.20' +nccl: +- '2' pin_run_as_build: python: min_pin: x.x @@ -33,8 +35,7 @@ target_platform: zip_keys: - - c_compiler_version - cxx_compiler_version + - cudnn - cuda_compiler_version - cdt_name - docker_image -- - python - - numpy diff --git a/.cirun.yml b/.cirun.yml index d4d9e851..bbf0ff7f 100755 --- a/.cirun.yml +++ b/.cirun.yml @@ -3,7 +3,7 @@ runners: name: cirun-openstack-gpu labels: ['linux', 'x64', 'self-hosted', 'cirun-openstack-gpu'] cloud: openstack - instance_type: gpu_large + instance_type: gpu_medium machine_image: ubuntu-2204-nvidia-container-runtime-20230107 region: RegionOne - # Custom runner 2 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 99ba51ba..146f0876 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @jaimergp @jchodera @mikemhenry @peastman \ No newline at end of file +* @asi1024 @emcastillo @jakirkham @kmaehashi @leofang @toslunar \ No newline at end of file diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index a0b5d369..08c254a4 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -13,14 +13,14 @@ jobs: fail-fast: false matrix: include: - - CONFIG: linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10numpy1.20python3.9.____cpython + - CONFIG: linux_64_c_compiler_version10cuda_compiler_version11.2cudnn8cxx_compiler_version10python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu labels: - self-hosted - linux - x64 - - cirun-openstack-gpu--${{ github.run_id }}-linux_64_c_compiler_version10cuda_c_h2a2c77454b + - cirun-openstack-gpu--${{ github.run_id }}-linux_64_c_compiler_version10cuda_c_h4b2e19d586 DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.2 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" steps: diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index 56a90854..92362398 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -21,22 +21,6 @@ if [ -z ${FEEDSTOCK_NAME} ]; then export FEEDSTOCK_NAME=$(basename ${FEEDSTOCK_ROOT}) fi -# MTU PATCH -sudo python3 < Work in progress - -# How to register the cloud provider (only once) - -Cirun needs the OpenStack GPU provider added. -We can use the web dashboard manually, or the [`cirun-py`](https://github.com/AktechLabs/cirun-py) tool: - -``` -$ conda create -n cirun -c conda-forge cirun -$ conda activate cirun -$ cirun --help -``` - -Then, get an API token from https://cirun.io/admin/api -and export it: - -``` -export CIRUN_API_KEY= -``` - -Now you can use this command to register the cloud provider: - -``` -$ cirun cloud connect openstack \ - --username XXXXXX \ - --password XXXXXX \ - --auth-url XXXXXX \ - --project-id XXXXXXX \ - --domain-id XXXXXXX \ - --network XXXXXX -``` - -This only needs to be done once for the entire organization. -Credentials are in the vault. - -## Giving access to repos - -First, the given repo must be given access to the Cirun app via -this [API endpoint](https://docs.github.com/en/rest/apps/installations?apiVersion=2022-11-28#add-a-repository-to-an-app-installation -): - -``` -PUT -/user/installations/{installation_id}/repositories/{repository_id} -``` - -`installation_id` is 18453316. It can be obtained from the [Organization settings> GitHub Apps](https://github.com/organizations/conda-forge/settings/installations) panel. Click on "Cirun Application" and get the identifier from the URL. - - -Then, on the Cirun end, the repo nees to be enabled. `cirun-py` tool makes this easy: - -``` -$ cirun repo add conda-forge/cf-autotick-bot-test-package-feedstock -``` - -### Removing access - -Very similar! On the Github side: - -``` -DELETE -/user/installations/{installation_id}/repositories/{repository_id} -``` - -On the Cirun side: - -``` -$ cirun repo remove conda-forge/cf-autotick-bot-test-package-feedstock -``` - -## Configure the feedstock - -On `conda-forge.yml`, add: - -```yaml -provider: - linux: - - azure # if you want to keep azure builds, not needed - - github_actions # REQUIRED -github_actions: - self_hosted: true - self_hosted_labels: - - cirun-openstack-gpu # having a cirun-* label will add the Cirun configs! -``` - -And rerender! This is what should happen after you open the new PR: - -1. A new Github Actions entry is visible in the CI, waiting for a self-hosted runner for around two minutes while Cirun sets things up. -2. Cirun finishes provisioning and a new CI entry pops up. Details are available in Cirun.io. -3. At this point, the self-hosted runner is available and will run the workflow steps. The OpenStack UI should also report a new instance running. - -## Troubleshooting - -- Cirun never reported anything (CI entry didn't appear). There is a problem with the configuration. Check `.cirun.yml` and the GHA workflow and make sure the labels match. - -## Notes - -- Only one conda-forge account should govern the Cirun configuration. We are using `conda-forge-daemon` to avoid issues with double access and other problems with multi-account configurations. This is a current limitation. -- VM Images are built at https://github.com/aktech/nvidia-openstack-image/ diff --git a/recipe/activate.bat b/recipe/activate.bat new file mode 100755 index 00000000..611d7027 --- /dev/null +++ b/recipe/activate.bat @@ -0,0 +1,10 @@ +@echo off +if defined CONDA_BUILD_STATE ( + @echo on +) + +if defined CUDA_PATH ( + set "CONDA_CUPY_CUDA_PATH=%CUDA_PATH%" +) + +set "CUDA_PATH=%CONDA_PREFIX%" diff --git a/recipe/activate.sh b/recipe/activate.sh new file mode 100755 index 00000000..265ac99e --- /dev/null +++ b/recipe/activate.sh @@ -0,0 +1,8 @@ +#!/bin/bash + + +if [[ -n ${CUDA_PATH:+x} ]]; then + export CONDA_CUPY_CUDA_PATH=$CUDA_PATH +fi + +export CUDA_PATH=$CONDA_PREFIX diff --git a/recipe/build_openmm.sh b/recipe/build_openmm.sh deleted file mode 100644 index 1ad92c63..00000000 --- a/recipe/build_openmm.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash - -set -ex - - -CMAKE_FLAGS="${CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${PREFIX} -DCMAKE_BUILD_TYPE=Release" -if [[ "$with_test_suite" == "true" ]]; then - CMAKE_FLAGS+=" -DBUILD_TESTING=ON -DOPENMM_BUILD_OPENCL_TESTS=ON" -else - CMAKE_FLAGS+=" -DBUILD_TESTING=OFF" -fi - - -if [[ "$target_platform" == linux* ]]; then - # CFLAGS - # JRG: Had to add -ldl to prevent linking errors (dlopen, etc) - MINIMAL_CFLAGS+=" -O3 -ldl" - CFLAGS+=" $MINIMAL_CFLAGS" - CXXFLAGS+=" $MINIMAL_CFLAGS" - - # CUDA is enabled in these platforms - if [[ "$target_platform" == linux-64 || "$target_platform" == linux-ppc64le ]]; then - # # CUDA_HOME is defined by nvcc metapackage - CMAKE_FLAGS+=" -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME}" - # CUDA tests won't build, disable for now - # See https://github.com/openmm/openmm/issues/2258#issuecomment-462223634 - CMAKE_FLAGS+=" -DOPENMM_BUILD_CUDA_TESTS=OFF" - # shadow some CMAKE_ARGS bits that interfere with CUDA detection - CMAKE_FLAGS+=" -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH" - fi - - # OpenCL ICD - CMAKE_FLAGS+=" -DOPENCL_INCLUDE_DIR=${PREFIX}/include" - CMAKE_FLAGS+=" -DOPENCL_LIBRARY=${PREFIX}/lib/libOpenCL${SHLIB_EXT}" - -elif [[ "$target_platform" == osx* ]]; then - if [[ "$opencl_impl" == khronos ]]; then - CMAKE_FLAGS+=" -DOPENCL_INCLUDE_DIR=${PREFIX}/include" - CMAKE_FLAGS+=" -DOPENCL_LIBRARY=${PREFIX}/lib/libOpenCL${SHLIB_EXT}" - fi - # When using opencl_impl == apple, CMake will auto-locate it, so no need to provide the flags - # On Conda Forge, this will result in: - # /Applications/Xcode_12.app/Contents/Developer/Platforms/MacOSX.platform/Developer/... - # ...SDKs/MacOSX10.9.sdk/System/Library/Frameworks/OpenCL.framework - # On local builds, it might be: - # /System/Library/Frameworks/OpenCL.framework/OpenCL -fi - -# Set location for FFTW3 on both linux and mac -CMAKE_FLAGS+=" -DFFTW_INCLUDES=${PREFIX}/include/" -CMAKE_FLAGS+=" -DFFTW_LIBRARY=${PREFIX}/lib/libfftw3f${SHLIB_EXT}" -CMAKE_FLAGS+=" -DFFTW_THREADS_LIBRARY=${PREFIX}/lib/libfftw3f_threads${SHLIB_EXT}" -# Disambiguate swig location -CMAKE_FLAGS+=" -DSWIG_EXECUTABLE=$(which swig)" - -# Build in subdirectory and install. -mkdir -p build -cd build -cmake ${CMAKE_FLAGS} ${SRC_DIR} -make -j$CPU_COUNT -make -j$CPU_COUNT install PythonInstall - -# Put examples into an appropriate subdirectory. -mkdir -p ${PREFIX}/share/openmm/ -mv ${PREFIX}/examples ${PREFIX}/share/openmm/ - -# Fix some overlinking warnings/errors -for lib in ${PREFIX}/lib/plugins/*${SHLIB_EXT}; do - ln -s $lib ${PREFIX}/lib/$(basename $lib) || true -done - -if [[ "$with_test_suite" == "true" ]]; then - mkdir -p ${PREFIX}/share/openmm/tests/ - # BSD find vs GNU find: -executable is only available in GNU find - # +0111 is somehow equivalent in BSD, but that's not compatible in GNU - # so we use different commands for each... - if [[ "$target_platform" == osx* ]]; then - find . -name "Test*" -perm +0111 -type f \ - -exec python $RECIPE_DIR/patch_osx_tests.py "{}" \; \ - -exec cp "{}" $PREFIX/share/openmm/tests/ \; - else - find . -name "Test*" -executable -type f -exec cp "{}" $PREFIX/share/openmm/tests/ \; - fi - cp -r python/tests $PREFIX/share/openmm/tests/python -fi diff --git a/recipe/conda_build_config.yaml b/recipe/conda_build_config.yaml new file mode 100644 index 00000000..b427a0ed --- /dev/null +++ b/recipe/conda_build_config.yaml @@ -0,0 +1,40 @@ +arm_variant_type: # [aarch64] + - sbsa # [aarch64] + +# The added keys below are needed to enable ppc64le/aarch64 CUDA builds; the linux64 +# and win64 builds are already handled in the conda-forge-pinning feedstock. Once we +# migrate to enable CUDA builds on all supported platforms, these can be removed too. + +docker_image: # [os.environ.get("BUILD_PLATFORM", "").startswith("linux")] + - quay.io/condaforge/linux-anvil-aarch64-cuda:11.0 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] + - quay.io/condaforge/linux-anvil-aarch64-cuda:11.1 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] + +c_compiler_version: # [linux] + - 9 # [aarch64] + - 9 # [aarch64] +cxx_compiler_version: # [linux] + - 9 # [aarch64] + - 9 # [aarch64] +fortran_compiler_version: # [linux] + - 9 # [aarch64] + - 9 # [aarch64] + +cuda_compiler: + - nvcc + +cuda_compiler_version: + - 11.0 # [aarch64] + - 11.1 # [aarch64] + +cudnn: + - 8 # [aarch64] + - 8 # [aarch64] + +cdt_name: # [linux] + - cos7 # [aarch64] + - cos7 # [aarch64] + +c_compiler: # [win] + - vs2017 # [win] +cxx_compiler: # [win] + - vs2017 # [win] diff --git a/recipe/deactivate.bat b/recipe/deactivate.bat new file mode 100755 index 00000000..dd9bd2e5 --- /dev/null +++ b/recipe/deactivate.bat @@ -0,0 +1,11 @@ +@echo off +if defined CONDA_BUILD_STATE ( + @echo on +) + +if not "%CONDA_CUPY_CUDA_PATH%"=="" ( + set "CUDA_PATH=%CONDA_CUPY_CUDA_PATH%" + set "CONDA_CUPY_CUDA_PATH=" +) else ( + set CUDA_PATH="" +) diff --git a/recipe/deactivate.sh b/recipe/deactivate.sh new file mode 100755 index 00000000..2128f20a --- /dev/null +++ b/recipe/deactivate.sh @@ -0,0 +1,9 @@ +#!/bin/bash + + +if [[ -n ${CONDA_CUPY_CUDA_PATH:+x} ]]; then + export CUDA_PATH=$CONDA_CUPY_CUDA_PATH + unset CONDA_CUPY_CUDA_PATH +else + unset CUDA_PATH +fi diff --git a/recipe/meta.yaml b/recipe/meta.yaml old mode 100644 new mode 100755 index 6e8b223f..83f3da89 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,124 +1,139 @@ -{% set name = "openmm" %} -{% set version = "7.7.0" %} -{% set build = 1 %} -{% set with_test_suite = "false" %} -{% set with_test_suite = "true" %} # [(osx and arm64)] +{% set name = "cupy" %} +{% set version = "11.5.0" %} +{% set sha256 = "4bc8565bded22cc89b210fd9fb48a5d5316f30701e12bb23852a60314e1f9f6e" %} + +{% if cuda_compiler_version in (None, "None", True, False) %} +{% set cuda_major = 0 %} +{% set cuda_minor = 0 %} +{% set cuda_major_minor = (0, 0) %} +{% else %} +{% set cuda_major = environ.get("cuda_compiler_version", "11.2").split(".")[0]|int %} +{% set cuda_minor = environ.get("cuda_compiler_version", "11.2").split(".")[1]|int %} +{% set cuda_major_minor = (cuda_major, cuda_minor) %} +{% endif %} package: - name: {{ name }} + name: {{ name|lower }} version: {{ version }} source: - git_url: https://github.com/openmm/{{ name }}.git - git_rev: "7.7.0" + - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz + sha256: {{ sha256 }} build: - number: {{ build }} - string: "py{{ PY_VER.replace('.', '') }}h{{ PKG_HASH }}_{{ build }}_{{ opencl_impl }}" # [osx] - track_features: # make Apple OpenCL low priority - - openmm_opencl_apple # [osx and (opencl_impl == 'apple')] - skip: true # [win or osx or cuda_compiler_version != '11.2' or py != 39] - script_env: - - CONFIG - - CUDA_HOME # [ppc64le] - missing_dso_whitelist: - - "*/libcuda.*" # [linux64 or ppc64le] - - "*/libOpenCL.*" # [unix] - - "*/nvcuda.dll" # [win] + number: 0 + skip: true # [not (linux64 and cuda_compiler_version == '11.2' and py == 39)] script: - - env with_test_suite={{ with_test_suite }} bash {{ RECIPE_DIR }}/build_openmm.sh # [unix] - - set with_test_suite={{ with_test_suite }} && {{ RECIPE_DIR }}\build_openmm.bat # [win] + # CuPy default detects CUDA from nvcc, but on Conda-Forge's dockers nvcc lives in a different place... + # With conda-forge/nvcc-feedstock#58, CUDA_PATH is set correctly + - export NVCC=$(which nvcc) # [linux] + - echo "nvcc is $NVCC, CUDA path is $CUDA_PATH" # [linux] + - for /f "tokens=* usebackq" %%f in (`where nvcc`) do (set "dummy=%%f" && call set "NVCC=%%dummy:\=\\%%") # [win] + # we use this WAR until we figure out how to proceed with conda-forge/nvcc-feedstock#88 + - set "NVCC=%NVCC% --use-local-env" # [win64] + - echo "nvcc is %NVCC%, CUDA path is %CUDA_PATH%" # [win] + {% if cuda_major_minor >= (11, 2) %} + - export CUSPARSELT_PATH=$PREFIX # [linux64 or win] + {% endif %} + # Workaround __ieee128 error; see https://github.com/LLNL/blt/issues/341 + - export NVCC="$NVCC -Xcompiler -mno-float128" # [ppc64le] + + - {{ PYTHON }} -m pip install . --no-deps -vv + - if errorlevel 1 exit 1 # [win] + + # copy activate/deactivate scripts + - mkdir -p "${PREFIX}/etc/conda/activate.d" # [linux] + - cp "${RECIPE_DIR}/activate.sh" "${PREFIX}/etc/conda/activate.d/cupy_activate.sh" # [linux] + - mkdir -p "${PREFIX}/etc/conda/deactivate.d" # [linux] + - cp "${RECIPE_DIR}/deactivate.sh" "${PREFIX}/etc/conda/deactivate.d/cupy_deactivate.sh" # [linux] + - if not exist %PREFIX%\etc\conda\activate.d mkdir %PREFIX%\etc\conda\activate.d # [win] + - copy %RECIPE_DIR%\activate.bat %PREFIX%\etc\conda\activate.d\cupy_activate.bat # [win] + - if not exist %PREFIX%\etc\conda\deactivate.d mkdir %PREFIX%\etc\conda\deactivate.d # [win] + - copy %RECIPE_DIR%\deactivate.bat %PREFIX%\etc\conda\deactivate.d\cupy_deactivate.bat # [win] + + # enable CuPy's preload mechanism + - mkdir -p "${SP_DIR}/cupy/.data/" # [linux] + - if not exist %SP_DIR%\cupy\.data mkdir %SP_DIR%\cupy\.data # [win] + {% if cuda_major_minor >= (11, 2) %} + - cp ${RECIPE_DIR}/preload_config/linux64_cuda11_wheel.json ${SP_DIR}/cupy/.data/_wheel.json # [linux] + - copy %RECIPE_DIR%\preload_config\win64_cuda11_wheel.json %SP_DIR%\cupy\.data\_wheel.json # [win] + {% else %} + - cp ${RECIPE_DIR}/preload_config/linux64_cuda{{ cuda_compiler_version }}_wheel.json ${SP_DIR}/cupy/.data/_wheel.json # [linux] + - copy %RECIPE_DIR%\preload_config\win64_cuda{{ cuda_compiler_version }}_wheel.json %SP_DIR%\cupy\.data\_wheel.json # [win] + {% endif %} + missing_dso_whitelist: + - '*/libcuda.*' # [linux] + - '*/nvcuda.dll' # [win] + ignore_run_exports: + - cudnn + - nccl # [linux] + - cutensor + {% if cuda_major_minor >= (11, 2) %} + - cusparselt # [linux64 or win] + {% endif %} requirements: build: - - python # [build_platform != target_platform] - - cross-python_{{ target_platform }} # [build_platform != target_platform] - - cython # [build_platform != target_platform] - - numpy # [build_platform != target_platform] - - swig # [build_platform != target_platform] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - {{ compiler('cuda') }} # [linux64 or ppc64le or win] - - {{ cdt('mesa-libgl-devel') }} # [linux] - - cmake - - make # [unix] - - jom # [win] - # needed for Python wrappers - - doxygen -{% if with_test_suite == 'true' %} - - conda-build # [osx] - - m2-findutils # [win] - - m2-coreutils # [win] -{% endif %} + - {{ compiler("c") }} + - {{ compiler("cxx") }} + - {{ compiler("cuda") }} + - sysroot_linux-64 2.17 # [linux64] host: - python - - fftw - - swig - - pthread-stubs - - lxml - - numpy - - cython - # OpenCL ICD - - ocl-icd # [linux] - - khronos-opencl-icd-loader # [win or (osx and opencl_impl == 'khronos')] + - pip + - setuptools + - cython >=0.29.22,<3 + - fastrlock >=0.5 + - cudnn + - nccl >=2.8 # [linux] + - cutensor + {% if cuda_major_minor >= (11, 2) %} + - cusparselt 0.2.0.* # [linux64 or win] + {% endif %} run: - python - - fftw - - numpy - # OpenCL ICD - - ocl-icd # [linux] - - ocl-icd-system # [linux] - - khronos-opencl-icd-loader # [win or (osx and opencl_impl == 'khronos')] - - ocl_icd_wrapper_apple # [osx and opencl_impl == 'khronos'] + - {{ pin_compatible('fastrlock', max_pin='x.x') }} + - numpy >=1.18 run_constrained: - - khronos-opencl-icd-loader ==9999999999 # [osx and opencl_impl == 'apple'] - - ocl_icd_wrapper_apple ==9999999999 # [osx and opencl_impl == 'apple'] - - python <3.9.9 # [py == 39 and win] + # Only GLIBC_2.17 or older symbols present + - __glibc >=2.17 # [linux] + - scipy >=1.4 + - optuna >=2 + - {{ pin_compatible('cudnn') }} + - {{ pin_compatible('nccl') }} # [linux] + - {{ pin_compatible('cutensor', lower_bound='1.3') }} + {% if cuda_major_minor >= (11, 2) %} + - {{ pin_compatible('cusparselt', max_pin='x.x') }} # [linux64 or win] + {% endif %} test: requires: - - git - - pocl # [linux] -{% if with_test_suite == 'true' %} - pytest - - pytest-xdist -{% endif %} - files: - - test_openmm.sh # [unix] - - test_openmm.bat # [win] - imports: - - openmm - commands: - - env with_test_suite={{ with_test_suite }} bash test_openmm.sh # [unix] - - set with_test_suite={{ with_test_suite }} && .\test_openmm.bat # [win] + - {{ compiler("c") }} + - {{ compiler("cxx") }} + - {{ compiler("cuda") }} # tests need nvcc -about: - home: http://openmm.org - license: LGPL-3.0-or-later - license_family: LGPL - license_file: - - docs-source/licenses/Licenses.txt - - docs-source/licenses/LGPL.txt - - docs-source/licenses/GPL.txt - summary: A high performance toolkit for molecular simulation. + source_files: + - tests - description: | - OpenMM is a toolkit for molecular simulation. It can be used either as a - stand-alone application for running simulations, or as a library you call - from your own code. It provides a combination of extreme flexibility - (through custom forces and integrators), openness, and high performance - (especially on recent GPUs) that make it truly unique among simulation - codes. OpenMM is MIT licensed with some LGPL portions (CUDA and OpenCL - platforms). - doc_url: http://docs.openmm.org - dev_url: https://github.com/openmm/openmm +about: + home: https://cupy.dev/ + license: MIT + license_family: MIT + license_file: LICENSE + summary: | + CuPy: NumPy & SciPy for GPU + dev_url: https://github.com/cupy/cupy/ + doc_url: https://docs.cupy.dev/en/stable/ extra: recipe-maintainers: - - mikemhenry - - jchodera - - jaimergp - - peastman + - jakirkham + - leofang + - kmaehashi + - asi1024 + - emcastillo + - toslunar diff --git a/recipe/preload_config/linux64_cuda10.2_wheel.json b/recipe/preload_config/linux64_cuda10.2_wheel.json new file mode 100644 index 00000000..9928a42e --- /dev/null +++ b/recipe/preload_config/linux64_cuda10.2_wheel.json @@ -0,0 +1,7 @@ +{ + "packaging": "conda", + "cuda": "10.2", + "cudnn": {}, + "nccl": {}, + "cutensor": {} +} diff --git a/recipe/preload_config/linux64_cuda11.0_wheel.json b/recipe/preload_config/linux64_cuda11.0_wheel.json new file mode 100644 index 00000000..5583026f --- /dev/null +++ b/recipe/preload_config/linux64_cuda11.0_wheel.json @@ -0,0 +1,7 @@ +{ + "packaging": "conda", + "cuda": "11.0", + "cudnn": {}, + "nccl": {}, + "cutensor": {} +} diff --git a/recipe/preload_config/linux64_cuda11.1_wheel.json b/recipe/preload_config/linux64_cuda11.1_wheel.json new file mode 100644 index 00000000..76bfae87 --- /dev/null +++ b/recipe/preload_config/linux64_cuda11.1_wheel.json @@ -0,0 +1,7 @@ +{ + "packaging": "conda", + "cuda": "11.1", + "cudnn": {}, + "nccl": {}, + "cutensor": {} +} diff --git a/recipe/preload_config/linux64_cuda11.2_wheel.json b/recipe/preload_config/linux64_cuda11.2_wheel.json new file mode 100644 index 00000000..d8d7d96b --- /dev/null +++ b/recipe/preload_config/linux64_cuda11.2_wheel.json @@ -0,0 +1,7 @@ +{ + "packaging": "conda", + "cuda": "11.2", + "cudnn": {}, + "nccl": {}, + "cutensor": {} +} diff --git a/recipe/preload_config/linux64_cuda11_wheel.json b/recipe/preload_config/linux64_cuda11_wheel.json new file mode 100644 index 00000000..fcaa624a --- /dev/null +++ b/recipe/preload_config/linux64_cuda11_wheel.json @@ -0,0 +1,7 @@ +{ + "packaging": "conda", + "cuda": "11.x", + "cudnn": {}, + "nccl": {}, + "cutensor": {} +} diff --git a/recipe/preload_config/win64_cuda10.2_wheel.json b/recipe/preload_config/win64_cuda10.2_wheel.json new file mode 100644 index 00000000..467e72f4 --- /dev/null +++ b/recipe/preload_config/win64_cuda10.2_wheel.json @@ -0,0 +1,6 @@ +{ + "packaging": "conda", + "cuda": "10.2", + "cudnn": {}, + "cutensor": {} +} diff --git a/recipe/preload_config/win64_cuda11.0_wheel.json b/recipe/preload_config/win64_cuda11.0_wheel.json new file mode 100644 index 00000000..5ec38c63 --- /dev/null +++ b/recipe/preload_config/win64_cuda11.0_wheel.json @@ -0,0 +1,6 @@ +{ + "packaging": "conda", + "cuda": "11.0", + "cudnn": {}, + "cutensor": {} +} diff --git a/recipe/preload_config/win64_cuda11.1_wheel.json b/recipe/preload_config/win64_cuda11.1_wheel.json new file mode 100644 index 00000000..21b92276 --- /dev/null +++ b/recipe/preload_config/win64_cuda11.1_wheel.json @@ -0,0 +1,6 @@ +{ + "packaging": "conda", + "cuda": "11.1", + "cudnn": {}, + "cutensor": {} +} diff --git a/recipe/preload_config/win64_cuda11.2_wheel.json b/recipe/preload_config/win64_cuda11.2_wheel.json new file mode 100644 index 00000000..1426948a --- /dev/null +++ b/recipe/preload_config/win64_cuda11.2_wheel.json @@ -0,0 +1,6 @@ +{ + "packaging": "conda", + "cuda": "11.2", + "cudnn": {}, + "cutensor": {} +} diff --git a/recipe/preload_config/win64_cuda11_wheel.json b/recipe/preload_config/win64_cuda11_wheel.json new file mode 100644 index 00000000..e177e630 --- /dev/null +++ b/recipe/preload_config/win64_cuda11_wheel.json @@ -0,0 +1,6 @@ +{ + "packaging": "conda", + "cuda": "11.x", + "cudnn": {}, + "cutensor": {} +} diff --git a/recipe/run_test.py b/recipe/run_test.py new file mode 100644 index 00000000..a104420e --- /dev/null +++ b/recipe/run_test.py @@ -0,0 +1,57 @@ +# Configure CuPy to use 1 GPU for testing +import os +import site +import sys +os.environ["CUPY_TEST_GPU_LIMIT"] = "1" + +# Check CUDA_PATH is set +cuda_path = os.environ.get('CUDA_PATH') +assert cuda_path is not None +print("CUDA_PATH:", cuda_path) + +# check _wheel.json exists +# note that we don't rely on cupy.__file__ because we cannot import CuPy on Windows +pypaths = site.getsitepackages() +for cp in pypaths: + config_json = os.path.join(cp, 'cupy/.data/_wheel.json') + if not os.path.isfile(config_json): + continue + else: + print("_wheel.json is found:", config_json) + break +else: + raise RuntimeError('_wheel.json is not found in the package') + +# Now that conda-forge docker images have libcuda.so, so "import cupy" would not fail +# on Linux. However, tests would fail on the Azure CI since there is no GPU. See the +# discussion in https://github.com/conda-forge/cupy-feedstock/pull/59#issuecomment-629584090. +# On Windows, this will fail because Windows has no driver stub. +try: + import cupy +except Exception as e: + if sys.platform.startswith('win32'): + print("No driver available on Windows. Exiting...") + sys.exit(0) + +# Ensure CuPy picks up the correct CUDA_VERSION +from cupy.cuda import driver +ver = driver.get_build_version() +cuda_ver = os.environ.get('cuda_compiler_version').split('.') +cuda_ver = int(cuda_ver[0]) * 1000 + int(cuda_ver[1]) * 10 +if ver != cuda_ver: + raise ValueError('CUDA version {0} != cuda_compiler_version {1}'.format( + ver, cuda_ver)) + +try: + # Print CuPy runtime info + # this line would fail if there is no GPU + cupy.show_config() +except Exception as e: + print('Got an error: \n%s' % str(e)) + print("No GPU available. Exiting without running CuPy's tests.") + sys.exit(0) + +## Run CuPy's test suite +#import py +#py.test.cmdline.main(["tests/cupy_tests"]) +#py.test.cmdline.main(["tests/cupyx_tests"]) diff --git a/recipe/test_openmm.sh b/recipe/test_openmm.sh deleted file mode 100644 index 10bfb773..00000000 --- a/recipe/test_openmm.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash -set -ex -with_cuda="no" - -# Existence tests -test -f $PREFIX/lib/libOpenMM$SHLIB_EXT -test -f $PREFIX/lib/plugins/libOpenMMCPU$SHLIB_EXT -test -f $PREFIX/lib/plugins/libOpenMMPME$SHLIB_EXT -test -f $PREFIX/lib/plugins/libOpenMMOpenCL$SHLIB_EXT -if [[ "$target_platform" == linux-64 || "$target_platform" == linux-ppc64le ]]; then - with_cuda="yes" - test -f $PREFIX/lib/plugins/libOpenMMCUDA$SHLIB_EXT - test -f $PREFIX/lib/plugins/libOpenMMCudaCompiler$SHLIB_EXT -fi - -## Do they work properly? -# Debug silent errors in plugin loading -python -c "import openmm as mm; print('---Loaded---', *mm.pluginLoadedLibNames, '---Failed---', *mm.Platform.getPluginLoadFailures(), sep='\n')" -# Check that hardcoded library path was correctly replaced by conda-build -python -c "import os, openmm.version as v; print(v.openmm_library_path); assert os.path.isdir(v.openmm_library_path), 'Directory does not exist'" - -# Check all platforms -if [[ "$target_platform" == linux-ppc64le || "$target_platform" == linux-aarch64 ]]; then - python -m openmm.testInstallation || true # OpenCL will fail but that's ok -else - python -m openmm.testInstallation -fi -if [[ $with_cuda == yes ]]; then - # Linux64 / PPC see all 4 platforms, but CUDA is not usable because there's no GPU there - n_platforms=4 -else - # MacOS / ARM only see 3 because CUDA is not available there - n_platforms=3 -fi -python -c "from openmm import Platform as P; n = P.getNumPlatforms(); assert n == $n_platforms, f'n_platforms ({n}) != $n_platforms'" - -# Run a small MD -cd ${PREFIX}/share/openmm/examples -python benchmark.py --test=rf --seconds=10 --platform=Reference -python benchmark.py --test=rf --seconds=10 --platform=CPU -if [[ -z ${CI-} ]]; then # Run only outside CI, assuming there will be a GPU there - python benchmark.py --test=rf --seconds=10 --platform=OpenCL - if [[ $with_cuda == yes ]]; then - python benchmark.py --test=rf --seconds=10 --platform=CUDA - fi -fi - -# Check version metadata looks ok, only for final releases, RCs are not checked! -if [[ ${PKG_VERSION} != *"rc"* && ${PKG_VERSION} != *"beta"* ]]; then - python -c "from openmm import Platform; v = Platform.getOpenMMVersion(); assert \"$PKG_VERSION\" in (v, v+'.0'), v + \"!=$PKG_VERSION\"" - git_revision=$(git ls-remote https://github.com/openmm/openmm.git $PKG_VERSION | awk '{ print $1}') - python -c "from openmm.version import git_revision; r = git_revision; assert r == \"$git_revision\", r + \"!=$git_revision\"" -else - echo "!!! WARNING !!!" - echo "This is a release candidate build ($PKG_VERSION). Please check versions and git hashes manually!" -fi - -if [[ $with_test_suite == "true" ]]; then - cd $PREFIX/share/openmm/tests - set +ex - - # C++ tests - summary=""; exitcode=0; count=0; - for f in Test*; do - if [[ -n ${CI-} && ( $f == *Cuda* || $f == *OpenCL* ) ]]; then continue; fi - ((count+=1)) - echo -e "\n#$count: $f" - # Retry three times so stochastic tests have a chance - attempts=0 - while true; do - ./${f} - thisexitcode=$? - ((attempts+=1)) - if [[ $thisexitcode == 0 || $attempts == 3 ]]; then break; fi - done - if [[ $thisexitcode != 0 ]]; then summary+="\n#$count ${f}"; fi - ((exitcode+=$thisexitcode)) - done - if [[ $exitcode != 0 ]]; then - echo "------------" - echo "Failed tests" - echo "------------" - echo -e "${summary}" - exit $exitcode - fi - - # Python tests - set -ex - cd python - python -m pytest -v -n $CPU_COUNT -fi From 3e502fe0f7ff0d8c4ecfb2132e41d88e02826017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 23 Jan 2023 16:46:02 +0100 Subject: [PATCH 17/71] only PRs --- .github/workflows/conda-build.yml | 2 +- conda-forge.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index 08c254a4..6f0aab22 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -3,7 +3,7 @@ # -*- mode: yaml -*- name: Build conda package -on: ['push', 'pull_request'] +on: ['pull_request'] jobs: build: diff --git a/conda-forge.yml b/conda-forge.yml index 6e348b43..4774e398 100644 --- a/conda-forge.yml +++ b/conda-forge.yml @@ -12,6 +12,6 @@ conda_build: upload_on_branch: main github_actions: self_hosted: true - self_hosted_triggers: ["push", "pull_request"] + self_hosted_triggers: ["pull_request"] self_hosted_labels: - cirun-openstack-gpu From 6cd3a9528d28a79f3eca62a0ad1f155f71a4d3c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 23 Jan 2023 17:17:12 +0100 Subject: [PATCH 18/71] use gpu_large --- .cirun.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirun.yml b/.cirun.yml index bbf0ff7f..d4d9e851 100755 --- a/.cirun.yml +++ b/.cirun.yml @@ -3,7 +3,7 @@ runners: name: cirun-openstack-gpu labels: ['linux', 'x64', 'self-hosted', 'cirun-openstack-gpu'] cloud: openstack - instance_type: gpu_medium + instance_type: gpu_large machine_image: ubuntu-2204-nvidia-container-runtime-20230107 region: RegionOne - # Custom runner 2 From 0d998722fb06bc9f651216788789df2939d61807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 23 Jan 2023 18:05:56 +0100 Subject: [PATCH 19/71] concurrency --- ...dnn8cxx_compiler_version10python3.9.____cpython.yaml} | 4 ++-- .github/workflows/conda-build.yml | 9 ++++++--- recipe/meta.yaml | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) rename .ci_support/{linux_64_c_compiler_version10cuda_compiler_version11.2cudnn8cxx_compiler_version10python3.9.____cpython.yaml => linux_64_c_compiler_version10cuda_compiler_version11.1cudnn8cxx_compiler_version10python3.9.____cpython.yaml} (90%) diff --git a/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cudnn8cxx_compiler_version10python3.9.____cpython.yaml b/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.1cudnn8cxx_compiler_version10python3.9.____cpython.yaml similarity index 90% rename from .ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cudnn8cxx_compiler_version10python3.9.____cpython.yaml rename to .ci_support/linux_64_c_compiler_version10cuda_compiler_version11.1cudnn8cxx_compiler_version10python3.9.____cpython.yaml index 6b73dc59..2cd8bd3e 100644 --- a/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cudnn8cxx_compiler_version10python3.9.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.1cudnn8cxx_compiler_version10python3.9.____cpython.yaml @@ -11,7 +11,7 @@ channel_targets: cuda_compiler: - nvcc cuda_compiler_version: -- '11.2' +- '11.1' cudnn: - '8' cutensor: @@ -21,7 +21,7 @@ cxx_compiler: cxx_compiler_version: - '10' docker_image: -- quay.io/condaforge/linux-anvil-cuda:11.2 +- quay.io/condaforge/linux-anvil-cuda:11.1 nccl: - '2' pin_run_as_build: diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index 6f0aab22..4801e033 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -4,6 +4,9 @@ name: Build conda package on: ['pull_request'] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true jobs: build: @@ -13,15 +16,15 @@ jobs: fail-fast: false matrix: include: - - CONFIG: linux_64_c_compiler_version10cuda_compiler_version11.2cudnn8cxx_compiler_version10python3.9.____cpython + - CONFIG: linux_64_c_compiler_version10cuda_compiler_version11.1cudnn8cxx_compiler_version10python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu labels: - self-hosted - linux - x64 - - cirun-openstack-gpu--${{ github.run_id }}-linux_64_c_compiler_version10cuda_c_h4b2e19d586 - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.2 + - cirun-openstack-gpu--${{ github.run_id }}-linux_64_c_compiler_version10cuda_c_h77ddf0f142 + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.1 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" steps: - name: Checkout code diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 83f3da89..39661783 100755 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -22,7 +22,7 @@ source: build: number: 0 - skip: true # [not (linux64 and cuda_compiler_version == '11.2' and py == 39)] + skip: true # [not (linux and cuda_compiler_version == '11.1' and py == 39)] script: # CuPy default detects CUDA from nvcc, but on Conda-Forge's dockers nvcc lives in a different place... # With conda-forge/nvcc-feedstock#58, CUDA_PATH is set correctly From 3f37548d28cfdb43994dba5010aa77abca432469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 23 Jan 2023 18:52:27 +0100 Subject: [PATCH 20/71] enable aarch64 --- ...iler_version11.1python3.9.____cpython.yaml | 45 +++++++++++++++++++ .github/workflows/conda-build.yml | 9 ++++ conda-forge.yml | 9 +++- 3 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 .ci_support/linux_aarch64_cuda_compiler_version11.1python3.9.____cpython.yaml diff --git a/.ci_support/linux_aarch64_cuda_compiler_version11.1python3.9.____cpython.yaml b/.ci_support/linux_aarch64_cuda_compiler_version11.1python3.9.____cpython.yaml new file mode 100644 index 00000000..9386bec6 --- /dev/null +++ b/.ci_support/linux_aarch64_cuda_compiler_version11.1python3.9.____cpython.yaml @@ -0,0 +1,45 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +c_compiler: +- gcc +c_compiler_version: +- '9' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '11.1' +cudnn: +- '8' +cutensor: +- '1' +cxx_compiler: +- gxx +cxx_compiler_version: +- '9' +docker_image: +- quay.io/condaforge/linux-anvil-aarch64-cuda:11.1 +nccl: +- '2' +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.9.* *_cpython +target_platform: +- linux-aarch64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version + - cudnn + - cuda_compiler_version + - cdt_name + - docker_image diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index 4801e033..9e3c9324 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -26,6 +26,15 @@ jobs: - cirun-openstack-gpu--${{ github.run_id }}-linux_64_c_compiler_version10cuda_c_h77ddf0f142 DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.1 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" + - CONFIG: linux_aarch64_cuda_compiler_version11.1python3.9.____cpython + UPLOAD_PACKAGES: True + os: ubuntu + labels: + - self-hosted + - linux + - cirun-openstack-gpu--${{ github.run_id }}-linux_aarch64_cuda_compiler_version_h95ed19b582 + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-aarch64-cuda:11.1 + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/conda-forge.yml b/conda-forge.yml index 4774e398..ae8ae754 100644 --- a/conda-forge.yml +++ b/conda-forge.yml @@ -4,9 +4,14 @@ conda_forge_output_validation: true github: branch_name: main tooling_branch_name: main +os_version: + linux_64: cos7 + linux_aarch64: cos7 + linux_ppc64le: cos7 provider: - linux: - - github_actions + linux: github_actions + linux_aarch64: github_actions + linux_ppc64le: github_actions conda_build: pkg_format: 2 upload_on_branch: main From c6f9608da97171df7603153f000b157f43645661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Tue, 24 Jan 2023 00:04:32 +0100 Subject: [PATCH 21/71] retrigger From b74076f911a55622afc08a448c0ff6f9c72d2720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Tue, 24 Jan 2023 17:03:34 +0100 Subject: [PATCH 22/71] retrigger From 39fb572d0693cf3c9c3f0483fdacb405788b0f7a Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 31 Jan 2023 10:10:19 +0100 Subject: [PATCH 23/71] Update recipe/run_test.py --- recipe/run_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipe/run_test.py b/recipe/run_test.py index a104420e..96814784 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -52,6 +52,6 @@ sys.exit(0) ## Run CuPy's test suite -#import py -#py.test.cmdline.main(["tests/cupy_tests"]) +import py +py.test.cmdline.main(["tests/cupy_tests"]) #py.test.cmdline.main(["tests/cupyx_tests"]) From 3a3c5eba53809bcced9899ed2fbbd22bd4494983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Tue, 31 Jan 2023 20:03:22 +0100 Subject: [PATCH 24/71] retrigger From a3be88d191100fb8794ce3269c1c70b2720a3bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Tue, 31 Jan 2023 20:04:45 +0100 Subject: [PATCH 25/71] more tests --- recipe/run_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/run_test.py b/recipe/run_test.py index 96814784..15be9e9f 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -54,4 +54,4 @@ ## Run CuPy's test suite import py py.test.cmdline.main(["tests/cupy_tests"]) -#py.test.cmdline.main(["tests/cupyx_tests"]) +py.test.cmdline.main(["tests/cupyx_tests"]) From a349984b44addcc41552f93faebdfadadb1695bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Tue, 31 Jan 2023 20:45:46 +0100 Subject: [PATCH 26/71] import pytest, not py.test --- recipe/run_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/recipe/run_test.py b/recipe/run_test.py index 15be9e9f..4a0d4894 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -52,6 +52,6 @@ sys.exit(0) ## Run CuPy's test suite -import py -py.test.cmdline.main(["tests/cupy_tests"]) -py.test.cmdline.main(["tests/cupyx_tests"]) +import pytest +pytest.cmdline.main(["tests/cupy_tests"]) +pytest.cmdline.main(["tests/cupyx_tests"]) From afad09c68169b4dbb720a61037512717b4e2effc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Wed, 1 Feb 2023 12:11:59 +0100 Subject: [PATCH 27/71] adjust test reqs --- recipe/meta.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 39661783..9c3644ed 100755 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -111,11 +111,14 @@ requirements: test: requires: - - pytest + - pytest >=7.2 - {{ compiler("c") }} - {{ compiler("cxx") }} - {{ compiler("cuda") }} # tests need nvcc - + - cupy + - cudnn + - setuptools <66.0a0 + - hypothesis >=6.37.2,<6.55.0 source_files: - tests From 548f8712256e29517bcdf6e7e6e886014875ede2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Wed, 1 Feb 2023 12:17:52 +0100 Subject: [PATCH 28/71] propagate exit errors --- recipe/run_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/recipe/run_test.py b/recipe/run_test.py index 4a0d4894..6704a181 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -53,5 +53,6 @@ ## Run CuPy's test suite import pytest -pytest.cmdline.main(["tests/cupy_tests"]) -pytest.cmdline.main(["tests/cupyx_tests"]) +cupy_exit_code = pytest.cmdline.main(["tests/cupy_tests"]) +cupyx_exit_code = pytest.cmdline.main(["tests/cupyx_tests"]) +sys.exit(cupy_exit_code + cupyx_exit_code) From 1a4b837022995355bc05f15b59896c62d1392574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Fri, 3 Feb 2023 12:49:06 +0100 Subject: [PATCH 29/71] xlarge? --- .cirun.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirun.yml b/.cirun.yml index d4d9e851..b5547fd7 100755 --- a/.cirun.yml +++ b/.cirun.yml @@ -3,7 +3,7 @@ runners: name: cirun-openstack-gpu labels: ['linux', 'x64', 'self-hosted', 'cirun-openstack-gpu'] cloud: openstack - instance_type: gpu_large + instance_type: gpu_xlarge machine_image: ubuntu-2204-nvidia-container-runtime-20230107 region: RegionOne - # Custom runner 2 From df811bec059765af7816055425adee7252fe11cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Sun, 5 Feb 2023 13:03:59 +0100 Subject: [PATCH 30/71] moar verbosity --- recipe/run_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipe/run_test.py b/recipe/run_test.py index 6704a181..bd6916f8 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -53,6 +53,6 @@ ## Run CuPy's test suite import pytest -cupy_exit_code = pytest.cmdline.main(["tests/cupy_tests"]) -cupyx_exit_code = pytest.cmdline.main(["tests/cupyx_tests"]) +cupy_exit_code = pytest.cmdline.main(["tests/cupy_tests", "-vvvs"]) +cupyx_exit_code = pytest.cmdline.main(["tests/cupyx_tests", "-vvvs"]) sys.exit(cupy_exit_code + cupyx_exit_code) From 2a4e11c8e912964ba292bf1b1f4be8f1f1940966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Thu, 9 Feb 2023 11:34:26 +0100 Subject: [PATCH 31/71] not slow --- recipe/run_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipe/run_test.py b/recipe/run_test.py index bd6916f8..f0b8257d 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -53,6 +53,6 @@ ## Run CuPy's test suite import pytest -cupy_exit_code = pytest.cmdline.main(["tests/cupy_tests", "-vvvs"]) -cupyx_exit_code = pytest.cmdline.main(["tests/cupyx_tests", "-vvvs"]) +cupy_exit_code = pytest.cmdline.main(["tests/cupy_tests", "-vvv", "-m", "not slow"]) +cupyx_exit_code = pytest.cmdline.main(["tests/cupyx_tests", "-vvv", "-m", "not slow"]) sys.exit(cupy_exit_code + cupyx_exit_code) From 99931288d7628e4c2003f67e7113fd4ded07613c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Thu, 9 Feb 2023 11:35:23 +0100 Subject: [PATCH 32/71] skip aarch --- recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 9c3644ed..339f6e36 100755 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -22,7 +22,7 @@ source: build: number: 0 - skip: true # [not (linux and cuda_compiler_version == '11.1' and py == 39)] + skip: true # [not (linux64 and cuda_compiler_version == '11.1' and py == 39)] script: # CuPy default detects CUDA from nvcc, but on Conda-Forge's dockers nvcc lives in a different place... # With conda-forge/nvcc-feedstock#58, CUDA_PATH is set correctly From ca54817383acddfb32b158dccd848233c4bdebad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Thu, 9 Feb 2023 11:37:56 +0100 Subject: [PATCH 33/71] MNT: Re-rendered with conda-build 3.21.7+168.ge0884b62, conda-smithy 0.0.0, and conda-forge-pinning 2023.02.08.20.12.13 --- ...iler_version11.1python3.9.____cpython.yaml | 45 ------------------- .cirun.yml | 2 +- .github/workflows/conda-build.yml | 9 ---- README.md | 8 ++-- 4 files changed, 5 insertions(+), 59 deletions(-) delete mode 100644 .ci_support/linux_aarch64_cuda_compiler_version11.1python3.9.____cpython.yaml diff --git a/.ci_support/linux_aarch64_cuda_compiler_version11.1python3.9.____cpython.yaml b/.ci_support/linux_aarch64_cuda_compiler_version11.1python3.9.____cpython.yaml deleted file mode 100644 index 9386bec6..00000000 --- a/.ci_support/linux_aarch64_cuda_compiler_version11.1python3.9.____cpython.yaml +++ /dev/null @@ -1,45 +0,0 @@ -BUILD: -- aarch64-conda_cos7-linux-gnu -c_compiler: -- gcc -c_compiler_version: -- '9' -cdt_arch: -- aarch64 -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '11.1' -cudnn: -- '8' -cutensor: -- '1' -cxx_compiler: -- gxx -cxx_compiler_version: -- '9' -docker_image: -- quay.io/condaforge/linux-anvil-aarch64-cuda:11.1 -nccl: -- '2' -pin_run_as_build: - python: - min_pin: x.x - max_pin: x.x -python: -- 3.9.* *_cpython -target_platform: -- linux-aarch64 -zip_keys: -- - c_compiler_version - - cxx_compiler_version - - cudnn - - cuda_compiler_version - - cdt_name - - docker_image diff --git a/.cirun.yml b/.cirun.yml index b5547fd7..d4d9e851 100755 --- a/.cirun.yml +++ b/.cirun.yml @@ -3,7 +3,7 @@ runners: name: cirun-openstack-gpu labels: ['linux', 'x64', 'self-hosted', 'cirun-openstack-gpu'] cloud: openstack - instance_type: gpu_xlarge + instance_type: gpu_large machine_image: ubuntu-2204-nvidia-container-runtime-20230107 region: RegionOne - # Custom runner 2 diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index 9e3c9324..4801e033 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -26,15 +26,6 @@ jobs: - cirun-openstack-gpu--${{ github.run_id }}-linux_64_c_compiler_version10cuda_c_h77ddf0f142 DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.1 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - - CONFIG: linux_aarch64_cuda_compiler_version11.1python3.9.____cpython - UPLOAD_PACKAGES: True - os: ubuntu - labels: - - self-hosted - - linux - - cirun-openstack-gpu--${{ github.run_id }}-linux_aarch64_cuda_compiler_version_h95ed19b582 - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-aarch64-cuda:11.1 - CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/README.md b/README.md index 225741a3..56d55138 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ -About cupy -========== +About cupy-feedstock +==================== + +Feedstock license: [BSD-3-Clause](https://github.com/conda-forge/cf-autotick-bot-test-package-feedstock/blob/main/LICENSE.txt) Home: https://cupy.dev/ Package license: MIT -Feedstock license: [BSD-3-Clause](https://github.com/conda-forge/cf-autotick-bot-test-package-feedstock/blob/main/LICENSE.txt) - Summary: CuPy: NumPy & SciPy for GPU From 46c5a742031ae04edbb3135b3ce5e8800f1070cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 13 Feb 2023 14:28:41 +0100 Subject: [PATCH 34/71] symlink gcc? --- recipe/run_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/recipe/run_test.py b/recipe/run_test.py index f0b8257d..19fba8e0 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -53,6 +53,12 @@ ## Run CuPy's test suite import pytest + +if sys.platform.startswith("linux"): + gcc = os.path.join(os.environ["PREFIX"], "bin", "gcc") + if "CC" in os.environ and not os.path.exists(gcc): + os.symlink(os.environ["CC"], gcc) + cupy_exit_code = pytest.cmdline.main(["tests/cupy_tests", "-vvv", "-m", "not slow"]) cupyx_exit_code = pytest.cmdline.main(["tests/cupyx_tests", "-vvv", "-m", "not slow"]) sys.exit(cupy_exit_code + cupyx_exit_code) From 279d10666d9726abe4c4cb7c55baa22c04517cfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 13 Feb 2023 18:14:25 +0100 Subject: [PATCH 35/71] add some test reqs --- recipe/meta.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 339f6e36..fc6f1a67 100755 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -119,6 +119,10 @@ test: - cudnn - setuptools <66.0a0 - hypothesis >=6.37.2,<6.55.0 + - scipy + - optuna + - cython + - numpy source_files: - tests From b2f7ed6e4784897fef322e019dfc8f14cc3c8a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 13 Feb 2023 18:14:46 +0100 Subject: [PATCH 36/71] less verbose --- recipe/run_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipe/run_test.py b/recipe/run_test.py index 19fba8e0..002ee4c2 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -59,6 +59,6 @@ if "CC" in os.environ and not os.path.exists(gcc): os.symlink(os.environ["CC"], gcc) -cupy_exit_code = pytest.cmdline.main(["tests/cupy_tests", "-vvv", "-m", "not slow"]) -cupyx_exit_code = pytest.cmdline.main(["tests/cupyx_tests", "-vvv", "-m", "not slow"]) +cupy_exit_code = pytest.cmdline.main(["tests/cupy_tests", "-vv", "-m", "not slow"]) +cupyx_exit_code = pytest.cmdline.main(["tests/cupyx_tests", "-vv", "-m", "not slow"]) sys.exit(cupy_exit_code + cupyx_exit_code) From 9b95680e9f186a4328727afa647f1bc9cd350497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Tue, 14 Feb 2023 13:38:06 +0100 Subject: [PATCH 37/71] symlink g++ too --- recipe/run_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/recipe/run_test.py b/recipe/run_test.py index 002ee4c2..5f596312 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -58,6 +58,9 @@ gcc = os.path.join(os.environ["PREFIX"], "bin", "gcc") if "CC" in os.environ and not os.path.exists(gcc): os.symlink(os.environ["CC"], gcc) + gxx = os.path.join(os.environ["PREFIX"], "bin", "g++") + if "CXX" in os.environ and not os.path.exists(gxx): + os.symlink(os.environ["CXX"], gxx) cupy_exit_code = pytest.cmdline.main(["tests/cupy_tests", "-vv", "-m", "not slow"]) cupyx_exit_code = pytest.cmdline.main(["tests/cupyx_tests", "-vv", "-m", "not slow"]) From 1f5d6ae85d9c6c3f3e0c0ed4b8a984a5968825fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Wed, 15 Feb 2023 12:13:46 +0100 Subject: [PATCH 38/71] pin numpy / scipy --- recipe/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index fc6f1a67..75f5034f 100755 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -119,10 +119,10 @@ test: - cudnn - setuptools <66.0a0 - hypothesis >=6.37.2,<6.55.0 - - scipy + - scipy <1.10.0a0 # https://github.com/conda-forge/cf-autotick-bot-test-package-feedstock/pull/466#issuecomment-1431145378 + - numpy <1.24.0a0 - optuna - cython - - numpy source_files: - tests From abe1718f9776ac90706f6783abd591040c9e5419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Wed, 15 Feb 2023 17:00:26 +0100 Subject: [PATCH 39/71] retrigger From d6cca0a5159e96f968b78d641ea300ef234c8cd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Thu, 16 Feb 2023 10:41:24 +0100 Subject: [PATCH 40/71] retrigger From 17ea28dee6333d3094f90fa5494ade15d0dcdc7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Thu, 16 Feb 2023 10:44:56 +0100 Subject: [PATCH 41/71] explicit timeout so default 360 is not used --- .github/workflows/conda-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index 4801e033..ca3cf7b6 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -12,6 +12,7 @@ jobs: build: name: ${{ matrix.CONFIG }} runs-on: ${{ matrix.labels }} + timeout-minutes: 720 # 12h strategy: fail-fast: false matrix: From 8b1448e53404d8e7563e8b617ab81b559f1694f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 20 Feb 2023 16:20:55 +0100 Subject: [PATCH 42/71] include setup.cfg in tests too --- recipe/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 75f5034f..54736af7 100755 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -125,6 +125,8 @@ test: - cython source_files: - tests + - setup.cfg + about: home: https://cupy.dev/ From 965737bfd1d8d16e0d8df4435bedf1c3835a9ab3 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 4 Oct 2023 15:53:47 +0200 Subject: [PATCH 43/71] update recipe --- .ci_support/migrations/cuda118.yaml | 99 ++++++++++++ .ci_support/migrations/cuda120.yaml | 90 +++++++++++ .ci_support/migrations/python311.yaml | 37 ----- recipe/conda_build_config.yaml | 36 ----- recipe/meta.yaml | 151 ++++++++++------- .../preload_config/linux64_cuda12_wheel.json | 7 + recipe/preload_config/win64_cuda12_wheel.json | 6 + recipe/run_test.py | 44 +++-- recipe/support_new_cuda_layout.diff | 152 ++++++++++++++++++ recipe/support_nvtx3_win.diff | 12 ++ 10 files changed, 487 insertions(+), 147 deletions(-) create mode 100644 .ci_support/migrations/cuda118.yaml create mode 100644 .ci_support/migrations/cuda120.yaml delete mode 100644 .ci_support/migrations/python311.yaml create mode 100644 recipe/preload_config/linux64_cuda12_wheel.json create mode 100644 recipe/preload_config/win64_cuda12_wheel.json create mode 100644 recipe/support_new_cuda_layout.diff create mode 100644 recipe/support_nvtx3_win.diff diff --git a/.ci_support/migrations/cuda118.yaml b/.ci_support/migrations/cuda118.yaml new file mode 100644 index 00000000..fdd298b9 --- /dev/null +++ b/.ci_support/migrations/cuda118.yaml @@ -0,0 +1,99 @@ +migrator_ts: 1692828152 +__migrator: + kind: + version + migration_number: + 1 + build_number: + 1 + paused: false + override_cbc_keys: + - cuda_compiler_stub + operation: key_add + check_solvable: false + primary_key: cuda_compiler_version + ordering: + cxx_compiler_version: + - 9 + - 8 + - 7 + c_compiler_version: + - 9 + - 8 + - 7 + fortran_compiler_version: + - 9 + - 8 + - 7 + docker_image: + # Native builds + - quay.io/condaforge/linux-anvil-comp7 # [os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-aarch64 # [os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] + - quay.io/condaforge/linux-anvil-ppc64le # [os.environ.get("BUILD_PLATFORM") == "linux-ppc64le"] + - quay.io/condaforge/linux-anvil-armv7l # [os.environ.get("BUILD_PLATFORM") == "linux-armv7l"] + + # Legacy CUDAs + - quay.io/condaforge/linux-anvil-cuda:9.2 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:10.0 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:10.1 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:10.2 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:11.0 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:11.1 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + + # CUDA 11.2 + - quay.io/condaforge/linux-anvil-cuda:11.2 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + # CUDA 11.2 arch: native compilation (build == target) + - quay.io/condaforge/linux-anvil-ppc64le-cuda:11.2 # [ppc64le and os.environ.get("BUILD_PLATFORM") == "linux-ppc64le"] + - quay.io/condaforge/linux-anvil-aarch64-cuda:11.2 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] + # CUDA 11.2 arch: cross-compilation (build != target) + - quay.io/condaforge/linux-anvil-cuda:11.2 # [ppc64le and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:11.2 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + + # CUDA 11.8 + - quay.io/condaforge/linux-anvil-cuda:11.8 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + # CUDA 11.8 arch: native compilation (build == target) + - quay.io/condaforge/linux-anvil-ppc64le-cuda:11.8 # [ppc64le and os.environ.get("BUILD_PLATFORM") == "linux-ppc64le"] + - quay.io/condaforge/linux-anvil-aarch64-cuda:11.8 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] + # CUDA 11.8 arch: cross-compilation (build != target) + - quay.io/condaforge/linux-anvil-cuda:11.8 # [ppc64le and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:11.8 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + + # Native CentOS 7 image + - quay.io/condaforge/linux-anvil-cos7-x86_64 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + cuda_compiler_version: + - None + - 10.2 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11.0 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11.1 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11.2 # [(linux or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11.8 # [(linux or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 12.0 # [(linux or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + commit_message: | + Rebuild for CUDA 11.8 w/arch support + +cuda_compiler: # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - nvcc # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +cuda_compiler_version: # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11.8 # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +c_compiler_version: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +cxx_compiler_version: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +fortran_compiler_version: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +cdt_name: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cos7 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +docker_image: # [os.environ.get("BUILD_PLATFORM", "").startswith("linux-") and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - quay.io/condaforge/linux-anvil-cuda:11.8 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + # case: native compilation (build == target) + - quay.io/condaforge/linux-anvil-ppc64le-cuda:11.8 # [ppc64le and os.environ.get("BUILD_PLATFORM") == "linux-ppc64le"] + - quay.io/condaforge/linux-anvil-aarch64-cuda:11.8 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] + # case: cross-compilation (build != target) + - quay.io/condaforge/linux-anvil-cuda:11.8 # [ppc64le and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:11.8 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] diff --git a/.ci_support/migrations/cuda120.yaml b/.ci_support/migrations/cuda120.yaml new file mode 100644 index 00000000..49da33d7 --- /dev/null +++ b/.ci_support/migrations/cuda120.yaml @@ -0,0 +1,90 @@ +migrator_ts: 1682985063 +__migrator: + kind: + version + migration_number: + 2 + build_number: + 1 + paused: false + override_cbc_keys: + - cuda_compiler_stub + operation: key_add + check_solvable: false + primary_key: cuda_compiler_version + ordering: + cxx_compiler_version: + - 9 + - 8 + - 7 + c_compiler_version: + - 9 + - 8 + - 7 + fortran_compiler_version: + - 9 + - 8 + - 7 + docker_image: + - quay.io/condaforge/linux-anvil-comp7 # [os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-aarch64 # [os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] + - quay.io/condaforge/linux-anvil-ppc64le # [os.environ.get("BUILD_PLATFORM") == "linux-ppc64le"] + - quay.io/condaforge/linux-anvil-armv7l # [os.environ.get("BUILD_PLATFORM") == "linux-armv7l"] + - quay.io/condaforge/linux-anvil-cuda:9.2 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:10.0 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:10.1 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:10.2 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:11.0 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:11.1 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:11.2 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + # case: native compilation (build == target) + - quay.io/condaforge/linux-anvil-ppc64le-cuda:11.2 # [ppc64le and os.environ.get("BUILD_PLATFORM") == "linux-ppc64le"] + - quay.io/condaforge/linux-anvil-aarch64-cuda:11.2 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] + # case: cross-compilation (build != target) + - quay.io/condaforge/linux-anvil-cuda:11.2 # [ppc64le and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:11.2 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + # case: non-CUDA builds + - quay.io/condaforge/linux-anvil-cos7-x86_64 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] + cuda_compiler_version: + - None + - 10.2 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11.0 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11.1 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11.2 # [(linux or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 12.0 # [(linux or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + commit_message: | + Rebuild for CUDA 12 w/arch support + + The transition to CUDA 12 SDK includes new packages for all CUDA libraries and + build tools. Notably, the cudatoolkit package no longer exists, and packages + should depend directly on the specific CUDA libraries (libcublas, libcusolver, + etc) as needed. For an in-depth overview of the changes and to report problems + [see this issue]( https://github.com/conda-forge/conda-forge.github.io/issues/1963 ). + Please feel free to raise any issues encountered there. Thank you! :pray: + +cuda_compiler: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cuda-nvcc # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +cuda_compiler_version: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 12.0 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +c_compiler_version: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 12 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +cxx_compiler_version: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 12 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +fortran_compiler_version: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 12 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +cdt_name: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cos7 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +docker_image: # [os.environ.get("BUILD_PLATFORM", "").startswith("linux-") and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - quay.io/condaforge/linux-anvil-cos7-x86_64 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64" and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + # case: native compilation (build == target) + - quay.io/condaforge/linux-anvil-ppc64le # [ppc64le and os.environ.get("BUILD_PLATFORM") == "linux-ppc64le"] + - quay.io/condaforge/linux-anvil-aarch64 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] + # case: cross-compilation (build != target) + - quay.io/condaforge/linux-anvil-cos7-x86_64 # [ppc64le and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cos7-x86_64 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] diff --git a/.ci_support/migrations/python311.yaml b/.ci_support/migrations/python311.yaml deleted file mode 100644 index 2715ab28..00000000 --- a/.ci_support/migrations/python311.yaml +++ /dev/null @@ -1,37 +0,0 @@ -migrator_ts: 1666686085 -__migrator: - migration_number: 1 - operation: key_add - primary_key: python - ordering: - python: - - 3.6.* *_cpython - - 3.7.* *_cpython - - 3.8.* *_cpython - - 3.9.* *_cpython - - 3.10.* *_cpython - - 3.11.* *_cpython # new entry - - 3.6.* *_73_pypy - - 3.7.* *_73_pypy - - 3.8.* *_73_pypy - - 3.9.* *_73_pypy - paused: false - longterm: True - pr_limit: 30 - max_solver_attempts: 10 # this will make the bot retry "not solvable" stuff 10 times - exclude: - # this shouldn't attempt to modify the python feedstocks - - python - - pypy3.6 - - pypy-meta - - cross-python - - python_abi - exclude_pinned_pkgs: false - -python: - - 3.11.* *_cpython -# additional entries to add for zip_keys -numpy: - - 1.23 -python_impl: - - cpython diff --git a/recipe/conda_build_config.yaml b/recipe/conda_build_config.yaml index b427a0ed..bd6e07bf 100644 --- a/recipe/conda_build_config.yaml +++ b/recipe/conda_build_config.yaml @@ -1,39 +1,3 @@ -arm_variant_type: # [aarch64] - - sbsa # [aarch64] - -# The added keys below are needed to enable ppc64le/aarch64 CUDA builds; the linux64 -# and win64 builds are already handled in the conda-forge-pinning feedstock. Once we -# migrate to enable CUDA builds on all supported platforms, these can be removed too. - -docker_image: # [os.environ.get("BUILD_PLATFORM", "").startswith("linux")] - - quay.io/condaforge/linux-anvil-aarch64-cuda:11.0 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] - - quay.io/condaforge/linux-anvil-aarch64-cuda:11.1 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] - -c_compiler_version: # [linux] - - 9 # [aarch64] - - 9 # [aarch64] -cxx_compiler_version: # [linux] - - 9 # [aarch64] - - 9 # [aarch64] -fortran_compiler_version: # [linux] - - 9 # [aarch64] - - 9 # [aarch64] - -cuda_compiler: - - nvcc - -cuda_compiler_version: - - 11.0 # [aarch64] - - 11.1 # [aarch64] - -cudnn: - - 8 # [aarch64] - - 8 # [aarch64] - -cdt_name: # [linux] - - cos7 # [aarch64] - - cos7 # [aarch64] - c_compiler: # [win] - vs2017 # [win] cxx_compiler: # [win] diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 54736af7..83c7d095 100755 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,15 +1,16 @@ {% set name = "cupy" %} -{% set version = "11.5.0" %} -{% set sha256 = "4bc8565bded22cc89b210fd9fb48a5d5316f30701e12bb23852a60314e1f9f6e" %} +{% set version = "12.2.0" %} +{% set sha256 = "f95ffd0afeacb617b048fe028ede07b97dc9e95aca1610a022b1f3d20a9a027e" %} + +{% set target_name = "x86_64-linux" %} # [linux64] +{% set target_name = "ppc64le-linux" %} # [ppc64le] +{% set target_name = "sbsa-linux" %} # [aarch64] +{% set target_name = "x64" %} # [win] {% if cuda_compiler_version in (None, "None", True, False) %} {% set cuda_major = 0 %} -{% set cuda_minor = 0 %} -{% set cuda_major_minor = (0, 0) %} {% else %} -{% set cuda_major = environ.get("cuda_compiler_version", "11.2").split(".")[0]|int %} -{% set cuda_minor = environ.get("cuda_compiler_version", "11.2").split(".")[1]|int %} -{% set cuda_major_minor = (cuda_major, cuda_minor) %} +{% set cuda_major = environ.get("cuda_compiler_version", "11.8").split(".")[0] | int %} {% endif %} package: @@ -19,22 +20,32 @@ package: source: - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz sha256: {{ sha256 }} + patches: + - support_new_cuda_layout.diff # [(linux and (cuda_compiler_version or "").startswith("12")) or (build_platform != target_platform)] + - support_nvtx3_win.diff # [win64] build: - number: 0 - skip: true # [not (linux64 and cuda_compiler_version == '11.1' and py == 39)] + number: 2 + # TODO: turn on win64 + CUDA 12 once it's ready + skip: true # [cuda_compiler_version not in ("11.8", "12.0") or (cuda_compiler_version == "12.0" and win64)] + skip: true # [win] + skip: true # [aarch64] + skip: true # [ppc64le] + skip: true # [py != 39] script: # CuPy default detects CUDA from nvcc, but on Conda-Forge's dockers nvcc lives in a different place... # With conda-forge/nvcc-feedstock#58, CUDA_PATH is set correctly - - export NVCC=$(which nvcc) # [linux] + - export NVCC="$(which nvcc)" # [linux] + {% if cuda_major >= 12 %} + - export CUPY_NUM_BUILD_JOBS=1 # [(aarch64 or ppc64le) and (build_platform == target_platform)] + - export CUDA_PATH=$PREFIX/targets/{{ target_name }} # [linux] + {% endif %} - echo "nvcc is $NVCC, CUDA path is $CUDA_PATH" # [linux] - for /f "tokens=* usebackq" %%f in (`where nvcc`) do (set "dummy=%%f" && call set "NVCC=%%dummy:\=\\%%") # [win] # we use this WAR until we figure out how to proceed with conda-forge/nvcc-feedstock#88 - set "NVCC=%NVCC% --use-local-env" # [win64] - echo "nvcc is %NVCC%, CUDA path is %CUDA_PATH%" # [win] - {% if cuda_major_minor >= (11, 2) %} - - export CUSPARSELT_PATH=$PREFIX # [linux64 or win] - {% endif %} + - export CUSPARSELT_PATH="${PREFIX}" # [(linux64 or aarch64 or win) and (cuda_compiler_version or "").startswith("11")] # Workaround __ieee128 error; see https://github.com/LLNL/blt/issues/341 - export NVCC="$NVCC -Xcompiler -mno-float128" # [ppc64le] @@ -52,32 +63,52 @@ build: - copy %RECIPE_DIR%\deactivate.bat %PREFIX%\etc\conda\deactivate.d\cupy_deactivate.bat # [win] # enable CuPy's preload mechanism - - mkdir -p "${SP_DIR}/cupy/.data/" # [linux] - - if not exist %SP_DIR%\cupy\.data mkdir %SP_DIR%\cupy\.data # [win] - {% if cuda_major_minor >= (11, 2) %} - - cp ${RECIPE_DIR}/preload_config/linux64_cuda11_wheel.json ${SP_DIR}/cupy/.data/_wheel.json # [linux] - - copy %RECIPE_DIR%\preload_config\win64_cuda11_wheel.json %SP_DIR%\cupy\.data\_wheel.json # [win] - {% else %} - - cp ${RECIPE_DIR}/preload_config/linux64_cuda{{ cuda_compiler_version }}_wheel.json ${SP_DIR}/cupy/.data/_wheel.json # [linux] - - copy %RECIPE_DIR%\preload_config\win64_cuda{{ cuda_compiler_version }}_wheel.json %SP_DIR%\cupy\.data\_wheel.json # [win] - {% endif %} + - mkdir -p "${SP_DIR}/cupy/.data/" # [linux] + - cp ${RECIPE_DIR}/preload_config/linux64_cuda{{ cuda_major }}_wheel.json ${SP_DIR}/cupy/.data/_wheel.json # [linux] + - if not exist %SP_DIR%\cupy\.data mkdir %SP_DIR%\cupy\.data # [win] + - copy %RECIPE_DIR%\preload_config\win64_cuda{{ cuda_major }}_wheel.json %SP_DIR%\cupy\.data\_wheel.json # [win] + + # cupy/cupy#7578 + - if not exist %SP_DIR%\cupy\.data\_depends.json exit 1 # [win] missing_dso_whitelist: - '*/libcuda.*' # [linux] - '*/nvcuda.dll' # [win] ignore_run_exports: - - cudnn - - nccl # [linux] + # optional dependencies + - cudnn # [not ((aarch64 or ppc64le) and (cuda_compiler_version or "").startswith("12"))] + - nccl # [linux] - cutensor - {% if cuda_major_minor >= (11, 2) %} - - cusparselt # [linux64 or win] - {% endif %} + - cusparselt # [(linux64 or aarch64 or win) and (cuda_compiler_version or "").startswith("11")] requirements: build: - {{ compiler("c") }} - {{ compiler("cxx") }} - {{ compiler("cuda") }} - - sysroot_linux-64 2.17 # [linux64] + #- cuda-nvcc + - sysroot_{{ target_platform }} 2.17 # [linux] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + - python # [build_platform != target_platform] + - cython >=0.29.22,<3 # [build_platform != target_platform] + # TODO: clean up + {% if cuda_major >= 12 %} + - cuda-driver-dev # [build_platform != target_platform] + - cuda-cudart-dev # [build_platform != target_platform] + - cuda-nvrtc-dev # [build_platform != target_platform] + - cuda-nvtx-dev # [build_platform != target_platform] + - cuda-profiler-api # [build_platform != target_platform] + - cuda-cccl_{{ build_platform }} # [build_platform != target_platform] + - libcublas-dev # [build_platform != target_platform] + - libcufft-dev # [build_platform != target_platform] + - libcurand-dev # [build_platform != target_platform] + - libcusolver-dev # [build_platform != target_platform] + - libcusparse-dev # [build_platform != target_platform] + {% endif %} + # optional dependencies for CUDA 11.2+ + - cudnn >=8.0.*,<9 # [build_platform != target_platform and (not ((aarch64 or ppc64le) and (cuda_compiler_version or "").startswith("12")))] + - nccl >=2.8,<3 # [build_platform != target_platform] + - cutensor >=1.4,<2 # [build_platform != target_platform] + - cusparselt 0.2.0.* # [(linux64 or aarch64 or win) and (cuda_compiler_version or "").startswith("11")] host: - python @@ -85,48 +116,54 @@ requirements: - setuptools - cython >=0.29.22,<3 - fastrlock >=0.5 - - cudnn - - nccl >=2.8 # [linux] - - cutensor - {% if cuda_major_minor >= (11, 2) %} - - cusparselt 0.2.0.* # [linux64 or win] + - cuda-version {{ cuda_compiler_version }} + - nvtx-c # [win64] + # TODO: clean up + {% if cuda_major >= 12 %} + - cuda-driver-dev + - cuda-cudart-dev + - cuda-nvrtc-dev + - cuda-nvtx-dev + - cuda-profiler-api + - cuda-cccl_{{ target_platform }} + - libcublas-dev + - libcufft-dev + - libcurand-dev + - libcusolver-dev + - libcusparse-dev {% endif %} + # optional dependencies + # TODO: see https://github.com/conda-forge/cudnn-feedstock/issues/58 + - cudnn >=8.0.*,<9 # [not ((aarch64 or ppc64le) and (cuda_compiler_version or "").startswith("12"))] + - nccl >=2.8,<3 # [linux] + - cutensor >=1.4,<2 + - cusparselt 0.2.0.* # [(linux64 or aarch64 or win) and (cuda_compiler_version or "").startswith("11")] run: - python - {{ pin_compatible('fastrlock', max_pin='x.x') }} - - numpy >=1.18 + - numpy >=1.21,<2 + - cuda-version >={{ cuda_major }}.0,<{{ cuda_major+1 }} # [(cuda_compiler_version or "").startswith("12")] + - cuda-version >={{ cuda_major }}.2,<{{ cuda_major+1 }} # [(cuda_compiler_version or "").startswith("11")] run_constrained: # Only GLIBC_2.17 or older symbols present - - __glibc >=2.17 # [linux] - - scipy >=1.4 - - optuna >=2 - - {{ pin_compatible('cudnn') }} - - {{ pin_compatible('nccl') }} # [linux] - - {{ pin_compatible('cutensor', lower_bound='1.3') }} - {% if cuda_major_minor >= (11, 2) %} - - {{ pin_compatible('cusparselt', max_pin='x.x') }} # [linux64 or win] - {% endif %} + - __glibc >=2.17 # [linux] + - scipy >=1.7,<2 + - optuna >=3,<4 + # TODO: Waiting for `aarch64` & `ppc64le` CUDA 12 support + - {{ pin_compatible('cudnn') }} # [not ((aarch64 or ppc64le) and (cuda_compiler_version or "").startswith("12"))] + - {{ pin_compatible('nccl') }} # [linux] + - {{ pin_compatible('cutensor', lower_bound='1.4') }} + - {{ pin_compatible('cusparselt', max_pin='x.x') }} # [(linux64 or aarch64 or win) and (cuda_compiler_version or "").startswith("11")] test: requires: - - pytest >=7.2 - - {{ compiler("c") }} - - {{ compiler("cxx") }} - - {{ compiler("cuda") }} # tests need nvcc - - cupy - - cudnn - - setuptools <66.0a0 - - hypothesis >=6.37.2,<6.55.0 - - scipy <1.10.0a0 # https://github.com/conda-forge/cf-autotick-bot-test-package-feedstock/pull/466#issuecomment-1431145378 - - numpy <1.24.0a0 - - optuna - - cython + # need the libcuda stub for import test + - cuda-driver-dev # [(cuda_compiler_version or "").startswith("12")] + source_files: - tests - - setup.cfg - about: home: https://cupy.dev/ diff --git a/recipe/preload_config/linux64_cuda12_wheel.json b/recipe/preload_config/linux64_cuda12_wheel.json new file mode 100644 index 00000000..07c010aa --- /dev/null +++ b/recipe/preload_config/linux64_cuda12_wheel.json @@ -0,0 +1,7 @@ +{ + "packaging": "conda", + "cuda": "12.x", + "cudnn": {}, + "nccl": {}, + "cutensor": {} +} diff --git a/recipe/preload_config/win64_cuda12_wheel.json b/recipe/preload_config/win64_cuda12_wheel.json new file mode 100644 index 00000000..d17337d7 --- /dev/null +++ b/recipe/preload_config/win64_cuda12_wheel.json @@ -0,0 +1,6 @@ +{ + "packaging": "conda", + "cuda": "12.x", + "cudnn": {}, + "cutensor": {} +} diff --git a/recipe/run_test.py b/recipe/run_test.py index 5f596312..b723210c 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -1,9 +1,19 @@ # Configure CuPy to use 1 GPU for testing +import ctypes import os +import platform import site import sys os.environ["CUPY_TEST_GPU_LIMIT"] = "1" +def get_target_name(): + plat = platform.processor() + if plat == "aarch64": + out = "sbsa-linux" + else: + out = f"{plat}-linux" + return out + # Check CUDA_PATH is set cuda_path = os.environ.get('CUDA_PATH') assert cuda_path is not None @@ -22,16 +32,26 @@ else: raise RuntimeError('_wheel.json is not found in the package') -# Now that conda-forge docker images have libcuda.so, so "import cupy" would not fail -# on Linux. However, tests would fail on the Azure CI since there is no GPU. See the -# discussion in https://github.com/conda-forge/cupy-feedstock/pull/59#issuecomment-629584090. -# On Windows, this will fail because Windows has no driver stub. +# dlopen the driver stub so that CuPy can be imported later +# Note: This would make the actual tests fail on GPU CI! +if sys.platform.startswith('linux') and '12' in os.environ.get('CONDA_OVERRIDE_CUDA', ''): + try: + stub = f"{os.environ['PREFIX']}/targets/{get_target_name()}/lib/stubs/libcuda.so" + stub = ctypes.CDLL(stub) + except Exception as e: + print(f"{stub=} was not loaded:") + raise + +# TODO: do we not ship a stub on Windows? try: import cupy except Exception as e: if sys.platform.startswith('win32'): print("No driver available on Windows. Exiting...") sys.exit(0) +else: + print("import ok") + sys.exit(0) # Ensure CuPy picks up the correct CUDA_VERSION from cupy.cuda import driver @@ -52,16 +72,6 @@ sys.exit(0) ## Run CuPy's test suite -import pytest - -if sys.platform.startswith("linux"): - gcc = os.path.join(os.environ["PREFIX"], "bin", "gcc") - if "CC" in os.environ and not os.path.exists(gcc): - os.symlink(os.environ["CC"], gcc) - gxx = os.path.join(os.environ["PREFIX"], "bin", "g++") - if "CXX" in os.environ and not os.path.exists(gxx): - os.symlink(os.environ["CXX"], gxx) - -cupy_exit_code = pytest.cmdline.main(["tests/cupy_tests", "-vv", "-m", "not slow"]) -cupyx_exit_code = pytest.cmdline.main(["tests/cupyx_tests", "-vv", "-m", "not slow"]) -sys.exit(cupy_exit_code + cupyx_exit_code) +#import py +#py.test.cmdline.main(["tests/cupy_tests"]) +#py.test.cmdline.main(["tests/cupyx_tests"]) diff --git a/recipe/support_new_cuda_layout.diff b/recipe/support_new_cuda_layout.diff new file mode 100644 index 00000000..4179d91b --- /dev/null +++ b/recipe/support_new_cuda_layout.diff @@ -0,0 +1,152 @@ +diff --git a/install/cupy_builder/cupy_setup_build.py b/install/cupy_builder/cupy_setup_build.py +index 8023e46ad..4349ebe05 100644 +--- a/install/cupy_builder/cupy_setup_build.py ++++ b/install/cupy_builder/cupy_setup_build.py +@@ -5,6 +5,7 @@ from distutils import ccompiler + from distutils import sysconfig + import os + import shutil ++import subprocess + import sys + + import setuptools +@@ -294,16 +295,36 @@ def make_extensions(ctx: Context, compiler, use_cython): + # deprecated since ROCm 4.2.0 + settings['define_macros'].append(('__HIP_PLATFORM_HCC__', '1')) + +- available_modules = [] +- if no_cuda: +- available_modules = [m['name'] for m in MODULES] +- else: +- available_modules, settings = preconfigure_modules( +- ctx, MODULES, compiler, settings) +- required_modules = get_required_modules(MODULES) +- if not (set(required_modules) <= set(available_modules)): +- raise Exception('Your CUDA environment is invalid. ' +- 'Please check above error log.') ++ try: ++ host_compiler = compiler ++ if os.environ.get('CONDA_BUILD_CROSS_COMPILATION'): ++ os.symlink(f'{os.environ["BUILD_PREFIX"]}/x86_64-conda-linux-gnu/bin/x86_64-conda-linux-gnu-ld', ++ f'{os.environ["BUILD_PREFIX"]}/bin/ld') ++ if os.environ.get('CONDA_BUILD_CROSS_COMPILATION') or os.environ.get('CONDA_OVERRIDE_CUDA', '0').startswith('12'): ++ # If cross-compiling, we need build_and_run() & build_shlib() to use the compiler ++ # on the build platform to generate stub files that are executable in the build ++ # environment, not the target environment. ++ compiler = ccompiler.new_compiler() ++ compiler.compiler = [os.environ['CC_FOR_BUILD'],] ++ compiler.compiler_cxx = [os.environ['CXX_FOR_BUILD'],] ++ compiler.compiler_so = [os.environ['CC_FOR_BUILD'],] ++ compiler.linker_exe = [os.environ['CC_FOR_BUILD'], f'-B{os.environ["BUILD_PREFIX"]}/bin'] ++ compiler.linker_so = [os.environ['CC_FOR_BUILD'], f'-B{os.environ["BUILD_PREFIX"]}/bin', '-shared'] ++ ++ available_modules = [] ++ if no_cuda: ++ available_modules = [m['name'] for m in MODULES] ++ else: ++ available_modules, settings = preconfigure_modules( ++ ctx, MODULES, compiler, settings) ++ required_modules = get_required_modules(MODULES) ++ if not (set(required_modules) <= set(available_modules)): ++ raise Exception('Your CUDA environment is invalid. ' ++ 'Please check above error log.') ++ finally: ++ compiler = host_compiler ++ if os.environ.get('CONDA_BUILD_CROSS_COMPILATION'): ++ os.remove(f'{os.environ["BUILD_PREFIX"]}/bin/ld') + + ret = [] + for module in MODULES: +diff --git a/install/cupy_builder/install_build.py b/install/cupy_builder/install_build.py +index d0317e324..759d27157 100644 +--- a/install/cupy_builder/install_build.py ++++ b/install/cupy_builder/install_build.py +@@ -1,7 +1,9 @@ + # mypy: ignore-errors + + import contextlib ++import logging + import os ++import platform + import re + import shlex + import shutil +@@ -16,6 +18,9 @@ from cupy_builder import _environment + from cupy_builder._context import Context + + ++if os.environ.get('CONDA_BUILD', '0') == '1': ++ logging.basicConfig(level=logging.DEBUG) ++ + PLATFORM_LINUX = sys.platform.startswith('linux') + PLATFORM_WIN32 = sys.platform.startswith('win32') + +@@ -689,9 +694,59 @@ def get_cusparselt_version(formatted=False): + return _cusparselt_version + + ++def conda_get_target_name(): ++ out = None ++ if PLATFORM_LINUX: ++ plat = platform.processor() ++ if plat == "aarch64": ++ out = "sbsa-linux" ++ else: ++ out = f"{plat}-linux" ++ else: ++ raise NotImplementedError ++ logging.debug(f"{out=}") ++ return out ++ ++ ++def conda_update_dirs(include_dirs, library_dirs): ++ # Note: These hacks are needed for the dependency detection stage to function, ++ # because we create a fresh compiler instance that does not honor CFLAGS etc set ++ # in the conda-build environment. ++ include_dirs = list(include_dirs) ++ library_dirs = list(library_dirs) ++ ++ if os.environ.get('CONDA_BUILD_CROSS_COMPILATION'): ++ # If we're cross compiling, we need to generate stub files that are ++ # executable in the build environment, not the target environment. ++ # This assumes, however, that the build/host environments see the same ++ # CUDA Toolkit. ++ if os.environ.get('CONDA_OVERRIDE_CUDA', '0').startswith('12'): ++ include_dirs.insert(0, f'{os.environ["BUILD_PREFIX"]}/targets/x86_64-linux/include') ++ library_dirs.insert(0, f'{os.environ["BUILD_PREFIX"]}/targets/x86_64-linux/lib') ++ library_dirs.insert(0, f'{os.environ["BUILD_PREFIX"]}/lib/stubs') ++ elif os.environ.get('CONDA_OVERRIDE_CUDA', '0').startswith('11'): ++ include_dirs.append('/usr/local/cuda/include') ++ library_dirs.append('/usr/local/cuda/lib64/stubs') ++ ++ # for optional dependencies ++ include_dirs.append(f'{os.environ["BUILD_PREFIX"]}/include') ++ library_dirs.append(f'{os.environ["BUILD_PREFIX"]}/lib') ++ ++ if os.environ.get('CONDA_OVERRIDE_CUDA', '0').startswith('12'): ++ include_dirs.append(f'{os.environ["BUILD_PREFIX"]}/targets/{conda_get_target_name()}/include') # for crt headers ++ library_dirs.append(f'{os.environ["PREFIX"]}/lib/stubs') ++ # for optional dependencies ++ include_dirs.append(f'{os.environ["PREFIX"]}/include') ++ library_dirs.append(f'{os.environ["PREFIX"]}/lib') ++ ++ return include_dirs, library_dirs ++ ++ + def build_shlib(compiler, source, libraries=(), + include_dirs=(), library_dirs=(), define_macros=None, + extra_compile_args=()): ++ include_dirs, library_dirs = conda_update_dirs(include_dirs, library_dirs) ++ + with _tempdir() as temp_dir: + fname = os.path.join(temp_dir, 'a.cpp') + with open(fname, 'w') as f: +@@ -717,6 +772,8 @@ def build_shlib(compiler, source, libraries=(), + def build_and_run(compiler, source, libraries=(), + include_dirs=(), library_dirs=(), define_macros=None, + extra_compile_args=()): ++ include_dirs, library_dirs = conda_update_dirs(include_dirs, library_dirs) ++ + with _tempdir() as temp_dir: + fname = os.path.join(temp_dir, 'a.cpp') + with open(fname, 'w') as f: diff --git a/recipe/support_nvtx3_win.diff b/recipe/support_nvtx3_win.diff new file mode 100644 index 00000000..fae5e035 --- /dev/null +++ b/recipe/support_nvtx3_win.diff @@ -0,0 +1,12 @@ +diff --git a/install/cupy_builder/_environment.py b/install/cupy_builder/_environment.py +index 914b62c00..6363c6b91 100644 +--- a/install/cupy_builder/_environment.py ++++ b/install/cupy_builder/_environment.py +@@ -33,5 +33,7 @@ def get_nvtx_path() -> Optional[str]: + nvtx = candidates[-1] + print(f'Using NVTX at: {nvtx}') + return nvtx ++ if os.environ.get('CONDA_BUILD', '0') == '1': ++ return os.environ['PREFIX'] + print('NVTX could not be found') + return None From cda7081134b48ec70d42e7b9236832dbe08d1e09 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 4 Oct 2023 15:55:07 +0200 Subject: [PATCH 44/71] rerender with latest https://github.com/conda-forge/conda-smithy/pull/1703 MNT: Re-rendered with conda-build 0.0.0.dev0+placeholder, conda-smithy 0.0.0, and conda-forge-pinning 2023.10.03.20.36.08 --- ...piler_version11python3.9.____cpython.yaml} | 10 ++-- ...mpiler_version12python3.9.____cpython.yaml | 41 +++++++++++++++++ .github/workflows/conda-build.yml | 46 +++++++------------ .scripts/build_steps.sh | 9 ++-- .scripts/logging_utils.sh | 4 +- conda-forge.yml | 1 + 6 files changed, 69 insertions(+), 42 deletions(-) rename .ci_support/{linux_64_c_compiler_version10cuda_compiler_version11.1cudnn8cxx_compiler_version10python3.9.____cpython.yaml => linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml} (85%) create mode 100644 .ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml diff --git a/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.1cudnn8cxx_compiler_version10python3.9.____cpython.yaml b/.ci_support/linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml similarity index 85% rename from .ci_support/linux_64_c_compiler_version10cuda_compiler_version11.1cudnn8cxx_compiler_version10python3.9.____cpython.yaml rename to .ci_support/linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml index 2cd8bd3e..bbade3db 100644 --- a/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.1cudnn8cxx_compiler_version10python3.9.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml @@ -1,7 +1,7 @@ c_compiler: - gcc c_compiler_version: -- '10' +- '11' cdt_name: - cos7 channel_sources: @@ -11,7 +11,7 @@ channel_targets: cuda_compiler: - nvcc cuda_compiler_version: -- '11.1' +- '11.8' cudnn: - '8' cutensor: @@ -19,9 +19,9 @@ cutensor: cxx_compiler: - gxx cxx_compiler_version: -- '10' +- '11' docker_image: -- quay.io/condaforge/linux-anvil-cuda:11.1 +- quay.io/condaforge/linux-anvil-cuda:11.8 nccl: - '2' pin_run_as_build: @@ -35,7 +35,7 @@ target_platform: zip_keys: - - c_compiler_version - cxx_compiler_version - - cudnn + - cuda_compiler - cuda_compiler_version - cdt_name - docker_image diff --git a/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml new file mode 100644 index 00000000..3f872fa1 --- /dev/null +++ b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml @@ -0,0 +1,41 @@ +c_compiler: +- gcc +c_compiler_version: +- '12' +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler: +- cuda-nvcc +cuda_compiler_version: +- '12.0' +cudnn: +- '8' +cutensor: +- '1' +cxx_compiler: +- gxx +cxx_compiler_version: +- '12' +docker_image: +- quay.io/condaforge/linux-anvil-cos7-x86_64 +nccl: +- '2' +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.9.* *_cpython +target_platform: +- linux-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version + - cuda_compiler + - cuda_compiler_version + - cdt_name + - docker_image diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index ca3cf7b6..20ffa9b0 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -12,20 +12,30 @@ jobs: build: name: ${{ matrix.CONFIG }} runs-on: ${{ matrix.labels }} - timeout-minutes: 720 # 12h + timeout-minutes: 720 strategy: fail-fast: false matrix: include: - - CONFIG: linux_64_c_compiler_version10cuda_compiler_version11.1cudnn8cxx_compiler_version10python3.9.____cpython + - CONFIG: linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu labels: - self-hosted - linux - x64 - - cirun-openstack-gpu--${{ github.run_id }}-linux_64_c_compiler_version10cuda_c_h77ddf0f142 - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.1 + - cirun-openstack-gpu--${{ github.run_id }}-linux_64_c_compiler_version11cuda_c_hfa7a9894d1 + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.8 + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" + - CONFIG: linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython + UPLOAD_PACKAGES: True + os: ubuntu + labels: + - self-hosted + - linux + - x64 + - cirun-openstack-gpu--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h625cfcfe2e + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" steps: - name: Checkout code @@ -84,35 +94,11 @@ jobs: miniforge-version: latest miniforge-variant: Mambaforge if: matrix.os == 'windows' + - name: Build on windows shell: cmd run: | - call activate base - mamba.exe install -c conda-forge 'python=3.9' conda-build conda pip boa conda-forge-ci-setup=3 - if errorlevel 1 exit 1 - setup_conda_rc .\ ".\recipe" .\.ci_support\%CONFIG%.yaml - if errorlevel 1 exit 1 - - - if EXIST LICENSE.txt ( - copy LICENSE.txt "recipe\\recipe-scripts-license.txt" - ) - conda.exe mambabuild "recipe" -m .ci_support\%CONFIG%.yaml - if errorlevel 1 exit 1 - set "FEEDSTOCK_NAME=%GITHUB_REPOSITORY:*/=%" - set "GIT_BRANCH=%GITHUB_REF:refs/heads/=%" - if /i "%GITHUB_EVENT_NAME%" == "pull_request" ( - set "IS_PR_BUILD=True" - ) else ( - set "IS_PR_BUILD=False" - ) - validate_recipe_outputs "%FEEDSTOCK_NAME%" - if errorlevel 1 exit 1 - if /i "%UPLOAD_PACKAGES%" == "true" ( - if /i "%IS_PR_BUILD%" == "false" ( - upload_package --validate --feedstock-name="%FEEDSTOCK_NAME%" .\ ".\recipe" .ci_support\%CONFIG%.yaml - ) - ) + call ".scripts\run_win_build.bat" env: PYTHONUNBUFFERED: 1 CONFIG: ${{ matrix.CONFIG }} diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 595f8b5e..438ed2b9 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -31,11 +31,10 @@ pkgs_dirs: CONDARC - -mamba install --update-specs --yes --quiet --channel conda-forge \ - conda-build pip boa conda-forge-ci-setup=3 -mamba update --update-specs --yes --quiet --channel conda-forge \ - conda-build pip boa conda-forge-ci-setup=3 +mamba install --update-specs --yes --quiet --channel conda-forge --strict-channel-priority \ + pip mamba conda-build boa conda-forge-ci-setup=3 +mamba update --update-specs --yes --quiet --channel conda-forge --strict-channel-priority \ + pip mamba conda-build boa conda-forge-ci-setup # set up the condarc setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" diff --git a/.scripts/logging_utils.sh b/.scripts/logging_utils.sh index 57bc95c2..aff009f0 100644 --- a/.scripts/logging_utils.sh +++ b/.scripts/logging_utils.sh @@ -12,7 +12,7 @@ function startgroup { echo "##[group]$1";; travis ) echo "$1" - echo -en 'travis_fold:start:'"${1// /}"'\\r';; + echo -en 'travis_fold:start:'"${1// /}"'\r';; github_actions ) echo "::group::$1";; * ) @@ -28,7 +28,7 @@ function endgroup { azure ) echo "##[endgroup]";; travis ) - echo -en 'travis_fold:end:'"${1// /}"'\\r';; + echo -en 'travis_fold:end:'"${1// /}"'\r';; github_actions ) echo "::endgroup::";; esac diff --git a/conda-forge.yml b/conda-forge.yml index ae8ae754..d98263b9 100644 --- a/conda-forge.yml +++ b/conda-forge.yml @@ -18,5 +18,6 @@ upload_on_branch: main github_actions: self_hosted: true self_hosted_triggers: ["pull_request"] + self_hosted_timeout_minutes: 720 self_hosted_labels: - cirun-openstack-gpu From 83021cc87e59c7f213ec74c2f86ea5163c043dfd Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 4 Oct 2023 16:11:45 +0200 Subject: [PATCH 45/71] Remove .cirun.yml (uses global instead) --- .cirun.yml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100755 .cirun.yml diff --git a/.cirun.yml b/.cirun.yml deleted file mode 100755 index d4d9e851..00000000 --- a/.cirun.yml +++ /dev/null @@ -1,15 +0,0 @@ -runners: - - # Custom runner 1 - name: cirun-openstack-gpu - labels: ['linux', 'x64', 'self-hosted', 'cirun-openstack-gpu'] - cloud: openstack - instance_type: gpu_large - machine_image: ubuntu-2204-nvidia-container-runtime-20230107 - region: RegionOne - - # Custom runner 2 - name: cirun-openstack-cpu - labels: ['linux', 'x64', 'self-hosted', 'cirun-openstack-cpu'] - cloud: openstack - instance_type: ci_medium - machine_image: ubuntu-2204-cloud-jammy-20221104 - region: RegionOne \ No newline at end of file From 69e0962e28b9d16b4bd9cce758ca8ca80bce605e Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 4 Oct 2023 17:00:58 +0200 Subject: [PATCH 46/71] fix runner label --- .github/workflows/conda-build.yml | 4 ++-- conda-forge.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index 20ffa9b0..5258f313 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -24,7 +24,7 @@ jobs: - self-hosted - linux - x64 - - cirun-openstack-gpu--${{ github.run_id }}-linux_64_c_compiler_version11cuda_c_hfa7a9894d1 + - cirun-openstack-gpu-large--${{ github.run_id }}-linux_64_c_compiler_version11cuda_c_hfa7a9894d1 DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.8 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython @@ -34,7 +34,7 @@ jobs: - self-hosted - linux - x64 - - cirun-openstack-gpu--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h625cfcfe2e + - cirun-openstack-gpu-large--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h625cfcfe2e DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" steps: diff --git a/conda-forge.yml b/conda-forge.yml index d98263b9..e2e3e2c6 100644 --- a/conda-forge.yml +++ b/conda-forge.yml @@ -20,4 +20,4 @@ github_actions: self_hosted_triggers: ["pull_request"] self_hosted_timeout_minutes: 720 self_hosted_labels: - - cirun-openstack-gpu + - cirun-openstack-gpu-large From bbba469c37e70a19968c2c17478a70b34e1ca260 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 4 Oct 2023 19:11:16 +0200 Subject: [PATCH 47/71] enable ppc and arm again --- ...mpiler_version11python3.9.____cpython.yaml | 45 +++++++++++++++++++ ...mpiler_version12python3.9.____cpython.yaml | 45 +++++++++++++++++++ ...mpiler_version11python3.9.____cpython.yaml | 41 +++++++++++++++++ ...mpiler_version12python3.9.____cpython.yaml | 41 +++++++++++++++++ .github/workflows/conda-build.yml | 36 +++++++++++++++ recipe/meta.yaml | 2 - 6 files changed, 208 insertions(+), 2 deletions(-) create mode 100644 .ci_support/linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml create mode 100644 .ci_support/linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml create mode 100644 .ci_support/linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml create mode 100644 .ci_support/linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml diff --git a/.ci_support/linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml b/.ci_support/linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml new file mode 100644 index 00000000..9135007a --- /dev/null +++ b/.ci_support/linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml @@ -0,0 +1,45 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +c_compiler: +- gcc +c_compiler_version: +- '11' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '11.8' +cudnn: +- '8' +cutensor: +- '1' +cxx_compiler: +- gxx +cxx_compiler_version: +- '11' +docker_image: +- quay.io/condaforge/linux-anvil-aarch64-cuda:11.8 +nccl: +- '2' +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.9.* *_cpython +target_platform: +- linux-aarch64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version + - cuda_compiler + - cuda_compiler_version + - cdt_name + - docker_image diff --git a/.ci_support/linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml b/.ci_support/linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml new file mode 100644 index 00000000..e46d940f --- /dev/null +++ b/.ci_support/linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml @@ -0,0 +1,45 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +c_compiler: +- gcc +c_compiler_version: +- '12' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler: +- cuda-nvcc +cuda_compiler_version: +- '12.0' +cudnn: +- '8' +cutensor: +- '1' +cxx_compiler: +- gxx +cxx_compiler_version: +- '12' +docker_image: +- quay.io/condaforge/linux-anvil-aarch64 +nccl: +- '2' +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.9.* *_cpython +target_platform: +- linux-aarch64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version + - cuda_compiler + - cuda_compiler_version + - cdt_name + - docker_image diff --git a/.ci_support/linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml b/.ci_support/linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml new file mode 100644 index 00000000..4eb0264c --- /dev/null +++ b/.ci_support/linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml @@ -0,0 +1,41 @@ +c_compiler: +- gcc +c_compiler_version: +- '11' +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '11.8' +cudnn: +- '8' +cutensor: +- '1' +cxx_compiler: +- gxx +cxx_compiler_version: +- '11' +docker_image: +- quay.io/condaforge/linux-anvil-ppc64le-cuda:11.8 +nccl: +- '2' +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.9.* *_cpython +target_platform: +- linux-ppc64le +zip_keys: +- - c_compiler_version + - cxx_compiler_version + - cuda_compiler + - cuda_compiler_version + - cdt_name + - docker_image diff --git a/.ci_support/linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml b/.ci_support/linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml new file mode 100644 index 00000000..f86c5893 --- /dev/null +++ b/.ci_support/linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml @@ -0,0 +1,41 @@ +c_compiler: +- gcc +c_compiler_version: +- '12' +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler: +- cuda-nvcc +cuda_compiler_version: +- '12.0' +cudnn: +- '8' +cutensor: +- '1' +cxx_compiler: +- gxx +cxx_compiler_version: +- '12' +docker_image: +- quay.io/condaforge/linux-anvil-ppc64le +nccl: +- '2' +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.9.* *_cpython +target_platform: +- linux-ppc64le +zip_keys: +- - c_compiler_version + - cxx_compiler_version + - cuda_compiler + - cuda_compiler_version + - cdt_name + - docker_image diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index 5258f313..a1c8923a 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -37,6 +37,42 @@ jobs: - cirun-openstack-gpu-large--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h625cfcfe2e DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" + - CONFIG: linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython + UPLOAD_PACKAGES: True + os: ubuntu + labels: + - self-hosted + - linux + - cirun-openstack-gpu-large--${{ github.run_id }}-linux_aarch64_c_compiler_version11c_hcf4a71f7ac + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-aarch64-cuda:11.8 + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" + - CONFIG: linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython + UPLOAD_PACKAGES: True + os: ubuntu + labels: + - self-hosted + - linux + - cirun-openstack-gpu-large--${{ github.run_id }}-linux_aarch64_c_compiler_version12c_hd91e7292ec + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-aarch64 + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" + - CONFIG: linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython + UPLOAD_PACKAGES: True + os: ubuntu + labels: + - self-hosted + - linux + - cirun-openstack-gpu-large--${{ github.run_id }}-linux_ppc64le_c_compiler_version11c_h98abef49c6 + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-ppc64le-cuda:11.8 + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" + - CONFIG: linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython + UPLOAD_PACKAGES: True + os: ubuntu + labels: + - self-hosted + - linux + - cirun-openstack-gpu-large--${{ github.run_id }}-linux_ppc64le_c_compiler_version12c_h9d041e80d9 + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-ppc64le + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 83c7d095..bcd30553 100755 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -29,8 +29,6 @@ build: # TODO: turn on win64 + CUDA 12 once it's ready skip: true # [cuda_compiler_version not in ("11.8", "12.0") or (cuda_compiler_version == "12.0" and win64)] skip: true # [win] - skip: true # [aarch64] - skip: true # [ppc64le] skip: true # [py != 39] script: # CuPy default detects CUDA from nvcc, but on Conda-Forge's dockers nvcc lives in a different place... From 8133ff1818ab777e021e86d4f05c44695fe7418a Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 5 Oct 2023 11:31:28 +0200 Subject: [PATCH 48/71] cross-compile --- ...8cxx_compiler_version11python3.9.____cpython.yaml | 2 +- ...0cxx_compiler_version12python3.9.____cpython.yaml | 2 +- ...8cxx_compiler_version11python3.9.____cpython.yaml | 2 +- ...0cxx_compiler_version12python3.9.____cpython.yaml | 2 +- .github/workflows/conda-build.yml | 12 ++++++++---- conda-forge.yml | 3 +++ 6 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.ci_support/linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml b/.ci_support/linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml index 9135007a..6a975279 100644 --- a/.ci_support/linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml +++ b/.ci_support/linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml @@ -25,7 +25,7 @@ cxx_compiler: cxx_compiler_version: - '11' docker_image: -- quay.io/condaforge/linux-anvil-aarch64-cuda:11.8 +- quay.io/condaforge/linux-anvil-cuda:11.8 nccl: - '2' pin_run_as_build: diff --git a/.ci_support/linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml b/.ci_support/linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml index e46d940f..7bf10727 100644 --- a/.ci_support/linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml +++ b/.ci_support/linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml @@ -25,7 +25,7 @@ cxx_compiler: cxx_compiler_version: - '12' docker_image: -- quay.io/condaforge/linux-anvil-aarch64 +- quay.io/condaforge/linux-anvil-cos7-x86_64 nccl: - '2' pin_run_as_build: diff --git a/.ci_support/linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml b/.ci_support/linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml index 4eb0264c..a8f65748 100644 --- a/.ci_support/linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml +++ b/.ci_support/linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml @@ -21,7 +21,7 @@ cxx_compiler: cxx_compiler_version: - '11' docker_image: -- quay.io/condaforge/linux-anvil-ppc64le-cuda:11.8 +- quay.io/condaforge/linux-anvil-cuda:11.8 nccl: - '2' pin_run_as_build: diff --git a/.ci_support/linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml b/.ci_support/linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml index f86c5893..1df31ed9 100644 --- a/.ci_support/linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml +++ b/.ci_support/linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml @@ -21,7 +21,7 @@ cxx_compiler: cxx_compiler_version: - '12' docker_image: -- quay.io/condaforge/linux-anvil-ppc64le +- quay.io/condaforge/linux-anvil-cos7-x86_64 nccl: - '2' pin_run_as_build: diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index a1c8923a..cd5fa97f 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -43,8 +43,9 @@ jobs: labels: - self-hosted - linux + - x64 - cirun-openstack-gpu-large--${{ github.run_id }}-linux_aarch64_c_compiler_version11c_hcf4a71f7ac - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-aarch64-cuda:11.8 + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.8 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython UPLOAD_PACKAGES: True @@ -52,8 +53,9 @@ jobs: labels: - self-hosted - linux + - x64 - cirun-openstack-gpu-large--${{ github.run_id }}-linux_aarch64_c_compiler_version12c_hd91e7292ec - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-aarch64 + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython UPLOAD_PACKAGES: True @@ -61,8 +63,9 @@ jobs: labels: - self-hosted - linux + - x64 - cirun-openstack-gpu-large--${{ github.run_id }}-linux_ppc64le_c_compiler_version11c_h98abef49c6 - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-ppc64le-cuda:11.8 + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.8 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython UPLOAD_PACKAGES: True @@ -70,8 +73,9 @@ jobs: labels: - self-hosted - linux + - x64 - cirun-openstack-gpu-large--${{ github.run_id }}-linux_ppc64le_c_compiler_version12c_h9d041e80d9 - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-ppc64le + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" steps: - name: Checkout code diff --git a/conda-forge.yml b/conda-forge.yml index e2e3e2c6..067ba260 100644 --- a/conda-forge.yml +++ b/conda-forge.yml @@ -21,3 +21,6 @@ github_actions: self_hosted_timeout_minutes: 720 self_hosted_labels: - cirun-openstack-gpu-large +build_platform: + linux_aarch64: linux_64 + linux_ppc64le: linux_64 From dd93b3abbdd88a45a47050ed05fd0d30162cb18c Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 9 Oct 2023 13:30:49 +0200 Subject: [PATCH 49/71] use BUILD_PREFIX --- recipe/run_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/run_test.py b/recipe/run_test.py index b723210c..dab00930 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -36,7 +36,7 @@ def get_target_name(): # Note: This would make the actual tests fail on GPU CI! if sys.platform.startswith('linux') and '12' in os.environ.get('CONDA_OVERRIDE_CUDA', ''): try: - stub = f"{os.environ['PREFIX']}/targets/{get_target_name()}/lib/stubs/libcuda.so" + stub = f"{os.environ['BUILD_PREFIX']}/targets/{get_target_name()}/lib/stubs/libcuda.so" stub = ctypes.CDLL(stub) except Exception as e: print(f"{stub=} was not loaded:") From 0291b6f1d4dd6ade2815ae5cd9f6c6481dbff3c3 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 9 Oct 2023 13:33:47 +0200 Subject: [PATCH 50/71] only test on native --- .scripts/build_steps.sh | 4 +++- conda-forge.yml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 438ed2b9..d0fc82af 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -44,7 +44,9 @@ source run_conda_forge_build_setup # make the build number clobber make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" - +if [[ "${HOST_PLATFORM}" != "${BUILD_PLATFORM}" ]] && [[ "${BUILD_WITH_CONDA_DEBUG:-0}" != 1 ]]; then + EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test" +fi ( endgroup "Configuring conda" ) 2> /dev/null diff --git a/conda-forge.yml b/conda-forge.yml index 067ba260..1cd09398 100644 --- a/conda-forge.yml +++ b/conda-forge.yml @@ -24,3 +24,4 @@ github_actions: build_platform: linux_aarch64: linux_64 linux_ppc64le: linux_64 +test: native From 789b3801a42cb119d9911730c2e917163fee54ad Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 10 Oct 2023 10:13:34 +0200 Subject: [PATCH 51/71] enable tests --- recipe/run_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/recipe/run_test.py b/recipe/run_test.py index dab00930..98c5cf3d 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -72,6 +72,6 @@ def get_target_name(): sys.exit(0) ## Run CuPy's test suite -#import py -#py.test.cmdline.main(["tests/cupy_tests"]) -#py.test.cmdline.main(["tests/cupyx_tests"]) +import py +py.test.cmdline.main(["tests/cupy_tests"]) +py.test.cmdline.main(["tests/cupyx_tests"]) From 65181b6d5beac8987b9c0ea97164beaf5fce4b6d Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 10 Oct 2023 10:59:42 +0200 Subject: [PATCH 52/71] remove early exit --- recipe/run_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/recipe/run_test.py b/recipe/run_test.py index 98c5cf3d..4ab6bcbc 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -49,9 +49,10 @@ def get_target_name(): if sys.platform.startswith('win32'): print("No driver available on Windows. Exiting...") sys.exit(0) + raise e else: print("import ok") - sys.exit(0) + # sys.exit(0) # Ensure CuPy picks up the correct CUDA_VERSION from cupy.cuda import driver From d3103d3d9d39b48c6c1a0bb6e6eb920730a73adf Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 10 Oct 2023 11:34:53 +0200 Subject: [PATCH 53/71] install pytest --- recipe/meta.yaml | 2 +- recipe/run_test.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index bcd30553..7365a95c 100755 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -159,7 +159,7 @@ test: requires: # need the libcuda stub for import test - cuda-driver-dev # [(cuda_compiler_version or "").startswith("12")] - + - pytest source_files: - tests diff --git a/recipe/run_test.py b/recipe/run_test.py index 4ab6bcbc..098a39d6 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -73,6 +73,6 @@ def get_target_name(): sys.exit(0) ## Run CuPy's test suite -import py -py.test.cmdline.main(["tests/cupy_tests"]) -py.test.cmdline.main(["tests/cupyx_tests"]) +import pytest +pytest.cmdline.main(["tests/cupy_tests"]) +pytest.cmdline.main(["tests/cupyx_tests"]) From dda5441a13104333c7682806e54e8df1e6fbba18 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 10 Oct 2023 11:35:02 +0200 Subject: [PATCH 54/71] print driver version --- recipe/run_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recipe/run_test.py b/recipe/run_test.py index 098a39d6..55fbce51 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -62,6 +62,8 @@ def get_target_name(): if ver != cuda_ver: raise ValueError('CUDA version {0} != cuda_compiler_version {1}'.format( ver, cuda_ver)) +else: + print('CUDA version {0} == cuda_compiler_version {1}'.format(ver, cuda_ver)) try: # Print CuPy runtime info From d3682625455c65f6519e767db990f152f54a50be Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 10 Oct 2023 11:37:00 +0200 Subject: [PATCH 55/71] remove stub? --- recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 7365a95c..952ed893 100755 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -158,7 +158,7 @@ requirements: test: requires: # need the libcuda stub for import test - - cuda-driver-dev # [(cuda_compiler_version or "").startswith("12")] + # - cuda-driver-dev # [(cuda_compiler_version or "").startswith("12")] - pytest source_files: - tests From 27db99005fc47849d09530a52ed2aa6084952802 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 10 Oct 2023 12:39:56 +0200 Subject: [PATCH 56/71] sys.exit pytest result --- recipe/meta.yaml | 4 +++- recipe/run_test.py | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 952ed893..036918e9 100755 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -159,7 +159,9 @@ test: requires: # need the libcuda stub for import test # - cuda-driver-dev # [(cuda_compiler_version or "").startswith("12")] - - pytest + - pytest >=7.2 + - hypothesis >=6.37.2,<6.55.0 + - setuptools source_files: - tests diff --git a/recipe/run_test.py b/recipe/run_test.py index 55fbce51..fdc24cc2 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -76,5 +76,4 @@ def get_target_name(): ## Run CuPy's test suite import pytest -pytest.cmdline.main(["tests/cupy_tests"]) -pytest.cmdline.main(["tests/cupyx_tests"]) +sys.exit(pytest.main(["tests/cupy_tests", "tests/cupyx_tests"])) From cabd2d1100f2303ce9a0cac90dfeff38c7589647 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 10 Oct 2023 14:23:17 +0200 Subject: [PATCH 57/71] more tests adjustments --- recipe/meta.yaml | 9 +++++++++ recipe/run_test.py | 12 +++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 036918e9..c7f2a8fe 100755 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -162,8 +162,17 @@ test: - pytest >=7.2 - hypothesis >=6.37.2,<6.55.0 - setuptools + - {{ compiler("c") }} + - {{ compiler("cxx") }} + - {{ compiler("cuda") }} # tests need nvcc + - cupy + - cudnn + - scipy >=1.7,<1.13 + - cython >=0.29.22,<3 + - optuna >=2.0 source_files: - tests + - setup.cfg about: home: https://cupy.dev/ diff --git a/recipe/run_test.py b/recipe/run_test.py index fdc24cc2..a099e1a4 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -75,5 +75,15 @@ def get_target_name(): sys.exit(0) ## Run CuPy's test suite +if sys.platform.startswith("linux"): + gcc = os.path.join(os.environ["PREFIX"], "bin", "gcc") + if "CC" in os.environ and not os.path.exists(gcc): + os.symlink(os.environ["CC"], gcc) + + gxx = os.path.join(os.environ["PREFIX"], "bin", "g++") + if "CXX" in os.environ and not os.path.exists(gxx): + os.symlink(os.environ["CXX"], gxx) + import pytest -sys.exit(pytest.main(["tests/cupy_tests", "tests/cupyx_tests"])) + +sys.exit(pytest.main(["tests/cupy_tests", "tests/cupyx_tests", "-vv", "-m", "not slow"])) From 0ece0ce7aa9a910d7bfd1ca371c3faee989f7439 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 12 Oct 2023 10:11:55 +0200 Subject: [PATCH 58/71] pin test-time numpy and scipy to known supported versions --- recipe/meta.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index c7f2a8fe..188b6465 100755 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -167,7 +167,8 @@ test: - {{ compiler("cuda") }} # tests need nvcc - cupy - cudnn - - scipy >=1.7,<1.13 + - numpy 1.24.* + - scipy 1.10.* - cython >=0.29.22,<3 - optuna >=2.0 source_files: From a4f5b5dcfb146563ce62d7ec681de30f87a7a816 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 12 Oct 2023 17:31:07 +0200 Subject: [PATCH 59/71] uncomment this block that would make tests fail on GPU? --- recipe/run_test.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/recipe/run_test.py b/recipe/run_test.py index a099e1a4..a7130e58 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -32,15 +32,15 @@ def get_target_name(): else: raise RuntimeError('_wheel.json is not found in the package') -# dlopen the driver stub so that CuPy can be imported later -# Note: This would make the actual tests fail on GPU CI! -if sys.platform.startswith('linux') and '12' in os.environ.get('CONDA_OVERRIDE_CUDA', ''): - try: - stub = f"{os.environ['BUILD_PREFIX']}/targets/{get_target_name()}/lib/stubs/libcuda.so" - stub = ctypes.CDLL(stub) - except Exception as e: - print(f"{stub=} was not loaded:") - raise +# # dlopen the driver stub so that CuPy can be imported later +# # Note: This would make the actual tests fail on GPU CI! +# if sys.platform.startswith('linux') and '12' in os.environ.get('CONDA_OVERRIDE_CUDA', ''): +# try: +# stub = f"{os.environ['BUILD_PREFIX']}/targets/{get_target_name()}/lib/stubs/libcuda.so" +# stub = ctypes.CDLL(stub) +# except Exception as e: +# print(f"{stub=} was not loaded:") +# raise # TODO: do we not ship a stub on Windows? try: From 72db1383edb189944b46046ca640ba2170761080 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 17 Oct 2023 10:08:18 +0200 Subject: [PATCH 60/71] add missing dev packages for CUDA 12 --- recipe/meta.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 188b6465..ff97059b 100755 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -171,6 +171,10 @@ test: - scipy 1.10.* - cython >=0.29.22,<3 - optuna >=2.0 + {% if cuda_major == 12 %} + - cuda-cudart-dev + - libcufft-dev + {% endif %} source_files: - tests - setup.cfg From 9558e401cc26289f07f947c20f924947116aed1f Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 17 Oct 2023 15:55:53 +0200 Subject: [PATCH 61/71] rerender --- .github/workflows/conda-build.yml | 38 ++++++------------------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index cd5fa97f..e21be481 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -11,7 +11,7 @@ concurrency: jobs: build: name: ${{ matrix.CONFIG }} - runs-on: ${{ matrix.labels }} + runs-on: ${{ matrix.runs_on }} timeout-minutes: 720 strategy: fail-fast: false @@ -20,61 +20,37 @@ jobs: - CONFIG: linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - labels: - - self-hosted - - linux - - x64 - - cirun-openstack-gpu-large--${{ github.run_id }}-linux_64_c_compiler_version11cuda_c_hfa7a9894d1 + runs_on: ['self-hosted', 'linux', 'x64', 'cirun-openstack-gpu-large--${{ github.run_id }}-linux_64_c_compiler_version11cuda_c_hfa7a9894d1'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.8 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - labels: - - self-hosted - - linux - - x64 - - cirun-openstack-gpu-large--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h625cfcfe2e + runs_on: ['self-hosted', 'linux', 'x64', 'cirun-openstack-gpu-large--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h625cfcfe2e'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - labels: - - self-hosted - - linux - - x64 - - cirun-openstack-gpu-large--${{ github.run_id }}-linux_aarch64_c_compiler_version11c_hcf4a71f7ac + runs_on: ['self-hosted', 'linux', 'x64', 'cirun-openstack-gpu-large--${{ github.run_id }}-linux_aarch64_c_compiler_version11c_hcf4a71f7ac'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.8 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - labels: - - self-hosted - - linux - - x64 - - cirun-openstack-gpu-large--${{ github.run_id }}-linux_aarch64_c_compiler_version12c_hd91e7292ec + runs_on: ['self-hosted', 'linux', 'x64', 'cirun-openstack-gpu-large--${{ github.run_id }}-linux_aarch64_c_compiler_version12c_hd91e7292ec'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - labels: - - self-hosted - - linux - - x64 - - cirun-openstack-gpu-large--${{ github.run_id }}-linux_ppc64le_c_compiler_version11c_h98abef49c6 + runs_on: ['self-hosted', 'linux', 'x64', 'cirun-openstack-gpu-large--${{ github.run_id }}-linux_ppc64le_c_compiler_version11c_h98abef49c6'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.8 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - labels: - - self-hosted - - linux - - x64 - - cirun-openstack-gpu-large--${{ github.run_id }}-linux_ppc64le_c_compiler_version12c_h9d041e80d9 + runs_on: ['self-hosted', 'linux', 'x64', 'cirun-openstack-gpu-large--${{ github.run_id }}-linux_ppc64le_c_compiler_version12c_h9d041e80d9'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" steps: From 83ea67dd2ecaedf769588dba9b8088f015fa3036 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 17 Oct 2023 15:56:25 +0200 Subject: [PATCH 62/71] Run tests with reduced verbosity to make logs manageable --- recipe/run_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/run_test.py b/recipe/run_test.py index a7130e58..2c7c5b1d 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -86,4 +86,4 @@ def get_target_name(): import pytest -sys.exit(pytest.main(["tests/cupy_tests", "tests/cupyx_tests", "-vv", "-m", "not slow"])) +sys.exit(pytest.main(["tests/cupy_tests", "tests/cupyx_tests", "-m", "not slow"])) From 0bc06ef0a2232a1b4bd2a10f60500fcad66e5884 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 19 Oct 2023 09:58:24 +0200 Subject: [PATCH 63/71] add libcufft-static --- recipe/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index ff97059b..722d08c0 100755 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -173,6 +173,7 @@ test: - optuna >=2.0 {% if cuda_major == 12 %} - cuda-cudart-dev + - libcufft-static - libcufft-dev {% endif %} source_files: From ab999a566ee66f9b4178cfe202065984f1e87b9f Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 19 Oct 2023 12:03:41 +0200 Subject: [PATCH 64/71] only cupy_tests for now --- recipe/run_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipe/run_test.py b/recipe/run_test.py index 2c7c5b1d..ced7d6e1 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -85,5 +85,5 @@ def get_target_name(): os.symlink(os.environ["CXX"], gxx) import pytest - -sys.exit(pytest.main(["tests/cupy_tests", "tests/cupyx_tests", "-m", "not slow"])) +# "tests/cupyx_tests" +sys.exit(pytest.main(["tests/cupy_tests", "-m", "not slow"])) From 27c4b84d3a261c8d08816097b5e6bd08eb9a4bd5 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Mon, 23 Oct 2023 10:09:15 -0500 Subject: [PATCH 65/71] rerender --- ...mpiler_version11python3.9.____cpython.yaml | 2 ++ ...mpiler_version12python3.9.____cpython.yaml | 2 ++ ...mpiler_version11python3.9.____cpython.yaml | 2 ++ ...mpiler_version12python3.9.____cpython.yaml | 2 ++ ...mpiler_version11python3.9.____cpython.yaml | 2 ++ ...mpiler_version12python3.9.____cpython.yaml | 2 ++ .github/workflows/conda-build.yml | 19 ++++++++----------- .scripts/build_steps.sh | 2 +- conda-forge.yml | 5 +---- recipe/conda_build_config.yaml | 5 +++++ 10 files changed, 27 insertions(+), 16 deletions(-) diff --git a/.ci_support/linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml b/.ci_support/linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml index bbade3db..eac981d5 100644 --- a/.ci_support/linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml @@ -22,6 +22,8 @@ cxx_compiler_version: - '11' docker_image: - quay.io/condaforge/linux-anvil-cuda:11.8 +github_actions_labels: +- cirun-openstack-gpu-large nccl: - '2' pin_run_as_build: diff --git a/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml index 3f872fa1..182fb545 100644 --- a/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml @@ -22,6 +22,8 @@ cxx_compiler_version: - '12' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 +github_actions_labels: +- cirun-openstack-gpu-large nccl: - '2' pin_run_as_build: diff --git a/.ci_support/linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml b/.ci_support/linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml index 6a975279..5873fbf7 100644 --- a/.ci_support/linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml +++ b/.ci_support/linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml @@ -26,6 +26,8 @@ cxx_compiler_version: - '11' docker_image: - quay.io/condaforge/linux-anvil-cuda:11.8 +github_actions_labels: +- cirun-openstack-cpu-large nccl: - '2' pin_run_as_build: diff --git a/.ci_support/linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml b/.ci_support/linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml index 7bf10727..78a0eb6d 100644 --- a/.ci_support/linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml +++ b/.ci_support/linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml @@ -26,6 +26,8 @@ cxx_compiler_version: - '12' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 +github_actions_labels: +- cirun-openstack-cpu-large nccl: - '2' pin_run_as_build: diff --git a/.ci_support/linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml b/.ci_support/linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml index a8f65748..4bd1f65e 100644 --- a/.ci_support/linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml +++ b/.ci_support/linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml @@ -22,6 +22,8 @@ cxx_compiler_version: - '11' docker_image: - quay.io/condaforge/linux-anvil-cuda:11.8 +github_actions_labels: +- cirun-openstack-cpu-large nccl: - '2' pin_run_as_build: diff --git a/.ci_support/linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml b/.ci_support/linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml index 1df31ed9..b2385447 100644 --- a/.ci_support/linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml +++ b/.ci_support/linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml @@ -22,6 +22,8 @@ cxx_compiler_version: - '12' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 +github_actions_labels: +- cirun-openstack-cpu-large nccl: - '2' pin_run_as_build: diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index e21be481..4d58b778 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -3,7 +3,7 @@ # -*- mode: yaml -*- name: Build conda package -on: ['pull_request'] +on: ['push'] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true @@ -20,40 +20,37 @@ jobs: - CONFIG: linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['self-hosted', 'linux', 'x64', 'cirun-openstack-gpu-large--${{ github.run_id }}-linux_64_c_compiler_version11cuda_c_hfa7a9894d1'] + runs_on: ['cirun-openstack-gpu-large--${{ github.run_id }}-linux_64_c_compiler_version11cuda_c_hfa7a9894d1', 'linux', 'x64', 'self-hosted'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.8 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['self-hosted', 'linux', 'x64', 'cirun-openstack-gpu-large--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h625cfcfe2e'] + runs_on: ['cirun-openstack-gpu-large--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h625cfcfe2e', 'linux', 'x64', 'self-hosted'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['self-hosted', 'linux', 'x64', 'cirun-openstack-gpu-large--${{ github.run_id }}-linux_aarch64_c_compiler_version11c_hcf4a71f7ac'] + runs_on: ['cirun-openstack-cpu-large--${{ github.run_id }}-linux_aarch64_c_compiler_version11c_hcf4a71f7ac', 'linux', 'x64', 'self-hosted'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.8 - CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['self-hosted', 'linux', 'x64', 'cirun-openstack-gpu-large--${{ github.run_id }}-linux_aarch64_c_compiler_version12c_hd91e7292ec'] + runs_on: ['cirun-openstack-cpu-large--${{ github.run_id }}-linux_aarch64_c_compiler_version12c_hd91e7292ec', 'linux', 'x64', 'self-hosted'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 - CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['self-hosted', 'linux', 'x64', 'cirun-openstack-gpu-large--${{ github.run_id }}-linux_ppc64le_c_compiler_version11c_h98abef49c6'] + runs_on: ['cirun-openstack-cpu-large--${{ github.run_id }}-linux_ppc64le_c_compiler_version11c_h98abef49c6', 'linux', 'x64', 'self-hosted'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.8 - CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['self-hosted', 'linux', 'x64', 'cirun-openstack-gpu-large--${{ github.run_id }}-linux_ppc64le_c_compiler_version12c_h9d041e80d9'] + runs_on: ['cirun-openstack-cpu-large--${{ github.run_id }}-linux_ppc64le_c_compiler_version12c_h9d041e80d9', 'linux', 'x64', 'self-hosted'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 - CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" steps: + - name: Checkout code uses: actions/checkout@v3 diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index d0fc82af..85e75053 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -34,7 +34,7 @@ CONDARC mamba install --update-specs --yes --quiet --channel conda-forge --strict-channel-priority \ pip mamba conda-build boa conda-forge-ci-setup=3 mamba update --update-specs --yes --quiet --channel conda-forge --strict-channel-priority \ - pip mamba conda-build boa conda-forge-ci-setup + pip mamba conda-build boa conda-forge-ci-setup=3 # set up the condarc setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" diff --git a/conda-forge.yml b/conda-forge.yml index 1cd09398..a528c20a 100644 --- a/conda-forge.yml +++ b/conda-forge.yml @@ -17,10 +17,7 @@ conda_build: upload_on_branch: main github_actions: self_hosted: true - self_hosted_triggers: ["pull_request"] - self_hosted_timeout_minutes: 720 - self_hosted_labels: - - cirun-openstack-gpu-large + timeout_minutes: 720 build_platform: linux_aarch64: linux_64 linux_ppc64le: linux_64 diff --git a/recipe/conda_build_config.yaml b/recipe/conda_build_config.yaml index bd6e07bf..f3a2c0de 100644 --- a/recipe/conda_build_config.yaml +++ b/recipe/conda_build_config.yaml @@ -2,3 +2,8 @@ c_compiler: # [win] - vs2017 # [win] cxx_compiler: # [win] - vs2017 # [win] + +github_actions_labels: + - hosted # [not linux] + - cirun-openstack-gpu-large # [linux and x86_64] + - cirun-openstack-cpu-large # [linux and not x86_64] From 954895dea6a29b73fab1b1377df902a930ae7fd7 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 25 Oct 2023 10:26:15 +0200 Subject: [PATCH 66/71] upgrade to xlarge, run on pull_request, rerender --- ...sion11.8cxx_compiler_version11python3.9.____cpython.yaml | 2 +- ...sion12.0cxx_compiler_version12python3.9.____cpython.yaml | 2 +- .github/workflows/conda-build.yml | 6 +++--- conda-forge.yml | 1 + recipe/conda_build_config.yaml | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.ci_support/linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml b/.ci_support/linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml index eac981d5..6dc4bf9b 100644 --- a/.ci_support/linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml @@ -23,7 +23,7 @@ cxx_compiler_version: docker_image: - quay.io/condaforge/linux-anvil-cuda:11.8 github_actions_labels: -- cirun-openstack-gpu-large +- cirun-openstack-gpu-xlarge nccl: - '2' pin_run_as_build: diff --git a/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml index 182fb545..4e83d29b 100644 --- a/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml @@ -23,7 +23,7 @@ cxx_compiler_version: docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 github_actions_labels: -- cirun-openstack-gpu-large +- cirun-openstack-gpu-xlarge nccl: - '2' pin_run_as_build: diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index 4d58b778..a00bb986 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -3,7 +3,7 @@ # -*- mode: yaml -*- name: Build conda package -on: ['push'] +on: ['pull_request'] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true @@ -20,13 +20,13 @@ jobs: - CONFIG: linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['cirun-openstack-gpu-large--${{ github.run_id }}-linux_64_c_compiler_version11cuda_c_hfa7a9894d1', 'linux', 'x64', 'self-hosted'] + runs_on: ['cirun-openstack-gpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version11cuda_c_hfa7a9894d1', 'linux', 'x64', 'self-hosted'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.8 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['cirun-openstack-gpu-large--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h625cfcfe2e', 'linux', 'x64', 'self-hosted'] + runs_on: ['cirun-openstack-gpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h625cfcfe2e', 'linux', 'x64', 'self-hosted'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython diff --git a/conda-forge.yml b/conda-forge.yml index a528c20a..c8f462ae 100644 --- a/conda-forge.yml +++ b/conda-forge.yml @@ -18,6 +18,7 @@ upload_on_branch: main github_actions: self_hosted: true timeout_minutes: 720 + triggers: [pull_request] build_platform: linux_aarch64: linux_64 linux_ppc64le: linux_64 diff --git a/recipe/conda_build_config.yaml b/recipe/conda_build_config.yaml index f3a2c0de..4845b3f0 100644 --- a/recipe/conda_build_config.yaml +++ b/recipe/conda_build_config.yaml @@ -5,5 +5,5 @@ cxx_compiler: # [win] github_actions_labels: - hosted # [not linux] - - cirun-openstack-gpu-large # [linux and x86_64] + - cirun-openstack-gpu-xlarge # [linux and x86_64] - cirun-openstack-cpu-large # [linux and not x86_64] From 035dcb46a80c24ab84c270e637c1ac361e06b75d Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 25 Oct 2023 14:43:05 +0200 Subject: [PATCH 67/71] add cupyx tests again --- recipe/run_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/recipe/run_test.py b/recipe/run_test.py index ced7d6e1..da4bfc38 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -85,5 +85,4 @@ def get_target_name(): os.symlink(os.environ["CXX"], gxx) import pytest -# "tests/cupyx_tests" -sys.exit(pytest.main(["tests/cupy_tests", "-m", "not slow"])) +sys.exit(pytest.main(["tests/cupy_tests", "tests/cupyx_tests", "-m", "not slow"])) From 68d7ee5ddc553791af185e1786694869089f4ae1 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 26 Oct 2023 10:37:44 +0200 Subject: [PATCH 68/71] store build artifacts --- .github/workflows/conda-build.yml | 59 +++++++++++- .scripts/create_conda_build_artifacts.sh | 113 +++++++++++++++++++++++ conda-forge.yml | 1 + 3 files changed, 172 insertions(+), 1 deletion(-) create mode 100755 .scripts/create_conda_build_artifacts.sh diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index a00bb986..9118573f 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -18,33 +18,39 @@ jobs: matrix: include: - CONFIG: linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython + SHORT_CONFIG: linux_64_c_compiler_version11cuda_c_hfa7a9894d1 UPLOAD_PACKAGES: True os: ubuntu runs_on: ['cirun-openstack-gpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version11cuda_c_hfa7a9894d1', 'linux', 'x64', 'self-hosted'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.8 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython + SHORT_CONFIG: linux_64_c_compiler_version12cuda_c_h625cfcfe2e UPLOAD_PACKAGES: True os: ubuntu runs_on: ['cirun-openstack-gpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h625cfcfe2e', 'linux', 'x64', 'self-hosted'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython + SHORT_CONFIG: linux_aarch64_c_compiler_version11c_hcf4a71f7ac UPLOAD_PACKAGES: True os: ubuntu runs_on: ['cirun-openstack-cpu-large--${{ github.run_id }}-linux_aarch64_c_compiler_version11c_hcf4a71f7ac', 'linux', 'x64', 'self-hosted'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.8 - CONFIG: linux_aarch64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython + SHORT_CONFIG: linux_aarch64_c_compiler_version12c_hd91e7292ec UPLOAD_PACKAGES: True os: ubuntu runs_on: ['cirun-openstack-cpu-large--${{ github.run_id }}-linux_aarch64_c_compiler_version12c_hd91e7292ec', 'linux', 'x64', 'self-hosted'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 - CONFIG: linux_ppc64le_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython + SHORT_CONFIG: linux_ppc64le_c_compiler_version11c_h98abef49c6 UPLOAD_PACKAGES: True os: ubuntu runs_on: ['cirun-openstack-cpu-large--${{ github.run_id }}-linux_ppc64le_c_compiler_version11c_h98abef49c6', 'linux', 'x64', 'self-hosted'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.8 - CONFIG: linux_ppc64le_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython + SHORT_CONFIG: linux_ppc64le_c_compiler_version12c_h9d041e80d9 UPLOAD_PACKAGES: True os: ubuntu runs_on: ['cirun-openstack-cpu-large--${{ github.run_id }}-linux_ppc64le_c_compiler_version12c_h9d041e80d9', 'linux', 'x64', 'self-hosted'] @@ -121,4 +127,55 @@ jobs: BINSTAR_TOKEN: ${{ secrets.BINSTAR_TOKEN }} FEEDSTOCK_TOKEN: ${{ secrets.FEEDSTOCK_TOKEN }} STAGING_BINSTAR_TOKEN: ${{ secrets.STAGING_BINSTAR_TOKEN }} - if: matrix.os == 'windows' \ No newline at end of file + if: matrix.os == 'windows' + - name: Prepare conda build artifacts + id: prepare-artifacts + shell: bash + if: ${{ always() }} + env: + CONFIG: ${{ matrix.CONFIG }} + SHORT_CONFIG: ${{ matrix.SHORT_CONFIG }} + OS: ${{ matrix.os }} + run: | + export CI=github_actions + export CI_RUN_ID=$GITHUB_RUN_ID + export FEEDSTOCK_NAME="$(basename $GITHUB_REPOSITORY)" + export ARTIFACT_STAGING_DIR="$GITHUB_WORKSPACE" + if [ $OS == "macos" ]; then + export CONDA_BLD_DIR="${MINIFORGE_HOME:-${HOME}/miniforge3}/conda-bld" + elif [ $OS == "windows" ]; then + export CONDA_BLD_DIR="${CONDA//\\//}/conda-bld" + else + export CONDA_BLD_DIR="build_artifacts" + fi + # Archive everything in CONDA_BLD_DIR except environments + # Archive the CONDA_BLD_DIR environments only when the job fails + # Use different prefix for successful and failed build artifacts + # so random failures don't prevent rebuilds from creating artifacts. + JOB_STATUS="${{ job.status }}" + if [ $JOB_STATUS == "failure" ]; then + export BLD_ARTIFACT_PREFIX="conda_artifacts" + export ENV_ARTIFACT_PREFIX="conda_envs" + else + export BLD_ARTIFACT_PREFIX="conda_pkgs" + fi + ./.scripts/create_conda_build_artifacts.sh + continue-on-error: true + + - name: Store conda build artifacts + uses: actions/upload-artifact@v3 + if: ${{ always() && steps.prepare-artifacts.outcome == 'success' }} + with: + name: ${{ steps.prepare-artifacts.outputs.BLD_ARTIFACT_NAME }} + path: ${{ steps.prepare-artifacts.outputs.BLD_ARTIFACT_PATH }} + retention-days: 14 + continue-on-error: true + + - name: Store conda build environment artifacts + uses: actions/upload-artifact@v3 + if: ${{ failure() && steps.prepare-artifacts.outcome == 'success' }} + with: + name: ${{ steps.prepare-artifacts.outputs.ENV_ARTIFACT_NAME }} + path: ${{ steps.prepare-artifacts.outputs.ENV_ARTIFACT_PATH }} + retention-days: 14 + continue-on-error: true \ No newline at end of file diff --git a/.scripts/create_conda_build_artifacts.sh b/.scripts/create_conda_build_artifacts.sh new file mode 100755 index 00000000..17ec0868 --- /dev/null +++ b/.scripts/create_conda_build_artifacts.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +# INPUTS (environment variables that need to be set before calling this script): +# +# CI (azure/github_actions/UNSET) +# CI_RUN_ID (unique identifier for the CI job run) +# FEEDSTOCK_NAME +# CONFIG (build matrix configuration string) +# SHORT_CONFIG (uniquely-shortened configuration string) +# CONDA_BLD_DIR (path to the conda-bld directory) +# ARTIFACT_STAGING_DIR (use working directory if unset) +# BLD_ARTIFACT_PREFIX (prefix for the conda build artifact name, skip if unset) +# ENV_ARTIFACT_PREFIX (prefix for the conda build environments artifact name, skip if unset) + +# OUTPUTS +# +# BLD_ARTIFACT_NAME +# BLD_ARTIFACT_PATH +# ENV_ARTIFACT_NAME +# ENV_ARTIFACT_PATH + +source .scripts/logging_utils.sh + +# DON'T do set -x, because it results in double echo-ing pipeline commands +# and that might end up inserting extraneous quotation marks in output variables +set -e + +# Check that the conda-build directory exists +if [ ! -d "$CONDA_BLD_DIR" ]; then + echo "conda-build directory does not exist" + exit 1 +fi + +# Set staging dir to the working dir, in Windows style if applicable +if [[ -z "${ARTIFACT_STAGING_DIR}" ]]; then + if pwd -W; then + ARTIFACT_STAGING_DIR=$(pwd -W) + else + ARTIFACT_STAGING_DIR=$PWD + fi +fi +echo "ARTIFACT_STAGING_DIR: $ARTIFACT_STAGING_DIR" + +FEEDSTOCK_ROOT=$(cd "$(dirname "$0")/.."; pwd;) +if [ -z ${FEEDSTOCK_NAME} ]; then + export FEEDSTOCK_NAME=$(basename ${FEEDSTOCK_ROOT}) +fi + +# Set a unique ID for the artifact(s), specialized for this particular job run +ARTIFACT_UNIQUE_ID="${CI_RUN_ID}_${CONFIG}" +if [[ ${#ARTIFACT_UNIQUE_ID} -gt 80 ]]; then + ARTIFACT_UNIQUE_ID="${CI_RUN_ID}_${SHORT_CONFIG}" +fi +echo "ARTIFACT_UNIQUE_ID: $ARTIFACT_UNIQUE_ID" + +# Set a descriptive ID for the archive(s), specialized for this particular job run +ARCHIVE_UNIQUE_ID="${CI_RUN_ID}_${CONFIG}" + +# Make the build artifact zip +if [[ ! -z "$BLD_ARTIFACT_PREFIX" ]]; then + export BLD_ARTIFACT_NAME="${BLD_ARTIFACT_PREFIX}_${ARTIFACT_UNIQUE_ID}" + export BLD_ARTIFACT_PATH="${ARTIFACT_STAGING_DIR}/${FEEDSTOCK_NAME}_${BLD_ARTIFACT_PREFIX}_${ARCHIVE_UNIQUE_ID}.zip" + + ( startgroup "Archive conda build directory" ) 2> /dev/null + + # Try 7z and fall back to zip if it fails (for cross-platform use) + if ! 7z a "$BLD_ARTIFACT_PATH" "$CONDA_BLD_DIR" '-xr!.git/' '-xr!_*_env*/' '-xr!*_cache/' -bb; then + pushd "$CONDA_BLD_DIR" + zip -r -y -T "$BLD_ARTIFACT_PATH" . -x '*.git/*' '*_*_env*/*' '*_cache/*' + popd + fi + + ( endgroup "Archive conda build directory" ) 2> /dev/null + + echo "BLD_ARTIFACT_NAME: $BLD_ARTIFACT_NAME" + echo "BLD_ARTIFACT_PATH: $BLD_ARTIFACT_PATH" + + if [[ "$CI" == "azure" ]]; then + echo "##vso[task.setVariable variable=BLD_ARTIFACT_NAME]$BLD_ARTIFACT_NAME" + echo "##vso[task.setVariable variable=BLD_ARTIFACT_PATH]$BLD_ARTIFACT_PATH" + elif [[ "$CI" == "github_actions" ]]; then + echo "BLD_ARTIFACT_NAME=$BLD_ARTIFACT_NAME" >> $GITHUB_OUTPUT + echo "BLD_ARTIFACT_PATH=$BLD_ARTIFACT_PATH" >> $GITHUB_OUTPUT + fi +fi + +# Make the environments artifact zip +if [[ ! -z "$ENV_ARTIFACT_PREFIX" ]]; then + export ENV_ARTIFACT_NAME="${ENV_ARTIFACT_PREFIX}_${ARTIFACT_UNIQUE_ID}" + export ENV_ARTIFACT_PATH="${ARTIFACT_STAGING_DIR}/${FEEDSTOCK_NAME}_${ENV_ARTIFACT_PREFIX}_${ARCHIVE_UNIQUE_ID}.zip" + + ( startgroup "Archive conda build environments" ) 2> /dev/null + + # Try 7z and fall back to zip if it fails (for cross-platform use) + if ! 7z a "$ENV_ARTIFACT_PATH" -r "$CONDA_BLD_DIR"/'_*_env*/' -bb; then + pushd "$CONDA_BLD_DIR" + zip -r -y -T "$ENV_ARTIFACT_PATH" . -i '*_*_env*/*' + popd + fi + + ( endgroup "Archive conda build environments" ) 2> /dev/null + + echo "ENV_ARTIFACT_NAME: $ENV_ARTIFACT_NAME" + echo "ENV_ARTIFACT_PATH: $ENV_ARTIFACT_PATH" + + if [[ "$CI" == "azure" ]]; then + echo "##vso[task.setVariable variable=ENV_ARTIFACT_NAME]$ENV_ARTIFACT_NAME" + echo "##vso[task.setVariable variable=ENV_ARTIFACT_PATH]$ENV_ARTIFACT_PATH" + elif [[ "$CI" == "github_actions" ]]; then + echo "ENV_ARTIFACT_NAME=$ENV_ARTIFACT_NAME" >> $GITHUB_OUTPUT + echo "ENV_ARTIFACT_PATH=$ENV_ARTIFACT_PATH" >> $GITHUB_OUTPUT + fi +fi \ No newline at end of file diff --git a/conda-forge.yml b/conda-forge.yml index c8f462ae..d44fd132 100644 --- a/conda-forge.yml +++ b/conda-forge.yml @@ -19,6 +19,7 @@ github_actions: self_hosted: true timeout_minutes: 720 triggers: [pull_request] + store_build_artifacts: true build_platform: linux_aarch64: linux_64 linux_ppc64le: linux_64 From 6188b68068fb9bf513555674772d1c07489f8b6a Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 26 Oct 2023 10:38:18 +0200 Subject: [PATCH 69/71] test only cupy_tests for quicker job --- recipe/run_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/run_test.py b/recipe/run_test.py index da4bfc38..b791f9dd 100644 --- a/recipe/run_test.py +++ b/recipe/run_test.py @@ -85,4 +85,4 @@ def get_target_name(): os.symlink(os.environ["CXX"], gxx) import pytest -sys.exit(pytest.main(["tests/cupy_tests", "tests/cupyx_tests", "-m", "not slow"])) +sys.exit(pytest.main(["tests/cupy_tests", "-m", "not slow"])) From 33d6471df565ffddacf5ea42c57c3196cae7719d Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 26 Oct 2023 10:39:25 +0200 Subject: [PATCH 70/71] use gpu-large --- ...ersion11.8cxx_compiler_version11python3.9.____cpython.yaml | 2 +- ...ersion12.0cxx_compiler_version12python3.9.____cpython.yaml | 2 +- .github/workflows/conda-build.yml | 4 ++-- recipe/conda_build_config.yaml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.ci_support/linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml b/.ci_support/linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml index 6dc4bf9b..eac981d5 100644 --- a/.ci_support/linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython.yaml @@ -23,7 +23,7 @@ cxx_compiler_version: docker_image: - quay.io/condaforge/linux-anvil-cuda:11.8 github_actions_labels: -- cirun-openstack-gpu-xlarge +- cirun-openstack-gpu-large nccl: - '2' pin_run_as_build: diff --git a/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml index 4e83d29b..182fb545 100644 --- a/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml @@ -23,7 +23,7 @@ cxx_compiler_version: docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 github_actions_labels: -- cirun-openstack-gpu-xlarge +- cirun-openstack-gpu-large nccl: - '2' pin_run_as_build: diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index 9118573f..bf82517e 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -21,14 +21,14 @@ jobs: SHORT_CONFIG: linux_64_c_compiler_version11cuda_c_hfa7a9894d1 UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['cirun-openstack-gpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version11cuda_c_hfa7a9894d1', 'linux', 'x64', 'self-hosted'] + runs_on: ['cirun-openstack-gpu-large--${{ github.run_id }}-linux_64_c_compiler_version11cuda_c_hfa7a9894d1', 'linux', 'x64', 'self-hosted'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.8 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython SHORT_CONFIG: linux_64_c_compiler_version12cuda_c_h625cfcfe2e UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['cirun-openstack-gpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h625cfcfe2e', 'linux', 'x64', 'self-hosted'] + runs_on: ['cirun-openstack-gpu-large--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h625cfcfe2e', 'linux', 'x64', 'self-hosted'] DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_aarch64_c_compiler_version11cuda_compilernvcccuda_compiler_version11.8cxx_compiler_version11python3.9.____cpython diff --git a/recipe/conda_build_config.yaml b/recipe/conda_build_config.yaml index 4845b3f0..f3a2c0de 100644 --- a/recipe/conda_build_config.yaml +++ b/recipe/conda_build_config.yaml @@ -5,5 +5,5 @@ cxx_compiler: # [win] github_actions_labels: - hosted # [not linux] - - cirun-openstack-gpu-xlarge # [linux and x86_64] + - cirun-openstack-gpu-large # [linux and x86_64] - cirun-openstack-cpu-large # [linux and not x86_64] From 6abc3c275278aa47d17e3e585337747e12e02eee Mon Sep 17 00:00:00 2001 From: "conda-forge-curator[bot]" <79913779+conda-forge-curator[bot]@users.noreply.github.com> Date: Wed, 8 May 2024 21:14:43 +0000 Subject: [PATCH 71/71] [ci skip] [skip ci] [cf admin skip] ***NO_CI*** admin migration CondaForgeAutomergeUpdate --- .github/workflows/automerge.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml index da1368a1..0535f6aa 100644 --- a/.github/workflows/automerge.yml +++ b/.github/workflows/automerge.yml @@ -9,8 +9,6 @@ jobs: runs-on: ubuntu-latest name: automerge steps: - - name: checkout - uses: actions/checkout@v3 - name: automerge-action id: automerge-action uses: conda-forge/automerge-action@main