Improve dependency updates

.github/dependabot.yml

@@ -1,16 +1,21 @@
 version: 2
+
 updates:
+
+# Run dependabot only for the development Python requirements (under `/.dev` folder)
 - package-ecosystem: pip
-  directory: "/"
+  directory: "/.dev"
   schedule:
     interval: weekly
     day: monday
     time: "05:18"
-  # Should be bigger than or equal to the total number of dependencies (currently 58)
-  open-pull-requests-limit: 60
+  # Should be bigger than or equal to the total number of dependencies (currently 5)
+  open-pull-requests-limit: 10
   target-branch: ci/dependabot-updates
   labels:
     - CI/CD
+
+# Run dependabot for all GitHub Actions
 - package-ecosystem: github-actions
   directory: "/"
   schedule:

.github/utils/direct_requirements.txt

@@ -0,0 +1,6 @@
+aioredis
+fastapi
+fastapi_plugins
+hypercorn
+oteapi-core
+pydantic

.github/utils/requirements_update_pr_body.txt

@@ -0,0 +1,8 @@
+### Update `requirements.txt`
+
+Automatically created PR from the [`ci_update_requirements.yml` workflow](https://github.com/EMMC-ASBL/oteapi-services/blob/master/.github/workflows/ci_update_requirements.yml).
+
+#### To-do
+
+- [ ] Check that the diff is sensible, and that tests and builds pass with the new dependency versions.
+- [ ] Make sure that the PR is **squash** merged, with a sensible commit message.

.github/workflows/ci_automerge_dependabot.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   update-dependabot-branch:
     name: Update permanent dependabot branch
-    if: github.repository_owner == 'EMMC-ASBL' && startsWith(github.event.pull_request.head.ref, 'dependabot/') && github.actor == 'dependabot[bot]'
+    if: github.repository_owner == 'EMMC-ASBL' && ( ( startsWith(github.event.pull_request.head.ref, 'dependabot/') && github.actor == 'dependabot[bot]' ) || ( github.event.pull_request.head.ref == 'ci/update-requirements' && github.actor 'TEAM4-0' ) )
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/ci_dependabot.yml

@@ -37,7 +37,7 @@ jobs:
           if [[ "${line}" =~ ^pre-commit.*$ ]]; then
             pre_commit="${line}"
           fi
-        done < requirements_dev.txt
+        done < .dev/requirements_dev.txt
 
         pip install ${pre_commit}
 

.github/workflows/ci_tests.yml

@@ -29,7 +29,7 @@ jobs:
           if [[ "${line}" =~ ^pre-commit.*$ ]]; then
             pre_commit="${line}"
           fi
-        done < requirements_dev.txt
+        done < .dev/requirements_dev.txt
 
         pip install ${pre_commit}
 
@@ -53,7 +53,7 @@ jobs:
       run: |
         python -m pip install -U pip
         pip install -U setuptools wheel
-        pip install -U -r requirements.txt -r requirements_dev.txt
+        pip install -U -r requirements.txt -r .dev/requirements_dev.txt
         pip install safety
 
     - name: Run pylint
@@ -107,7 +107,7 @@ jobs:
       run: |
         python -m pip install -U pip
         pip install -U setuptools wheel
-        pip install -r requirements.txt -r requirements_dev.txt
+        pip install -r requirements.txt -r .dev/requirements_dev.txt
 
     - name: Test with pytest
       run: pytest -vvv --cov-report=xml --cov=app

.github/workflows/ci_update_requirements.yml

@@ -0,0 +1,83 @@
+name: CI - Check for updates to `requirements.txt`
+
+on:
+  schedule:
+    # At 7:30 every Monday (6:30 UTC)
+    - cron: "30 6 * * 1"
+
+env:
+  DEPENDABOT_BRANCH: ci/dependabot-updates
+  GIT_USER_NAME: "TEAM 4.0[bot]"
+  GIT_USER_EMAIL: "Team4.0@SINTEF.no"
+
+jobs:
+  update-requirements:
+    name: Update `requirements.txt`
+    if: github.repository_owner == 'EMMC-ASBL'
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        ref: ${{ env.DEPENDABOT_BRANCH }}
+        fetch-depth: 0
+
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+
+    - name: Set up git config
+      run: |
+        git config --global user.name "${{ env.GIT_USER_NAME }}"
+        git config --global user.email "${{ env.GIT_USER_EMAIL }}"
+
+    - name: Install immediate requirements
+      run: |
+        python -m pip install -U pip
+        pip install -U setuptools wheel
+        pip install --no-cache-dir --upgrade-strategy=eager -U -r .github/utils/direct_requirements.txt
+
+    - name: Run `pip freeze` and update current `requirements.txt`
+      run: |
+        pip freeze > requirements_new.txt
+        if [ "$(diff requirements.txt requirements_new.txt)" ]; then
+          # There are updates !
+          mv -f requirements_new.txt requirements.txt
+          echo -e "requirements.txt has been updated. Diff:\n$(git diff requirements.txt)"
+          echo "UPDATE_DEPS=true" >> $GITHUB_ENV
+
+          git add requirements.txt
+          git commit -m "Update \`requirements.txt\`"
+        else
+          # No changes
+          echo "No changes found for requirements.txt."
+          echo "UPDATE_DEPS=false" >> $GITHUB_ENV
+        fi
+
+    - name: Fetch PR body
+      if: env.UPDATE_DEPS == 'true'
+      id: pr_body
+      uses: chuhlomin/render-template@v1.4
+      with:
+        template: .github/utils/requirements_update_pr_body.txt
+
+    - name: Create PR
+      if: env.UPDATE_DEPS == 'true'
+      id: cpr
+      uses: peter-evans/create-pull-request@v3
+      with:
+        token: ${{ secrets.RELEASE_PAT }}
+        commit-message: "Update `requirements.txt`"
+        committer: "${{ env.GIT_USER_NAME }} <${{ env.GIT_USER_EMAIL }}>"
+        author: "${{ env.GIT_USER_NAME }} <${{ env.GIT_USER_EMAIL }}>"
+        branch: ci/update-requirements
+        delete-branch: true
+        title: "[Auto-generated] Update `requirements.txt`"
+        body: ${{ steps.pr_body.outputs.result }}
+        labels: CI/CD
+
+    - name: Information
+      if: env.UPDATE_DEPS == 'true'
+      run: 'echo "${{ steps.cpr.outputs.pull-request-operation }} PR #${{ steps.cpr.outputs.pull-request-number }}: ${{ steps.cpr.outputs.pull-request-url }}"'

Dockerfile

@@ -24,7 +24,7 @@ FROM base as development
 # Copy development parts
 COPY tests tests/
 COPY .git .git/
-COPY requirements_dev.txt .pre-commit-config.yaml .pylintrc ./
+COPY .dev/requirements_dev.txt .pre-commit-config.yaml .pylintrc ./
 
 # Run static security check, linters, and pytest with code coverage
 RUN pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org -r requirements_dev.txt \