pull-request #101
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Jobs that run on every pull-request, regardless of its contents. This file is | |
| # gradually transitioning to run almost everything, with rules defined in this | |
| # file rather than across lots of workflow files. If that effort is successful, | |
| # we can rename it to `tests.yaml` or something similarly general at some point. | |
| name: pull-request | |
| on: | |
| pull_request: | |
| # Add `labeled`, so we can trigger a new run by adding a `pr-test-all` | |
| # label, which we then use to trigger a `test-all` run. | |
| types: [opened, reopened, synchronize, labeled] | |
| push: | |
| branches: | |
| - main | |
| schedule: | |
| # Pick a random time, something that others won't pick, to reduce GH's demand variance. | |
| - cron: "49 10 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| test-all: | |
| type: boolean | |
| default: false | |
| run-nightly: | |
| type: boolean | |
| default: false | |
| workflow_call: | |
| inputs: | |
| test-all: | |
| type: boolean | |
| default: false | |
| run-nightly: | |
| type: boolean | |
| default: false | |
| concurrency: | |
| # This seems to require a custom suffix; I _think_ because this calls | |
| # `test-all`, which calls `test-python`, and if `test-python` runs too, then | |
| # we have two workflows with the same concurrency group, which GitHub calls a | |
| # deadlock. Here's an example: | |
| # https://github.com/PRQL/prql/actions/runs/3945798229. By adding | |
| # `-pull-request` here, the job that runs under `test-all` (called from here) | |
| # will have the suffix, and so won't deadlock. But I'm not certain, and it | |
| # doesn't seem possible to inspect these values to understand the true cause. | |
| group: ${{ github.workflow }}-${{ github.ref }}-pull-request | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| CLICOLOR_FORCE: 1 | |
| RUSTFLAGS: "-C debuginfo=0" | |
| jobs: | |
| # This assesses whether we need to run jobs. It's currently only implemented | |
| # for a few jobs, but we're expanding it to other jobs, bringing the jobs | |
| # into this workflow. Some of them are defined only by the changes in PR, | |
| # others also define a set of other criteria, such as whether a label has been | |
| # added, or we're on `main` branch. | |
| rules: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: read | |
| outputs: | |
| book: ${{ steps.changes.outputs.book }} | |
| dotnet: ${{ steps.changes.outputs.dotnet }} | |
| elixir: ${{ steps.changes.outputs.elixir }} | |
| java: ${{ steps.changes.outputs.java }} | |
| js: ${{ steps.changes.outputs.js }} | |
| lib: ${{ steps.changes.outputs.lib }} | |
| # Really run all tests | |
| nightly: ${{ steps.nightly.outputs.run }} | |
| php: ${{ steps.changes.outputs.php }} | |
| python: ${{ steps.changes.outputs.python }} | |
| rust: ${{ steps.changes.outputs.rust }} | |
| # Run most test (`all` is a misnomer!) | |
| test-all: ${{ steps.all.outputs.run }} | |
| web: ${{ steps.changes.outputs.web }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: dorny/paths-filter@v2 | |
| id: changes | |
| with: | |
| filters: | | |
| all: | |
| # We're expanding to consolidate with `test-all.yaml`. | |
| - Cargo.lock | |
| - "**/Cargo.*" | |
| - .github/workflows/** | |
| book: | |
| - .github/workflows/check-links-book.yaml | |
| - web/book/** | |
| dotnet: | |
| - bindings/prql-dotnet/** | |
| - bindings/prql-lib/** | |
| - .github/workflows/test-dotnet.yaml | |
| elixir: | |
| - bindings/prql-elixir/** | |
| - bindings/prql-lib/** | |
| - .github/workflows/test-elixir.yaml | |
| java: | |
| - bindings/prql-java/** | |
| - bindings/prql-lib/** | |
| - .github/workflows/test-java.yaml | |
| js: | |
| - bindings/prql-js/** | |
| - .github/workflows/test-js.yaml | |
| lib: | |
| - bindings/prql-lib/** | |
| - .github/workflows/test-lib.yaml | |
| nightly: | |
| - .github/workflows/nightly.yaml | |
| php: | |
| - bindings/prql-php/** | |
| - bindings/prql-lib/** | |
| - .github/workflows/test-php.yaml | |
| python: | |
| - bindings/prql-python/** | |
| - .github/workflows/test-python.yaml | |
| rust: | |
| - "**/*.rs" | |
| - crates/** | |
| - web/book/** | |
| web: | |
| - "web/**" | |
| - ".github/workflows/build-web.yaml" | |
| - "**.md" | |
| - id: nightly | |
| run: | |
| echo "run=${{ steps.changes.outputs.nightly == 'true' || | |
| inputs.run-nightly || | |
| contains(github.event.pull_request.labels.*.name, 'pr-nightly') || | |
| github.event.schedule }}" >> "$GITHUB_OUTPUT" | |
| - id: all | |
| # TODO: actionlint annoyingly blocks this — try and find a way of getting | |
| # it back without too much trouble... | |
| # contains(github.event.pull_request.title, '!') || | |
| run: | |
| echo "run=${{ steps.changes.outputs.all == 'true' || inputs.test-all | |
| || github.ref == 'refs/heads/main' || | |
| contains(github.event.pull_request.labels.*.name, 'pr-test-all') || | |
| steps.nightly.outputs.run == 'true' }}" >> "$GITHUB_OUTPUT" | |
| test-rust: | |
| needs: rules | |
| if: | |
| needs.rules.outputs.rust == 'true' || needs.rules.outputs.test-all == | |
| 'true' | |
| uses: ./.github/workflows/test-rust.yaml | |
| strategy: | |
| matrix: | |
| include: | |
| - target: x86_64-unknown-linux-gnu | |
| - target: wasm32-unknown-unknown | |
| with: | |
| os: ubuntu-latest | |
| target: ${{ matrix.target }} | |
| # We previously ran with integration tests, but removing until | |
| # https://github.com/duckdb/duckdb-rs/issues/178 is fixed given compile | |
| # times | |
| features: "" | |
| nightly: ${{ needs.rules.outputs.nightly == 'true' }} | |
| test-python: | |
| needs: rules | |
| if: | |
| needs.rules.outputs.python == 'true' || needs.rules.outputs.test-all == | |
| 'true' | |
| uses: ./.github/workflows/test-python.yaml | |
| with: | |
| # Only run on ubuntu unless there's a lang-specific change or we're | |
| # running nightly. | |
| # | |
| # An alternative to these somewhat horrible expressions would be | |
| # `test-python` & `test-python-more` workflows; though it would use up our | |
| # 20 workflow limit. | |
| oss: | |
| ${{ (needs.rules.outputs.python == 'true' || needs.rules.outputs.nightly | |
| == 'true') && '["ubuntu-latest", "macos-latest", "windows-latest"]' || | |
| '["ubuntu-latest"]' }} | |
| test-js: | |
| needs: rules | |
| if: | |
| needs.rules.outputs.js == 'true' || needs.rules.outputs.test-all == 'true' | |
| uses: ./.github/workflows/test-js.yaml | |
| with: | |
| # Only run on ubuntu unless there's a lang-specific change or we're running nightly. | |
| oss: | |
| ${{ (needs.rules.outputs.js == 'true' || needs.rules.outputs.nightly == | |
| 'true') && '["ubuntu-latest", "macos-latest", "windows-latest"]' || | |
| '["ubuntu-latest"]' }} | |
| test-dotnet: | |
| needs: rules | |
| if: | |
| needs.rules.outputs.dotnet == 'true' || needs.rules.outputs.test-all == | |
| 'true' | |
| uses: ./.github/workflows/test-dotnet.yaml | |
| test-php: | |
| needs: rules | |
| if: | |
| needs.rules.outputs.php == 'true' || needs.rules.outputs.test-all == | |
| 'true' | |
| uses: ./.github/workflows/test-php.yaml | |
| test-java: | |
| needs: rules | |
| if: | |
| needs.rules.outputs.java == 'true' || needs.rules.outputs.test-all == | |
| 'true' | |
| uses: ./.github/workflows/test-java.yaml | |
| with: | |
| # Currently we never run windows | |
| oss: | |
| ${{ (needs.rules.outputs.java == 'true' || needs.rules.outputs.nightly | |
| == 'true') && '["ubuntu-latest", "macos-latest"]' || '["ubuntu-latest"]' | |
| }} | |
| test-elixir: | |
| needs: rules | |
| if: | |
| needs.rules.outputs.elixir == 'true' || needs.rules.outputs.test-all == | |
| 'true' | |
| uses: ./.github/workflows/test-elixir.yaml | |
| with: | |
| # Currently we never run Mac, see prql-elixir docs for details | |
| oss: | |
| ${{ (needs.rules.outputs.elixir == 'true' || needs.rules.outputs.nightly | |
| == 'true') && '["ubuntu-latest", "windows-latest"]' || | |
| '["ubuntu-latest"]' }} | |
| test-lib: | |
| needs: rules | |
| if: | |
| needs.rules.outputs.lib == 'true' || needs.rules.outputs.test-all == | |
| 'true' | |
| uses: ./.github/workflows/test-lib.yaml | |
| build-web: | |
| needs: rules | |
| if: | |
| needs.rules.outputs.web == 'true' || needs.rules.outputs.test-all == | |
| 'true' | |
| uses: ./.github/workflows/build-web.yaml | |
| lint-megalinter: | |
| uses: ./.github/workflows/lint-megalinter.yaml | |
| test-all: | |
| uses: ./.github/workflows/test-all.yaml | |
| needs: rules | |
| if: needs.rules.outputs.test-all == 'true' | |
| publish-web: | |
| uses: ./.github/workflows/publish-web.yaml | |
| if: contains(github.event.pull_request.labels.*.name, 'pr-publish-web') | |
| nightly: | |
| needs: rules | |
| uses: ./.github/workflows/nightly.yaml | |
| if: needs.rules.outputs.nightly == 'true' | |
| check-links-markdown: | |
| # Another option is https://github.com/lycheeverse/lychee, but it was | |
| # weirdly difficult to exclude a directory, and I managed to get | |
| # rate-limited by GH because of it scanning node_modules. | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: gaurav-nelson/github-action-markdown-link-check@v1 | |
| with: | |
| config-file: .config/.markdown-link-check-local.json | |
| base-branch: main | |
| # We also run without this in the nightly workflow | |
| check-modified-files-only: "yes" | |
| check-links-book: | |
| # We also have a check-links-markdown job, however it will not spot mdbook | |
| # mistakes such as forgetting to list an .md file in SUMMARY.md. | |
| # Running a link checker on the generated HTML is more reliable. | |
| needs: rules | |
| if: | |
| needs.rules.outputs.book == 'true' || needs.rules.outputs.nightly == | |
| 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - uses: baptiste0928/cargo-install@v2 | |
| with: | |
| crate: mdbook | |
| # the link checker | |
| - uses: baptiste0928/cargo-install@v2 | |
| with: | |
| crate: hyperlink | |
| - run: ./.github/set_version.sh | |
| - name: Cache | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| prefix-key: ${{ env.version }} | |
| shared-key: web | |
| # Created by `build-web` | |
| save-if: false | |
| # Only build the book — rather than `build-web` which also builds the playground | |
| - name: Build the mdbook | |
| run: mdbook build web/book/ | |
| - name: Check links | |
| run: hyperlink web/book/book/ | |
| measure-code-cov: | |
| runs-on: ubuntu-latest | |
| needs: rules | |
| # TODO: Would be great to have this running on every PR, but | |
| # waiting on https://github.com/PRQL/prql/issues/2870. We could enable it | |
| # but not wait for it? | |
| if: needs.rules.outputs.test-all == 'true' | |
| steps: | |
| - name: 📂 Checkout code | |
| uses: actions/checkout@v3 | |
| - uses: baptiste0928/cargo-install@v2 | |
| with: | |
| crate: cargo-tarpaulin | |
| - run: ./.github/set_version.sh | |
| shell: bash | |
| - name: 💰 Cache | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| prefix-key: ${{ env.version }} | |
| - run: | |
| cargo tarpaulin --skip-clean --all-targets --features=test-dbs | |
| --out=Xml | |
| - name: Upload to codecov.io | |
| uses: codecov/codecov-action@v3 | |
| - name: Upload code coverage results | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: code-coverage-report | |
| path: cobertura.xml | |
| check-ok-to-merge: | |
| # This doesn't run anything, but offers a task for us to tell GitHub | |
| # everything in this workflow has passed and, unlike including each task in | |
| # the branch's GitHub required tests, will pass when a task is skipped. | |
| # | |
| # We're gradually increasing this to cover other workflows, such as | |
| # `test-js` by moving the triggers for those into this workflow, and using | |
| # an external action to assess whether the paths have changed. | |
| if: always() | |
| needs: | |
| - build-web | |
| - check-links-book | |
| - check-links-markdown | |
| - lint-megalinter | |
| - nightly | |
| - publish-web | |
| - test-all | |
| - test-js | |
| - test-elixir | |
| - test-dotnet | |
| - test-java | |
| - test-lib | |
| - test-rust | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Decide whether the needed jobs succeeded or failed | |
| uses: re-actors/alls-green@release/v1 | |
| with: | |
| # We don't include `check-links-markdown`, since occasionally we'll want to merge | |
| # something which temporarily fails that, such as if we're changing the | |
| # location of a file in this repo which is linked to. | |
| # | |
| # We're currently including `nightly` because I'm not sure whether | |
| # it's always reliable; e.g. `cargo-audit` | |
| allowed-failures: check-links-markdown, nightly | |
| # Easier than writing out every job — and it's our own rules deciding | |
| # whether jobs are skipped or not. | |
| allowed-skips: ${{ toJSON(needs) }} | |
| jobs: ${{ toJSON(needs) }} | |
| build-prqlc: | |
| runs-on: ${{ matrix.os }} | |
| needs: rules | |
| if: needs.rules.outputs.rust == 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # Match the features with the available caches from tests | |
| - os: ubuntu-latest | |
| target: x86_64-unknown-linux-musl | |
| features: "" | |
| # TODO: Until we have tests for these, we don't have a cache for them. | |
| # If we can add tests, then re-enable them. They run on `release.yaml` | |
| # regardless. | |
| # | |
| # - os: ubuntu-latest | |
| # target: aarch64-unknown-linux-musl | |
| # - os: macos-latest | |
| # target: aarch64-apple-darwin | |
| - os: macos-latest | |
| target: x86_64-apple-darwin | |
| features: test-dbs | |
| - os: windows-latest | |
| target: x86_64-pc-windows-msvc | |
| features: "" | |
| steps: | |
| - name: 📂 Checkout code | |
| uses: actions/checkout@v3 | |
| - uses: ./.github/actions/build-prqlc | |
| with: | |
| target: ${{ matrix.target }} | |
| profile: dev | |
| features: ${{ matrix.features }} | |
| # These are the same env variables as in `test-rust.yaml`. Custom actions | |
| # don't allow setting env variables for the whole job, so we do it here. (We | |
| # could change the `build-prqlc` action to a workflow...). | |
| env: | |
| CARGO_TERM_COLOR: always | |
| CLICOLOR_FORCE: 1 | |
| RUSTFLAGS: "-C debuginfo=0" | |
| create-issue-on-nightly-failure: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - rules | |
| - check-ok-to-merge | |
| - nightly | |
| if: | |
| # We care that it's on a schedule as well as it running on nightly — we | |
| # don't want to trigger just on a `pr-nightly` label | |
| always() && github.event.schedule && needs.rules.outputs.nightly && | |
| contains(needs.*.result, 'failure') | |
| steps: | |
| - uses: JasonEtco/create-an-issue@v2 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| filename: .github/nightly-failure.yaml | |
| update_existing: true | |
| search_existing: open |