build-warehouse-image.yml

name: Test, visualize, and build dbt project

on:
  push:
    branches:
      - 'main'
    paths:
      - '.github/workflows/build-warehouse-image.yml'
      - 'warehouse/**'
  pull_request:
    paths:
      - '.github/workflows/build-warehouse-image.yml'
      - 'warehouse/**'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
  check:
    name: check python
    runs-on: ubuntu-latest
    steps:
      - run: sudo apt-get install -y libgraphviz-dev graphviz-dev
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - run: curl -sSL https://install.python-poetry.org | python -
      - run: cd warehouse && poetry install && poetry run mypy scripts

  compile:
    name: check dbt
    runs-on: ubuntu-latest
    steps:
      - run: sudo apt-get install -y libgraphviz-dev graphviz graphviz-dev
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - run: curl -sSL https://install.python-poetry.org | python -
      - uses: 'google-github-actions/auth@v2'
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'
      - uses: google-github-actions/setup-gcloud@v2
      - name: Compile dbt project
        working-directory: warehouse
        run: |
          poetry install
          poetry run dbt deps
          poetry run dbt compile --target prod --full-refresh
          poetry run dbt docs generate --target prod --no-compile
      - uses: 'google-github-actions/upload-cloud-storage@v1'
        with:
          path: './warehouse/logs/'
          destination: 'calitp-ci-artifacts/${{github.workflow}}/run_id=${{github.run_id}}/job=${{github.job}}/logs/'
      - uses: 'google-github-actions/upload-cloud-storage@v1'
        with:
          path: './warehouse/target/'
          glob: '*.json'
          destination: 'calitp-ci-artifacts/${{github.workflow}}/run_id=${{github.run_id}}/job=${{github.job}}/target/'
      # Only do visualization if we actually changed models and we are merging against main
      - uses: tj-actions/changed-files@v41
        if: ${{ github.event_name == 'pull_request' }}
        id: changed-files-warehouse
        with:
          files: 'warehouse/models/**/*.sql'
      # install a specific version of node before cml https://github.com/iterative/cml/issues/1377
      - uses: actions/setup-node@v1
        with:
          node-version: '16'
      - uses: iterative/setup-cml@v1
        if: steps.changed-files-warehouse.outputs.any_changed == 'true'
      - name: Create GitHub comment
        if: steps.changed-files-warehouse.outputs.any_changed == 'true'
        env:
          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          cd warehouse
          gsutil cp -r gs://calitp-dbt-artifacts/latest/ .
          poetry run python scripts/visualize.py ci-report
          cml comment update target/report.md

  build_push:
    name: package warehouse image
    runs-on: ubuntu-latest
    needs: [check]
    steps:
      - uses: actions/checkout@v3
      - uses: tj-actions/changed-files@v41
        with:
          files: |
            warehouse/Dockerfile
            warehouse/packages.yml
            warehouse/poetry.lock
            warehouse/pyproject.yml
            warehouse/requirements.txt
      - uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - uses: docker/build-push-action@v4
        if: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || steps.changed-files-specific.outputs.any_changed == 'true' }}
        with:
          context: warehouse
          push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
          tags: ghcr.io/${{github.repository}}/warehouse:latest