Skip to content

Commit

Permalink
Merge branch 'main' into dependabot/github_actions/actions/setup-pyth…
Browse files Browse the repository at this point in the history
…on-5
  • Loading branch information
VersusFacit committed Aug 6, 2024
2 parents 69c51a0 + 38fd49a commit d1a97a8
Show file tree
Hide file tree
Showing 14 changed files with 267 additions and 46 deletions.
7 changes: 7 additions & 0 deletions .changes/unreleased/Fixes-20240625-170324.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
kind: Fixes
body: 'Handle unit test fixtures where typing goes wrong from first value in column
being Null. '
time: 2024-06-25T17:03:24.73937-07:00
custom:
Author: versusfacit
Issue: "821"
6 changes: 6 additions & 0 deletions .changes/unreleased/Under the Hood-20240719-133151.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Under the Hood
body: Remove `freezegun` as a testing dependency; this package is no longer used
time: 2024-07-19T13:31:51.503575-04:00
custom:
Author: mikealfare
Issue: "1136"
12 changes: 12 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,25 @@ updates:
schedule:
interval: "daily"
rebase-strategy: "disabled"
ignore:
- dependency-name: "*"
update-types:
- version-update:semver-patch
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
rebase-strategy: "disabled"
ignore:
- dependency-name: "*"
update-types:
- version-update:semver-patch
- package-ecosystem: "docker"
directory: "/docker"
schedule:
interval: "weekly"
rebase-strategy: "disabled"
ignore:
- dependency-name: "*"
update-types:
- version-update:semver-patch
20 changes: 0 additions & 20 deletions .github/scripts/update_dbt_core_branch.sh

This file was deleted.

21 changes: 21 additions & 0 deletions .github/scripts/update_dev_dependency_branches.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/bin/bash -e
set -e


dbt_adapters_branch=$1
dbt_core_branch=$2
dbt_common_branch=$3
target_req_file="dev-requirements.txt"
core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g"
adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g"
common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g"
if [[ "$OSTYPE" == darwin* ]]; then
# mac ships with a different version of sed that requires a delimiter arg
sed -i "" "$adapters_req_sed_pattern" $target_req_file
sed -i "" "$core_req_sed_pattern" $target_req_file
sed -i "" "$common_req_sed_pattern" $target_req_file
else
sed -i "$adapters_req_sed_pattern" $target_req_file
sed -i "$core_req_sed_pattern" $target_req_file
sed -i "$common_req_sed_pattern" $target_req_file
fi
84 changes: 65 additions & 19 deletions .github/workflows/integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@

name: Adapter Integration Tests

run-name: "${{ (contains(github.event_name, 'workflow_') && inputs.name) || github.event_name }}: ${{ (contains(github.event_name, 'workflow_') && inputs.adapter_branch) || github.ref_name }} by @${{ github.actor }}"

on:
# pushes to release branches
push:
Expand All @@ -34,10 +36,31 @@ on:
# manual trigger
workflow_dispatch:
inputs:
dbt-core-branch:
description: "branch of dbt-core to use in dev-requirements.txt"
name:
description: "Name to associate with run (example: 'dbt-adapters-242')"
required: false
type: string
default: "Adapter Integration Tests"
adapter_branch:
description: "The branch of this adapter repository to use"
type: string
required: false
default: "main"
dbt_adapters_branch:
description: "The branch of dbt-adapters to use"
type: string
required: false
default: "main"
dbt_core_branch:
description: "The branch of dbt-core to use"
type: string
required: false
default: "main"
dbt_common_branch:
description: "The branch of dbt-common to use"
type: string
required: false
default: "main"

# explicitly turn off permissions for `GITHUB_TOKEN`
permissions: read-all
Expand Down Expand Up @@ -76,6 +99,8 @@ jobs:
python-version: "3.10"
- os: ubuntu-22.04
python-version: "3.11"
- os: ubuntu-22.04
python-version: "3.12"

env:
TOXENV: integration-redshift
Expand All @@ -88,13 +113,20 @@ jobs:
DD_SERVICE: ${{ github.event.repository.name }}

steps:
- name: Check out the repository
if: github.event_name != 'pull_request_target'
- name: Check out the repository (push)
if: github.event_name == 'push'
uses: actions/checkout@v4
with:
persist-credentials: false

# explicity checkout the branch for the PR,
- name: Check out the repository (workflow_dispatch)
if: github.event_name == 'workflow_dispatch'
uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ inputs.adapter_branch }}

# explicitly checkout the branch for the PR,
# this is necessary for the `pull_request_target` event
- name: Check out the repository (PR)
if: github.event_name == 'pull_request_target'
Expand All @@ -108,19 +140,22 @@ jobs:
with:
python-version: ${{ matrix.python-version }}

- name: Update Adapters and Core branches (update dev_requirements.txt)
if: github.event_name == 'workflow_dispatch'
run: |
./.github/scripts/update_dev_dependency_branches.sh \
${{ inputs.dbt_adapters_branch }} \
${{ inputs.dbt_core_branch }} \
${{ inputs.dbt_common_branch }}
cat dev-requirements.txt
- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip install tox
python -m pip --version
tox --version
- name: Update dev_requirements.txt
if: inputs.dbt-core-branch != ''
run: |
pip install bumpversion
./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }}
- name: Create AWS IAM profiles
run: |
aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID
Expand Down Expand Up @@ -201,7 +236,7 @@ jobs:
fail-fast: false
max-parallel: 1
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]

env:
TOXENV: integration-redshift
Expand All @@ -215,13 +250,20 @@ jobs:
DD_SERVICE: ${{ github.event.repository.name }}

steps:
- name: Check out the repository
if: github.event_name != 'pull_request_target'
- name: Check out the repository (push)
if: github.event_name == 'push'
uses: actions/checkout@v3
with:
persist-credentials: false

# explicity checkout the branch for the PR,
- name: Check out the repository (workflow_dispatch)
if: github.event_name == 'workflow_dispatch'
uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ inputs.adapter_branch }}

# explicitly checkout the branch for the PR,
# this is necessary for the `pull_request_target` event
- name: Check out the repository (PR)
if: github.event_name == 'pull_request_target'
Expand All @@ -242,11 +284,15 @@ jobs:
python -m pip --version
tox --version
- name: Update dev_requirements.txt
if: inputs.dbt-core-branch != ''
- name: Update Adapters and Core branches (update dev_requirements.txt)
if: github.event_name == 'workflow_dispatch'
run: |
pip install bumpversion
./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }}
./.github/scripts/update_dev_dependency_branches.sh \
${{ inputs.dbt_adapters_branch }} \
${{ inputs.dbt_core_branch }} \
${{ inputs.dbt_common_branch }}
cat dev-requirements.txt
- name: Run tox (redshift)
env:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ['3.8', '3.9', '3.10', '3.11']
python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']

env:
TOXENV: "unit"
Expand Down Expand Up @@ -174,7 +174,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-22.04, macos-12, windows-2022]
python-version: ['3.8', '3.9', '3.10', '3.11']
python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']

steps:
- name: Set up Python ${{ matrix.python-version }}
Expand Down
1 change: 1 addition & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ repos:
- --target-version=py39
- --target-version=py310
- --target-version=py311
- --target-version=py312

- repo: https://github.com/pycqa/flake8
rev: 7.0.0
Expand Down
11 changes: 11 additions & 0 deletions dbt/include/redshift/macros/adapters/unit_testing.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{%- macro redshift__validate_fixture_rows(rows, row_number) -%}
{%- if rows is not none and rows|length > 0 -%}
{%- set row = rows[0] -%}
{%- for key, value in row.items() -%}
{%- if value is none -%}
{%- set fixture_name = "expected output" if model.resource_type == 'unit_test' else ("'" ~ model.name ~ "'") -%}
{{ exceptions.raise_compiler_error("Unit test fixture " ~ fixture_name ~ " in " ~ model.name ~ " does not have any row free of null values, which may cause type mismatch errors during unit test execution.") }}
{%- endif -%}
{%- endfor -%}
{%- endif -%}
{%- endmacro -%}
3 changes: 1 addition & 2 deletions dev-requirements.txt
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# install latest changes in dbt-core + dbt-postgres
git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
git+https://github.com/dbt-labs/dbt-adapters.git
git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter
git+https://github.com/dbt-labs/dbt-common.git
git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core
git+https://github.com/dbt-labs/dbt-postgres.git

# dev
Expand All @@ -12,7 +12,6 @@ pre-commit~=3.5.0;python_version <"3.9"

# test
ddtrace==2.3.0
freezegun~=1.4
pytest~=7.4
pytest-csv~=3.0
pytest-dotenv~=0.5.2
Expand Down
1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ def _plugin_version() -> str:
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
],
python_requires=">=3.8",
)
73 changes: 73 additions & 0 deletions tests/functional/adapter/unit_testing/fixtures.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
model_none_value_base = """
{{ config(materialized="table") }}
select 1 as id, 'a' as col1
"""

model_none_value_model = """
{{config(materialized="table")}}
select * from {{ ref('none_value_base') }}
"""


test_none_column_value_doesnt_throw_error_csv = """
unit_tests:
- name: test_simple
model: none_value_model
given:
- input: ref('none_value_base')
format: csv
rows: |
id,col1
,d
,e
6,f
expect:
format: csv
rows: |
id,col1
,d
,e
6,f
"""

test_none_column_value_doesnt_throw_error_dct = """
unit_tests:
- name: test_simple
model: none_value_model
given:
- input: ref('none_value_base')
rows:
- { "id": , "col1": "d"}
- { "id": , "col1": "e"}
- { "id": 6, "col1": "f"}
expect:
rows:
- {id: , "col1": "d"}
- {id: , "col1": "e"}
- {id: 6, "col1": "f"}
"""

test_none_column_value_will_throw_error = """
unit_tests:
- name: test_simple
model: none_value_model
given:
- input: ref('none_value_base')
rows:
- { "id": , "col1": "d"}
- { "id": , "col1": "e"}
- { "id": 6, "col1": }
expect:
rows:
- {id: , "col1": "d"}
- {id: , "col1": "e"}
- {id: 6, "col1": }
"""
Loading

0 comments on commit d1a97a8

Please sign in to comment.