Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions macros/edr/tests/test_accepted_range_with_context.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
{% test accepted_range_with_context(
model,
column_name,
min_value=none,
max_value=none,
inclusive=true,
context_columns=none
) %}
{%- if min_value is none and max_value is none %}
{{
exceptions.raise_compiler_error(
"accepted_range_with_context: at least one of min_value or max_value must be provided."
)
}}
{%- endif %}

{%- if context_columns is not none and context_columns is iterable and context_columns is not string %}
{%- set existing_column_names = (
adapter.get_columns_in_relation(model)
| map(attribute="name")
| map("lower")
| list
) %}
{%- set select_cols = [column_name] %}
{%- for col in context_columns %}
{%- if col | lower == column_name | lower %}
{# already included, skip #}
{%- elif col | lower not in existing_column_names %}
{%- do log(
"WARNING [accepted_range_with_context]: column '"
~ col
~ "' does not exist in model '"
~ model.name
~ "' and will be skipped.",
info=true,
) %}
{%- else %} {%- do select_cols.append(col) %}
{%- endif %}
{%- endfor %}
{%- set select_clause = select_cols | join(", ") %}
{%- else %} {%- set select_clause = "*" %}
{%- endif %}

select {{ select_clause }}
from {{ model }}
where
1 = 2
{%- if min_value is not none %}
or not {{ column_name }} >{{- "=" if inclusive }} {{ min_value }}
{%- endif %}
{%- if max_value is not none %}
or not {{ column_name }} <{{- "=" if inclusive }} {{ max_value }}
{%- endif %}
{% endtest %}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{% test expect_column_values_to_be_unique_with_context(
model, column_name, row_condition=none, context_columns=none
) %}
{%- set all_columns = (
adapter.get_columns_in_relation(model) | map(attribute="name") | list
) %}
{%- set all_columns_lower = all_columns | map("lower") | list %}

{%- if context_columns is not none and context_columns is iterable and context_columns is not string %}
{%- set select_cols = [column_name] %}
{%- for col in context_columns %}
{%- if col | lower == column_name | lower %}
{# already included, skip #}
{%- elif col | lower not in all_columns_lower %}
{%- do log(
"WARNING [expect_column_values_to_be_unique_with_context]: column '"
~ col
~ "' does not exist in model '"
~ model.name
~ "' and will be skipped.",
info=true,
) %}
{%- else %} {%- do select_cols.append(col) %}
{%- endif %}
{%- endfor %}
{%- set select_clause = select_cols | join(", ") %}
{%- else %} {%- set select_clause = all_columns | join(", ") %}
{%- endif %}

select {{ select_clause }}
from
(
select *, count(*) over (partition by {{ column_name }}) as n_records
from {{ model }}
{%- if row_condition %} where {{ row_condition }} {%- endif %}
) validation
where n_records > 1
{% endtest %}
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
{% test expect_column_values_to_match_regex_with_context(
model,
column_name,
regex,
row_condition=none,
is_raw=false,
flags="",
context_columns=none
) %}
{%- if context_columns is not none and context_columns is iterable and context_columns is not string %}
{%- set existing_column_names = (
adapter.get_columns_in_relation(model)
| map(attribute="name")
| map("lower")
| list
) %}
{%- set select_cols = [column_name] %}
{%- for col in context_columns %}
{%- if col | lower == column_name | lower %}
{# already included, skip #}
{%- elif col | lower not in existing_column_names %}
{%- do log(
"WARNING [expect_column_values_to_match_regex_with_context]: column '"
~ col
~ "' does not exist in model '"
~ model.name
~ "' and will be skipped.",
info=true,
) %}
{%- else %} {%- do select_cols.append(col) %}
{%- endif %}
{%- endfor %}
{%- set select_clause = select_cols | join(", ") %}
{%- else %} {%- set select_clause = "*" %}
{%- endif %}

select {{ select_clause }}
from {{ model }}
where
{{
dbt_expectations.regexp_instr(
column_name, regex, is_raw=is_raw, flags=flags
)
}} = 0 {%- if row_condition %} and {{ row_condition }} {%- endif %}
{% endtest %}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
{% test expect_column_values_to_not_be_null_with_context(
model, column_name, row_condition=none, context_columns=none
) %}
{%- if context_columns is not none and context_columns is iterable and context_columns is not string %}
{%- set existing_column_names = (
adapter.get_columns_in_relation(model)
| map(attribute="name")
| map("lower")
| list
) %}
{%- set select_cols = [column_name] %}
{%- for col in context_columns %}
{%- if col | lower == column_name | lower %}
{# already included, skip #}
{%- elif col | lower not in existing_column_names %}
{%- do log(
"WARNING [expect_column_values_to_not_be_null_with_context]: column '"
~ col
~ "' does not exist in model '"
~ model.name
~ "' and will be skipped.",
info=true,
) %}
{%- else %} {%- do select_cols.append(col) %}
{%- endif %}
{%- endfor %}
{%- set select_clause = select_cols | join(", ") %}
{%- else %} {%- set select_clause = "*" %}
{%- endif %}

select {{ select_clause }}
from {{ model }}
where
{{ column_name }} is null
{%- if row_condition %} and {{ row_condition }} {%- endif %}
{% endtest %}
31 changes: 31 additions & 0 deletions macros/edr/tests/test_not_null_with_context.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
{% test not_null_with_context(model, column_name, context_columns=none) %}
{%- if context_columns is not none and context_columns is iterable and context_columns is not string %}
{%- set existing_column_names = (
adapter.get_columns_in_relation(model)
| map(attribute="name")
| map("lower")
| list
) %}

{%- set select_cols = [column_name] %}
{%- for col in context_columns %}
{%- if col | lower == column_name | lower %}
{# already included, skip #}
{%- elif col | lower not in existing_column_names %}
{%- do log(
"WARNING [not_null_with_context]: column '"
~ col
~ "' does not exist in model '"
~ model.name
~ "' and will be skipped.",
info=true,
) %}
{%- else %} {%- do select_cols.append(col) %}
{%- endif %}
{%- endfor %}
select {{ select_cols | join(", ") }}
from {{ model }}
where {{ column_name }} is null
{%- else %} select * from {{ model }} where {{ column_name }} is null
{%- endif %}
{% endtest %}
35 changes: 35 additions & 0 deletions macros/edr/tests/test_relationships_with_context.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{% test relationships_with_context(
model, column_name, to, field, context_columns=none
) %}
{%- if context_columns is not none and context_columns is iterable and context_columns is not string %}
{%- set existing_column_names = (
adapter.get_columns_in_relation(model)
| map(attribute="name")
| map("lower")
| list
) %}
{%- set select_cols = ["child." ~ column_name] %}
{%- for col in context_columns %}
{%- if col | lower == column_name | lower %}
{# already included, skip #}
{%- elif col | lower not in existing_column_names %}
{%- do log(
"WARNING [relationships_with_context]: column '"
~ col
~ "' does not exist in model '"
~ model.name
~ "' and will be skipped.",
info=true,
) %}
{%- else %} {%- do select_cols.append("child." ~ col) %}
{%- endif %}
{%- endfor %}
{%- set select_clause = select_cols | join(", ") %}
{%- else %} {%- set select_clause = "child.*" %}
{%- endif %}

select {{ select_clause }}
from {{ model }} as child
left join {{ to }} as parent on child.{{ column_name }} = parent.{{ field }}
where child.{{ column_name }} is not null and parent.{{ field }} is null
{% endtest %}
30 changes: 30 additions & 0 deletions macros/utils/common_test_configs.sql
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,36 @@
"collect_metrics": {
"description": "Collects metrics for the specified column or table. The test will always pass."
},
"not_null_with_context": {
"quality_dimension": "completeness",
"failed_row_count_calc": "count(*)",
"description": "Validates that there are no null values in a column, returning additional context columns alongside failing rows. If no context_columns are specified, all columns are returned.",
},
"accepted_range_with_context": {
"quality_dimension": "validity",
"failed_row_count_calc": "count(*)",
"description": "Validates that column values fall within an accepted range, returning additional context columns alongside failing rows. If no context_columns are specified, all columns are returned.",
},
"expect_column_values_to_not_be_null_with_context": {
"quality_dimension": "completeness",
"failed_row_count_calc": "count(*)",
"description": "Expects column values to not be null, returning additional context columns alongside failing rows. If no context_columns are specified, all columns are returned.",
},
"expect_column_values_to_be_unique_with_context": {
"quality_dimension": "uniqueness",
"failed_row_count_calc": "count(*)",
"description": "Expects column values to be unique, returning all duplicate rows with additional context columns. If no context_columns are specified, all columns are returned.",
},
"expect_column_values_to_match_regex_with_context": {
"quality_dimension": "validity",
"failed_row_count_calc": "count(*)",
"description": "Expects column values to match a given regular expression, returning additional context columns alongside failing rows. If no context_columns are specified, all columns are returned.",
},
"relationships_with_context": {
"quality_dimension": "consistency",
"failed_row_count_calc": "count(*)",
"description": "Validates referential integrity between a child and parent table, returning additional context columns alongside failing rows. If no context_columns are specified, all columns are returned.",
},
},
} %}
{% do return(common_tests_configs_mapping) %}
Expand Down
Loading