diff --git a/macros/edr/tests/test_accepted_range_with_context.sql b/macros/edr/tests/test_accepted_range_with_context.sql new file mode 100644 index 000000000..1dafb0ce6 --- /dev/null +++ b/macros/edr/tests/test_accepted_range_with_context.sql @@ -0,0 +1,54 @@ +{% test accepted_range_with_context( + model, + column_name, + min_value=none, + max_value=none, + inclusive=true, + context_columns=none +) %} + {%- if min_value is none and max_value is none %} + {{ + exceptions.raise_compiler_error( + "accepted_range_with_context: at least one of min_value or max_value must be provided." + ) + }} + {%- endif %} + + {%- if context_columns is not none and context_columns is iterable and context_columns is not string %} + {%- set existing_column_names = ( + adapter.get_columns_in_relation(model) + | map(attribute="name") + | map("lower") + | list + ) %} + {%- set select_cols = [column_name] %} + {%- for col in context_columns %} + {%- if col | lower == column_name | lower %} + {# already included, skip #} + {%- elif col | lower not in existing_column_names %} + {%- do log( + "WARNING [accepted_range_with_context]: column '" + ~ col + ~ "' does not exist in model '" + ~ model.name + ~ "' and will be skipped.", + info=true, + ) %} + {%- else %} {%- do select_cols.append(col) %} + {%- endif %} + {%- endfor %} + {%- set select_clause = select_cols | join(", ") %} + {%- else %} {%- set select_clause = "*" %} + {%- endif %} + + select {{ select_clause }} + from {{ model }} + where + 1 = 2 + {%- if min_value is not none %} + or not {{ column_name }} >{{- "=" if inclusive }} {{ min_value }} + {%- endif %} + {%- if max_value is not none %} + or not {{ column_name }} <{{- "=" if inclusive }} {{ max_value }} + {%- endif %} +{% endtest %} diff --git a/macros/edr/tests/test_expect_column_values_to_be_unique_with_context.sql b/macros/edr/tests/test_expect_column_values_to_be_unique_with_context.sql new file mode 100644 index 000000000..a6fdfd01d --- /dev/null +++ b/macros/edr/tests/test_expect_column_values_to_be_unique_with_context.sql @@ -0,0 +1,38 @@ +{% test expect_column_values_to_be_unique_with_context( + model, column_name, row_condition=none, context_columns=none +) %} + {%- set all_columns = ( + adapter.get_columns_in_relation(model) | map(attribute="name") | list + ) %} + {%- set all_columns_lower = all_columns | map("lower") | list %} + + {%- if context_columns is not none and context_columns is iterable and context_columns is not string %} + {%- set select_cols = [column_name] %} + {%- for col in context_columns %} + {%- if col | lower == column_name | lower %} + {# already included, skip #} + {%- elif col | lower not in all_columns_lower %} + {%- do log( + "WARNING [expect_column_values_to_be_unique_with_context]: column '" + ~ col + ~ "' does not exist in model '" + ~ model.name + ~ "' and will be skipped.", + info=true, + ) %} + {%- else %} {%- do select_cols.append(col) %} + {%- endif %} + {%- endfor %} + {%- set select_clause = select_cols | join(", ") %} + {%- else %} {%- set select_clause = all_columns | join(", ") %} + {%- endif %} + + select {{ select_clause }} + from + ( + select *, count(*) over (partition by {{ column_name }}) as n_records + from {{ model }} + {%- if row_condition %} where {{ row_condition }} {%- endif %} + ) validation + where n_records > 1 +{% endtest %} diff --git a/macros/edr/tests/test_expect_column_values_to_match_regex_with_context.sql b/macros/edr/tests/test_expect_column_values_to_match_regex_with_context.sql new file mode 100644 index 000000000..61b8a9fac --- /dev/null +++ b/macros/edr/tests/test_expect_column_values_to_match_regex_with_context.sql @@ -0,0 +1,45 @@ +{% test expect_column_values_to_match_regex_with_context( + model, + column_name, + regex, + row_condition=none, + is_raw=false, + flags="", + context_columns=none +) %} + {%- if context_columns is not none and context_columns is iterable and context_columns is not string %} + {%- set existing_column_names = ( + adapter.get_columns_in_relation(model) + | map(attribute="name") + | map("lower") + | list + ) %} + {%- set select_cols = [column_name] %} + {%- for col in context_columns %} + {%- if col | lower == column_name | lower %} + {# already included, skip #} + {%- elif col | lower not in existing_column_names %} + {%- do log( + "WARNING [expect_column_values_to_match_regex_with_context]: column '" + ~ col + ~ "' does not exist in model '" + ~ model.name + ~ "' and will be skipped.", + info=true, + ) %} + {%- else %} {%- do select_cols.append(col) %} + {%- endif %} + {%- endfor %} + {%- set select_clause = select_cols | join(", ") %} + {%- else %} {%- set select_clause = "*" %} + {%- endif %} + + select {{ select_clause }} + from {{ model }} + where + {{ + dbt_expectations.regexp_instr( + column_name, regex, is_raw=is_raw, flags=flags + ) + }} = 0 {%- if row_condition %} and {{ row_condition }} {%- endif %} +{% endtest %} diff --git a/macros/edr/tests/test_expect_column_values_to_not_be_null_with_context.sql b/macros/edr/tests/test_expect_column_values_to_not_be_null_with_context.sql new file mode 100644 index 000000000..df3fbac46 --- /dev/null +++ b/macros/edr/tests/test_expect_column_values_to_not_be_null_with_context.sql @@ -0,0 +1,36 @@ +{% test expect_column_values_to_not_be_null_with_context( + model, column_name, row_condition=none, context_columns=none +) %} + {%- if context_columns is not none and context_columns is iterable and context_columns is not string %} + {%- set existing_column_names = ( + adapter.get_columns_in_relation(model) + | map(attribute="name") + | map("lower") + | list + ) %} + {%- set select_cols = [column_name] %} + {%- for col in context_columns %} + {%- if col | lower == column_name | lower %} + {# already included, skip #} + {%- elif col | lower not in existing_column_names %} + {%- do log( + "WARNING [expect_column_values_to_not_be_null_with_context]: column '" + ~ col + ~ "' does not exist in model '" + ~ model.name + ~ "' and will be skipped.", + info=true, + ) %} + {%- else %} {%- do select_cols.append(col) %} + {%- endif %} + {%- endfor %} + {%- set select_clause = select_cols | join(", ") %} + {%- else %} {%- set select_clause = "*" %} + {%- endif %} + + select {{ select_clause }} + from {{ model }} + where + {{ column_name }} is null + {%- if row_condition %} and {{ row_condition }} {%- endif %} +{% endtest %} diff --git a/macros/edr/tests/test_not_null_with_context.sql b/macros/edr/tests/test_not_null_with_context.sql new file mode 100644 index 000000000..0073ea35b --- /dev/null +++ b/macros/edr/tests/test_not_null_with_context.sql @@ -0,0 +1,31 @@ +{% test not_null_with_context(model, column_name, context_columns=none) %} + {%- if context_columns is not none and context_columns is iterable and context_columns is not string %} + {%- set existing_column_names = ( + adapter.get_columns_in_relation(model) + | map(attribute="name") + | map("lower") + | list + ) %} + + {%- set select_cols = [column_name] %} + {%- for col in context_columns %} + {%- if col | lower == column_name | lower %} + {# already included, skip #} + {%- elif col | lower not in existing_column_names %} + {%- do log( + "WARNING [not_null_with_context]: column '" + ~ col + ~ "' does not exist in model '" + ~ model.name + ~ "' and will be skipped.", + info=true, + ) %} + {%- else %} {%- do select_cols.append(col) %} + {%- endif %} + {%- endfor %} + select {{ select_cols | join(", ") }} + from {{ model }} + where {{ column_name }} is null + {%- else %} select * from {{ model }} where {{ column_name }} is null + {%- endif %} +{% endtest %} diff --git a/macros/edr/tests/test_relationships_with_context.sql b/macros/edr/tests/test_relationships_with_context.sql new file mode 100644 index 000000000..3a4fe37e5 --- /dev/null +++ b/macros/edr/tests/test_relationships_with_context.sql @@ -0,0 +1,35 @@ +{% test relationships_with_context( + model, column_name, to, field, context_columns=none +) %} + {%- if context_columns is not none and context_columns is iterable and context_columns is not string %} + {%- set existing_column_names = ( + adapter.get_columns_in_relation(model) + | map(attribute="name") + | map("lower") + | list + ) %} + {%- set select_cols = ["child." ~ column_name] %} + {%- for col in context_columns %} + {%- if col | lower == column_name | lower %} + {# already included, skip #} + {%- elif col | lower not in existing_column_names %} + {%- do log( + "WARNING [relationships_with_context]: column '" + ~ col + ~ "' does not exist in model '" + ~ model.name + ~ "' and will be skipped.", + info=true, + ) %} + {%- else %} {%- do select_cols.append("child." ~ col) %} + {%- endif %} + {%- endfor %} + {%- set select_clause = select_cols | join(", ") %} + {%- else %} {%- set select_clause = "child.*" %} + {%- endif %} + + select {{ select_clause }} + from {{ model }} as child + left join {{ to }} as parent on child.{{ column_name }} = parent.{{ field }} + where child.{{ column_name }} is not null and parent.{{ field }} is null +{% endtest %} diff --git a/macros/utils/common_test_configs.sql b/macros/utils/common_test_configs.sql index 039aceaac..26edd2e15 100644 --- a/macros/utils/common_test_configs.sql +++ b/macros/utils/common_test_configs.sql @@ -450,6 +450,36 @@ "collect_metrics": { "description": "Collects metrics for the specified column or table. The test will always pass." }, + "not_null_with_context": { + "quality_dimension": "completeness", + "failed_row_count_calc": "count(*)", + "description": "Validates that there are no null values in a column, returning additional context columns alongside failing rows. If no context_columns are specified, all columns are returned.", + }, + "accepted_range_with_context": { + "quality_dimension": "validity", + "failed_row_count_calc": "count(*)", + "description": "Validates that column values fall within an accepted range, returning additional context columns alongside failing rows. If no context_columns are specified, all columns are returned.", + }, + "expect_column_values_to_not_be_null_with_context": { + "quality_dimension": "completeness", + "failed_row_count_calc": "count(*)", + "description": "Expects column values to not be null, returning additional context columns alongside failing rows. If no context_columns are specified, all columns are returned.", + }, + "expect_column_values_to_be_unique_with_context": { + "quality_dimension": "uniqueness", + "failed_row_count_calc": "count(*)", + "description": "Expects column values to be unique, returning all duplicate rows with additional context columns. If no context_columns are specified, all columns are returned.", + }, + "expect_column_values_to_match_regex_with_context": { + "quality_dimension": "validity", + "failed_row_count_calc": "count(*)", + "description": "Expects column values to match a given regular expression, returning additional context columns alongside failing rows. If no context_columns are specified, all columns are returned.", + }, + "relationships_with_context": { + "quality_dimension": "consistency", + "failed_row_count_calc": "count(*)", + "description": "Validates referential integrity between a child and parent table, returning additional context columns alongside failing rows. If no context_columns are specified, all columns are returned.", + }, }, } %} {% do return(common_tests_configs_mapping) %}