From 133936af9f96c50fd6b89346ce8df0d60de5d268 Mon Sep 17 00:00:00 2001 From: Trey Spiller Date: Wed, 21 Jan 2026 17:14:03 -0600 Subject: [PATCH 1/4] bump sqlglot to 28.6.0 --- examples/sushi/models/customers.sql | 2 +- pyproject.toml | 2 +- sqlmesh/core/config/connection.py | 6 +-- sqlmesh/core/config/scheduler.py | 2 +- sqlmesh/core/engine_adapter/base.py | 5 +- sqlmesh/core/linter/rules/builtin.py | 2 +- sqlmesh/core/loader.py | 2 +- sqlmesh/core/metric/rewriter.py | 3 +- sqlmesh/core/model/definition.py | 3 +- sqlmesh/core/test/definition.py | 4 +- tests/core/engine_adapter/test_bigquery.py | 4 +- tests/core/engine_adapter/test_clickhouse.py | 24 ++++----- tests/core/test_audit.py | 54 +++++++++---------- tests/core/test_context.py | 6 +-- tests/core/test_model.py | 14 ++--- tests/dbt/test_model.py | 4 +- tests/dbt/test_transformation.py | 2 +- .../lsp/test_reference_model_column_prefix.py | 11 ++-- tests/lsp/test_reference_model_find_all.py | 6 +-- tests/utils/test_cache.py | 4 +- 20 files changed, 79 insertions(+), 81 deletions(-) diff --git a/examples/sushi/models/customers.sql b/examples/sushi/models/customers.sql index f91f1166e8..d2bda09ed3 100644 --- a/examples/sushi/models/customers.sql +++ b/examples/sushi/models/customers.sql @@ -42,4 +42,4 @@ LEFT JOIN ( ON o.customer_id = m.customer_id LEFT JOIN raw.demographics AS d ON o.customer_id = d.customer_id -WHERE sushi.orders.customer_id > 0 \ No newline at end of file +WHERE o.customer_id > 0 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 1a674dea72..bf86114956 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "requests", "rich[jupyter]", "ruamel.yaml", - "sqlglot[rs]~=27.28.0", + "sqlglot[rs]~=28.6.0", "tenacity", "time-machine", "json-stream" diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py index 4e11fc626f..9e3a210e5e 100644 --- a/sqlmesh/core/config/connection.py +++ b/sqlmesh/core/config/connection.py @@ -2334,7 +2334,7 @@ def init(cursor: t.Any) -> None: for tpe in subclasses( __name__, ConnectionConfig, - exclude=(ConnectionConfig, BaseDuckDBConnectionConfig), + exclude={ConnectionConfig, BaseDuckDBConnectionConfig}, ) } @@ -2343,7 +2343,7 @@ def init(cursor: t.Any) -> None: for tpe in subclasses( __name__, ConnectionConfig, - exclude=(ConnectionConfig, BaseDuckDBConnectionConfig), + exclude={ConnectionConfig, BaseDuckDBConnectionConfig}, ) } @@ -2355,7 +2355,7 @@ def init(cursor: t.Any) -> None: for tpe in subclasses( __name__, ConnectionConfig, - exclude=(ConnectionConfig, BaseDuckDBConnectionConfig), + exclude={ConnectionConfig, BaseDuckDBConnectionConfig}, ) } diff --git a/sqlmesh/core/config/scheduler.py b/sqlmesh/core/config/scheduler.py index 69adcafe70..970defee62 100644 --- a/sqlmesh/core/config/scheduler.py +++ b/sqlmesh/core/config/scheduler.py @@ -146,7 +146,7 @@ def get_default_catalog_per_gateway(self, context: GenericContext) -> t.Dict[str SCHEDULER_CONFIG_TO_TYPE = { tpe.all_field_infos()["type_"].default: tpe - for tpe in subclasses(__name__, BaseConfig, exclude=(BaseConfig,)) + for tpe in subclasses(__name__, BaseConfig, exclude={BaseConfig}) } diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index a7a8e2f707..50e22c4609 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -2861,7 +2861,7 @@ def _order_projections_and_filter( return query query = t.cast(exp.Query, query.copy()) - with_ = query.args.pop("with", None) + with_ = query.args.pop("with_", None) or query.args.pop("with", None) select_exprs: t.List[exp.Expression] = [ exp.column(c, quoted=True) for c in target_columns_to_types @@ -2877,7 +2877,8 @@ def _order_projections_and_filter( query = query.where(where, copy=False) if with_: - query.set("with", with_) + with_key = "with_" if "with_" in query.arg_types else "with" + query.set(with_key, with_) return query diff --git a/sqlmesh/core/linter/rules/builtin.py b/sqlmesh/core/linter/rules/builtin.py index c28822a154..4547ac0528 100644 --- a/sqlmesh/core/linter/rules/builtin.py +++ b/sqlmesh/core/linter/rules/builtin.py @@ -318,4 +318,4 @@ def check_model(self, model: Model) -> t.Optional[RuleViolation]: return None -BUILTIN_RULES = RuleSet(subclasses(__name__, Rule, (Rule,))) +BUILTIN_RULES = RuleSet(subclasses(__name__, Rule, exclude={Rule})) diff --git a/sqlmesh/core/loader.py b/sqlmesh/core/loader.py index a43f5f28ff..4b7b1bac02 100644 --- a/sqlmesh/core/loader.py +++ b/sqlmesh/core/loader.py @@ -840,7 +840,7 @@ def _load_linting_rules(self) -> RuleSet: if os.path.getsize(path): self._track_file(path) module = import_python_file(path, self.config_path) - module_rules = subclasses(module.__name__, Rule, (Rule,)) + module_rules = subclasses(module.__name__, Rule, exclude={Rule}) for user_rule in module_rules: user_rules[user_rule.name] = user_rule diff --git a/sqlmesh/core/metric/rewriter.py b/sqlmesh/core/metric/rewriter.py index 3519a77e68..af68df0cc9 100644 --- a/sqlmesh/core/metric/rewriter.py +++ b/sqlmesh/core/metric/rewriter.py @@ -57,7 +57,8 @@ def _build_sources(self, projections: t.List[exp.Expression]) -> SourceAggsAndJo return sources def _expand(self, select: exp.Select) -> None: - base = select.args["from"].this.find(exp.Table) + from_clause = t.cast(exp.From, select.args.get("from") or select.args.get("from_")) + base = from_clause.this.find(exp.Table) base_alias = base.alias_or_name base_name = exp.table_name(base) diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index 9154b4ec2f..e4068027ac 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -753,7 +753,8 @@ def ctas_query(self, **render_kwarg: t.Any) -> exp.Query: query = self.render_query_or_raise(**render_kwarg).limit(0) for select_or_set_op in query.find_all(exp.Select, exp.SetOperation): - if isinstance(select_or_set_op, exp.Select) and select_or_set_op.args.get("from"): + from_clause = select_or_set_op.args.get("from") or select_or_set_op.args.get("from_") + if isinstance(select_or_set_op, exp.Select) and from_clause: select_or_set_op.where(exp.false(), copy=False) if self.managed_columns: diff --git a/sqlmesh/core/test/definition.py b/sqlmesh/core/test/definition.py index 8694ec6024..2fe05c9d69 100644 --- a/sqlmesh/core/test/definition.py +++ b/sqlmesh/core/test/definition.py @@ -711,7 +711,7 @@ def runTest(self) -> None: query = self._render_model_query() sql = query.sql(self._test_adapter_dialect, pretty=self.engine_adapter._pretty_sql) - with_clause = query.args.get("with") + with_clause = query.args.get("with_") or query.args.get("with") if with_clause: self.test_ctes( @@ -905,7 +905,7 @@ def generate_test( if isinstance(model, SqlModel): assert isinstance(test, SqlModelTest) model_query = test._render_model_query() - with_clause = model_query.args.get("with") + with_clause = model_query.args.get("with_") or model_query.args.get("with") if with_clause and include_ctes: ctes = {} diff --git a/tests/core/engine_adapter/test_bigquery.py b/tests/core/engine_adapter/test_bigquery.py index 047613e47a..9a6bc7d851 100644 --- a/tests/core/engine_adapter/test_bigquery.py +++ b/tests/core/engine_adapter/test_bigquery.py @@ -1245,7 +1245,7 @@ def test_sync_grants_config(make_mocked_engine_adapter: t.Callable, mocker: Mock executed_sql = executed_query.sql(dialect="bigquery") expected_sql = ( "SELECT privilege_type, grantee FROM `project`.`region-us-central1`.`INFORMATION_SCHEMA.OBJECT_PRIVILEGES` AS OBJECT_PRIVILEGES " - "WHERE object_schema = 'dataset' AND object_name = 'test_table' AND SPLIT(grantee, ':')[OFFSET(1)] <> session_user()" + "WHERE object_schema = 'dataset' AND object_name = 'test_table' AND SPLIT(grantee, ':')[OFFSET(1)] <> SESSION_USER()" ) assert executed_sql == expected_sql @@ -1306,7 +1306,7 @@ def test_sync_grants_config_with_overlaps( executed_sql = executed_query.sql(dialect="bigquery") expected_sql = ( "SELECT privilege_type, grantee FROM `project`.`region-us-central1`.`INFORMATION_SCHEMA.OBJECT_PRIVILEGES` AS OBJECT_PRIVILEGES " - "WHERE object_schema = 'dataset' AND object_name = 'test_table' AND SPLIT(grantee, ':')[OFFSET(1)] <> session_user()" + "WHERE object_schema = 'dataset' AND object_name = 'test_table' AND SPLIT(grantee, ':')[OFFSET(1)] <> SESSION_USER()" ) assert executed_sql == expected_sql diff --git a/tests/core/engine_adapter/test_clickhouse.py b/tests/core/engine_adapter/test_clickhouse.py index 188ae7f394..54fbe7c323 100644 --- a/tests/core/engine_adapter/test_clickhouse.py +++ b/tests/core/engine_adapter/test_clickhouse.py @@ -327,16 +327,16 @@ def build_properties_sql(storage_format="", order_by="", primary_key="", propert assert ( build_properties_sql( - order_by="ORDER_BY = 'timestamp with fill to toStartOfDay(toDateTime64(\\'2024-07-11\\', 3)) step toIntervalDay(1) interpolate(price as price)'," + order_by="ORDER_BY = 'timestamp with fill to dateTrunc(\\'DAY\\', toDateTime64(\\'2024-07-11\\', 3)) step toIntervalDay(1) interpolate(price as price)'," ) - == "ENGINE=MergeTree ORDER BY (timestamp WITH FILL TO toStartOfDay(toDateTime64('2024-07-11', 3)) STEP toIntervalDay(1) INTERPOLATE (price AS price))" + == "ENGINE=MergeTree ORDER BY (timestamp WITH FILL TO dateTrunc('DAY', toDateTime64('2024-07-11', 3)) STEP toIntervalDay(1) INTERPOLATE (price AS price))" ) assert ( build_properties_sql( - order_by="ORDER_BY = (\"a\", 'timestamp with fill to toStartOfDay(toDateTime64(\\'2024-07-11\\', 3)) step toIntervalDay(1) interpolate(price as price)')," + order_by="ORDER_BY = (\"a\", 'timestamp with fill to dateTrunc(\\'DAY\\', toDateTime64(\\'2024-07-11\\', 3)) step toIntervalDay(1) interpolate(price as price)')," ) - == "ENGINE=MergeTree ORDER BY (\"a\", timestamp WITH FILL TO toStartOfDay(toDateTime64('2024-07-11', 3)) STEP toIntervalDay(1) INTERPOLATE (price AS price))" + == "ENGINE=MergeTree ORDER BY (\"a\", timestamp WITH FILL TO dateTrunc('DAY', toDateTime64('2024-07-11', 3)) STEP toIntervalDay(1) INTERPOLATE (price AS price))" ) assert ( @@ -368,7 +368,7 @@ def test_partitioned_by_expr(make_mocked_engine_adapter: t.Callable): assert ( model.partitioned_by[0].sql("clickhouse") - == """toMonday(CAST("ds" AS DateTime64(9, 'UTC')))""" + == """dateTrunc('WEEK', CAST("ds" AS DateTime64(9, 'UTC')))""" ) # user specifies without time column, unknown time column type @@ -393,7 +393,7 @@ def test_partitioned_by_expr(make_mocked_engine_adapter: t.Callable): ) assert [p.sql("clickhouse") for p in model.partitioned_by] == [ - """toMonday(CAST("ds" AS DateTime64(9, 'UTC')))""", + """dateTrunc('WEEK', CAST("ds" AS DateTime64(9, 'UTC')))""", '"x"', ] @@ -417,7 +417,7 @@ def test_partitioned_by_expr(make_mocked_engine_adapter: t.Callable): ) ) - assert model.partitioned_by[0].sql("clickhouse") == 'toMonday("ds")' + assert model.partitioned_by[0].sql("clickhouse") == """dateTrunc('WEEK', "ds")""" # user doesn't specify, non-conformable time column type model = load_sql_based_model( @@ -441,7 +441,7 @@ def test_partitioned_by_expr(make_mocked_engine_adapter: t.Callable): assert ( model.partitioned_by[0].sql("clickhouse") - == """CAST(toMonday(CAST("ds" AS DateTime64(9, 'UTC'))) AS String)""" + == """CAST(dateTrunc('WEEK', CAST("ds" AS DateTime64(9, 'UTC'))) AS String)""" ) # user specifies partitioned_by with time column @@ -993,7 +993,7 @@ def test_insert_overwrite_by_condition_replace_partitioned( temp_table_mock.return_value = make_temp_table_name(table_name, "abcd") fetchone_mock = mocker.patch("sqlmesh.core.engine_adapter.ClickhouseEngineAdapter.fetchone") - fetchone_mock.return_value = "toMonday(ds)" + fetchone_mock.return_value = "dateTrunc('WEEK', ds)" insert_table_name = make_temp_table_name("new_records", "abcd") existing_table_name = make_temp_table_name("existing_records", "abcd") @@ -1069,7 +1069,7 @@ def test_insert_overwrite_by_condition_where_partitioned( temp_table_mock.return_value = make_temp_table_name(table_name, "abcd") fetchone_mock = mocker.patch("sqlmesh.core.engine_adapter.ClickhouseEngineAdapter.fetchone") - fetchone_mock.return_value = "toMonday(ds)" + fetchone_mock.return_value = "dateTrunc('WEEK', ds)" fetchall_mock = mocker.patch("sqlmesh.core.engine_adapter.ClickhouseEngineAdapter.fetchall") fetchall_mock.side_effect = [ @@ -1175,7 +1175,7 @@ def test_insert_overwrite_by_condition_by_key_partitioned( temp_table_mock.return_value = make_temp_table_name(table_name, "abcd") fetchone_mock = mocker.patch("sqlmesh.core.engine_adapter.ClickhouseEngineAdapter.fetchone") - fetchone_mock.side_effect = ["toMonday(ds)", "toMonday(ds)"] + fetchone_mock.side_effect = ["dateTrunc('WEEK', ds)", "dateTrunc('WEEK', ds)"] fetchall_mock = mocker.patch("sqlmesh.core.engine_adapter.ClickhouseEngineAdapter.fetchall") fetchall_mock.side_effect = [ @@ -1240,7 +1240,7 @@ def test_insert_overwrite_by_condition_inc_by_partition( temp_table_mock.return_value = make_temp_table_name(table_name, "abcd") fetchone_mock = mocker.patch("sqlmesh.core.engine_adapter.ClickhouseEngineAdapter.fetchone") - fetchone_mock.return_value = "toMonday(ds)" + fetchone_mock.return_value = "dateTrunc('WEEK', ds)" fetchall_mock = mocker.patch("sqlmesh.core.engine_adapter.ClickhouseEngineAdapter.fetchall") fetchall_mock.return_value = [("1",), ("2",), ("4",)] diff --git a/tests/core/test_audit.py b/tests/core/test_audit.py index 2ffcbbc4b2..66897ed088 100644 --- a/tests/core/test_audit.py +++ b/tests/core/test_audit.py @@ -397,7 +397,7 @@ def test_no_query(): def test_macro(model: Model): - expected_query = """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE "a" IS NULL""" + expected_query = """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE "a" IS NULL""" audit = ModelAudit( name="test_audit", @@ -456,7 +456,7 @@ def test_not_null_audit(model: Model): ) assert ( rendered_query_a.sql() - == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE "a" IS NULL AND TRUE""" + == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE "a" IS NULL AND TRUE""" ) rendered_query_a_and_b = model.render_audit_query( @@ -465,7 +465,7 @@ def test_not_null_audit(model: Model): ) assert ( rendered_query_a_and_b.sql() - == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE ("a" IS NULL OR "b" IS NULL) AND TRUE""" + == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE ("a" IS NULL OR "b" IS NULL) AND TRUE""" ) @@ -476,7 +476,7 @@ def test_not_null_audit_default_catalog(model_default_catalog: Model): ) assert ( rendered_query_a.sql() - == """SELECT * FROM (SELECT * FROM "test_catalog"."db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE "a" IS NULL AND TRUE""" + == """SELECT * FROM (SELECT * FROM "test_catalog"."db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE "a" IS NULL AND TRUE""" ) rendered_query_a_and_b = model_default_catalog.render_audit_query( @@ -485,7 +485,7 @@ def test_not_null_audit_default_catalog(model_default_catalog: Model): ) assert ( rendered_query_a_and_b.sql() - == """SELECT * FROM (SELECT * FROM "test_catalog"."db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE ("a" IS NULL OR "b" IS NULL) AND TRUE""" + == """SELECT * FROM (SELECT * FROM "test_catalog"."db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE ("a" IS NULL OR "b" IS NULL) AND TRUE""" ) @@ -495,7 +495,7 @@ def test_unique_values_audit(model: Model): ) assert ( rendered_query_a.sql() - == 'SELECT * FROM (SELECT ROW_NUMBER() OVER (PARTITION BY "a" ORDER BY "a") AS "rank_a" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_q_0" WHERE "b" IS NULL) AS "_q_1" WHERE "rank_a" > 1' + == 'SELECT * FROM (SELECT ROW_NUMBER() OVER (PARTITION BY "a" ORDER BY "a") AS "rank_a" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_0" WHERE "b" IS NULL) AS "_1" WHERE "rank_a" > 1' ) rendered_query_a_and_b = model.render_audit_query( @@ -503,7 +503,7 @@ def test_unique_values_audit(model: Model): ) assert ( rendered_query_a_and_b.sql() - == 'SELECT * FROM (SELECT ROW_NUMBER() OVER (PARTITION BY "a" ORDER BY "a") AS "rank_a", ROW_NUMBER() OVER (PARTITION BY "b" ORDER BY "b") AS "rank_b" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_q_0" WHERE TRUE) AS "_q_1" WHERE "rank_a" > 1 OR "rank_b" > 1' + == 'SELECT * FROM (SELECT ROW_NUMBER() OVER (PARTITION BY "a" ORDER BY "a") AS "rank_a", ROW_NUMBER() OVER (PARTITION BY "b" ORDER BY "b") AS "rank_b" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_0" WHERE TRUE) AS "_1" WHERE "rank_a" > 1 OR "rank_b" > 1' ) @@ -515,7 +515,7 @@ def test_accepted_values_audit(model: Model): ) assert ( rendered_query.sql() - == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE NOT "a" IN ('value_a', 'value_b') AND TRUE""" + == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE NOT "a" IN ('value_a', 'value_b') AND TRUE""" ) @@ -526,7 +526,7 @@ def test_number_of_rows_audit(model: Model): ) assert ( rendered_query.sql() - == """SELECT COUNT(*) FROM (SELECT 1 FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE TRUE LIMIT 0 + 1) AS "_q_1" HAVING COUNT(*) <= 0""" + == """SELECT COUNT(*) FROM (SELECT 1 FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE TRUE LIMIT 0 + 1) AS "_1" HAVING COUNT(*) <= 0""" ) @@ -537,7 +537,7 @@ def test_forall_audit(model: Model): ) assert ( rendered_query_a.sql() - == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE NOT ("a" >= "b") AND TRUE""" + == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE NOT ("a" >= "b") AND TRUE""" ) rendered_query_a = model.render_audit_query( @@ -546,7 +546,7 @@ def test_forall_audit(model: Model): ) assert ( rendered_query_a.sql() - == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE (NOT ("a" >= "b") OR NOT ("c" + "d" - "e" < 1.0)) AND TRUE""" + == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE (NOT ("a" >= "b") OR NOT ("c" + "d" - "e" < 1.0)) AND TRUE""" ) rendered_query_a = model.render_audit_query( @@ -556,7 +556,7 @@ def test_forall_audit(model: Model): ) assert ( rendered_query_a.sql() - == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE (NOT ("a" >= "b") OR NOT ("c" + "d" - "e" < 1.0)) AND "f" = 42""" + == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE (NOT ("a" >= "b") OR NOT ("c" + "d" - "e" < 1.0)) AND "f" = 42""" ) @@ -566,21 +566,21 @@ def test_accepted_range_audit(model: Model): ) assert ( rendered_query.sql() - == 'SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_q_0" WHERE "a" < 0 AND TRUE' + == 'SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_0" WHERE "a" < 0 AND TRUE' ) rendered_query = model.render_audit_query( builtin.accepted_range_audit, column=exp.to_column("a"), max_v=100, inclusive=exp.false() ) assert ( rendered_query.sql() - == 'SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_q_0" WHERE "a" >= 100 AND TRUE' + == 'SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_0" WHERE "a" >= 100 AND TRUE' ) rendered_query = model.render_audit_query( builtin.accepted_range_audit, column=exp.to_column("a"), min_v=100, max_v=100 ) assert ( rendered_query.sql() - == 'SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_q_0" WHERE ("a" < 100 OR "a" > 100) AND TRUE' + == 'SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_0" WHERE ("a" < 100 OR "a" > 100) AND TRUE' ) @@ -591,7 +591,7 @@ def test_at_least_one_audit(model: Model): ) assert ( rendered_query.sql() - == 'SELECT 1 AS "1" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_q_0" WHERE TRUE GROUP BY 1 HAVING COUNT("a") = 0' + == 'SELECT 1 AS "1" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_0" WHERE TRUE GROUP BY 1 HAVING COUNT("a") = 0' ) @@ -603,7 +603,7 @@ def test_mutually_exclusive_ranges_audit(model: Model): ) assert ( rendered_query.sql() - == '''WITH "window_functions" AS (SELECT "a" AS "lower_bound", "a" AS "upper_bound", LEAD("a") OVER (ORDER BY "a", "a") AS "next_lower_bound", ROW_NUMBER() OVER (ORDER BY "a" DESC, "a" DESC) = 1 AS "is_last_record" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE TRUE), "calc" AS (SELECT *, COALESCE("lower_bound" <= "upper_bound", FALSE) AS "lower_bound_lte_upper_bound", COALESCE("upper_bound" <= "next_lower_bound", "is_last_record", FALSE) AS "upper_bound_lte_next_lower_bound" FROM "window_functions" AS "window_functions"), "validation_errors" AS (SELECT * FROM "calc" AS "calc" WHERE NOT ("lower_bound_lte_upper_bound" AND "upper_bound_lte_next_lower_bound")) SELECT * FROM "validation_errors" AS "validation_errors"''' + == '''WITH "window_functions" AS (SELECT "a" AS "lower_bound", "a" AS "upper_bound", LEAD("a") OVER (ORDER BY "a", "a") AS "next_lower_bound", ROW_NUMBER() OVER (ORDER BY "a" DESC, "a" DESC) = 1 AS "is_last_record" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE TRUE), "calc" AS (SELECT *, COALESCE("lower_bound" <= "upper_bound", FALSE) AS "lower_bound_lte_upper_bound", COALESCE("upper_bound" <= "next_lower_bound", "is_last_record", FALSE) AS "upper_bound_lte_next_lower_bound" FROM "window_functions" AS "window_functions"), "validation_errors" AS (SELECT * FROM "calc" AS "calc" WHERE NOT ("lower_bound_lte_upper_bound" AND "upper_bound_lte_next_lower_bound")) SELECT * FROM "validation_errors" AS "validation_errors"''' ) @@ -614,7 +614,7 @@ def test_sequential_values_audit(model: Model): ) assert ( rendered_query.sql() - == '''WITH "windowed" AS (SELECT "a", LAG("a") OVER (ORDER BY "a") AS "prv" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE TRUE), "validation_errors" AS (SELECT * FROM "windowed" AS "windowed" WHERE NOT ("a" = "prv" + 1)) SELECT * FROM "validation_errors" AS "validation_errors"''' + == '''WITH "windowed" AS (SELECT "a", LAG("a") OVER (ORDER BY "a") AS "prv" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE TRUE), "validation_errors" AS (SELECT * FROM "windowed" AS "windowed" WHERE NOT ("a" = "prv" + 1)) SELECT * FROM "validation_errors" AS "validation_errors"''' ) @@ -627,7 +627,7 @@ def test_chi_square_audit(model: Model): ) assert ( rendered_query.sql() - == """WITH "samples" AS (SELECT "a" AS "x_a", "b" AS "x_b" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE (NOT "a" IS NULL AND NOT "b" IS NULL) AND TRUE), "contingency_table" AS (SELECT "x_a", "x_b", COUNT(*) AS "observed", (SELECT COUNT(*) FROM "samples" AS "t" WHERE "r"."x_a" = "t"."x_a") AS "tot_a", (SELECT COUNT(*) FROM "samples" AS "t" WHERE "r"."x_b" = "t"."x_b") AS "tot_b", (SELECT COUNT(*) FROM "samples" AS "samples") AS "g_t" /* g_t is the grand total */ FROM "samples" AS "r" GROUP BY "x_a", "x_b") SELECT ((SELECT COUNT(DISTINCT "x_a") FROM "contingency_table" AS "contingency_table") - 1) * ((SELECT COUNT(DISTINCT "x_b") FROM "contingency_table" AS "contingency_table") - 1) AS "degrees_of_freedom", SUM(("observed" - ("tot_a" * "tot_b" / "g_t")) * ("observed" - ("tot_a" * "tot_b" / "g_t")) / ("tot_a" * "tot_b" / "g_t")) AS "chi_square" FROM "contingency_table" AS "contingency_table" /* H0: the two variables are independent */ /* H1: the two variables are dependent */ /* if chi_square > critical_value, reject H0 */ /* if chi_square <= critical_value, fail to reject H0 */ HAVING NOT "chi_square" > 9.48773""" + == """WITH "samples" AS (SELECT "a" AS "x_a", "b" AS "x_b" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE (NOT "a" IS NULL AND NOT "b" IS NULL) AND TRUE), "contingency_table" AS (SELECT "x_a", "x_b", COUNT(*) AS "observed", (SELECT COUNT(*) FROM "samples" AS "t" WHERE "r"."x_a" = "t"."x_a") AS "tot_a", (SELECT COUNT(*) FROM "samples" AS "t" WHERE "r"."x_b" = "t"."x_b") AS "tot_b", (SELECT COUNT(*) FROM "samples" AS "samples") AS "g_t" /* g_t is the grand total */ FROM "samples" AS "r" GROUP BY "x_a", "x_b") SELECT ((SELECT COUNT(DISTINCT "x_a") FROM "contingency_table" AS "contingency_table") - 1) * ((SELECT COUNT(DISTINCT "x_b") FROM "contingency_table" AS "contingency_table") - 1) AS "degrees_of_freedom", SUM(("observed" - ("tot_a" * "tot_b" / "g_t")) * ("observed" - ("tot_a" * "tot_b" / "g_t")) / ("tot_a" * "tot_b" / "g_t")) AS "chi_square" FROM "contingency_table" AS "contingency_table" /* H0: the two variables are independent */ /* H1: the two variables are dependent */ /* if chi_square > critical_value, reject H0 */ /* if chi_square <= critical_value, fail to reject H0 */ HAVING NOT "chi_square" > 9.48773""" ) @@ -639,7 +639,7 @@ def test_pattern_audits(model: Model): ) assert ( rendered_query.sql() - == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_q_0" WHERE (NOT REGEXP_LIKE("a", \'^\\d.*\') AND NOT REGEXP_LIKE("a", \'.*!$\')) AND TRUE""" + == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_0" WHERE (NOT REGEXP_LIKE("a", \'^\\d.*\') AND NOT REGEXP_LIKE("a", \'.*!$\')) AND TRUE""" ) rendered_query = model.render_audit_query( @@ -649,7 +649,7 @@ def test_pattern_audits(model: Model): ) assert ( rendered_query.sql() - == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_q_0" WHERE (REGEXP_LIKE("a", \'^\\d.*\') OR REGEXP_LIKE("a", \'.*!$\')) AND TRUE""" + == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_0" WHERE (REGEXP_LIKE("a", \'^\\d.*\') OR REGEXP_LIKE("a", \'.*!$\')) AND TRUE""" ) rendered_query = model.render_audit_query( @@ -659,7 +659,7 @@ def test_pattern_audits(model: Model): ) assert ( rendered_query.sql() - == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_q_0" WHERE (NOT "a" LIKE \'jim%\' AND NOT "a" LIKE \'pam%\') AND TRUE""" + == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_0" WHERE (NOT "a" LIKE \'jim%\' AND NOT "a" LIKE \'pam%\') AND TRUE""" ) rendered_query = model.render_audit_query( @@ -669,7 +669,7 @@ def test_pattern_audits(model: Model): ) assert ( rendered_query.sql() - == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_q_0" WHERE ("a" LIKE \'jim%\' OR "a" LIKE \'pam%\') AND TRUE""" + == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN \'1970-01-01\' AND \'1970-01-01\') AS "_0" WHERE ("a" LIKE \'jim%\' OR "a" LIKE \'pam%\') AND TRUE""" ) @@ -814,7 +814,7 @@ def test_string_length_between_audit(model: Model): ) assert ( rendered_query.sql() - == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE (LENGTH("x") < 1 OR LENGTH("x") > 5) AND TRUE""" + == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE (LENGTH("x") < 1 OR LENGTH("x") > 5) AND TRUE""" ) @@ -824,7 +824,7 @@ def test_not_constant_audit(model: Model): ) assert ( rendered_query.sql() - == """SELECT 1 AS "1" FROM (SELECT COUNT(DISTINCT "x") AS "t_cardinality" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE "x" > 1) AS "r" WHERE "r"."t_cardinality" <= 1""" + == """SELECT 1 AS "1" FROM (SELECT COUNT(DISTINCT "x") AS "t_cardinality" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE "x" > 1) AS "r" WHERE "r"."t_cardinality" <= 1""" ) @@ -836,7 +836,7 @@ def test_condition_with_macro_var(model: Model): ) assert ( rendered_query.sql(dialect="duckdb") - == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_q_0" WHERE "x" IS NULL AND "dt" BETWEEN CAST('1970-01-01 00:00:00+00:00' AS TIMESTAMPTZ) AND CAST('1970-01-01 23:59:59.999999+00:00' AS TIMESTAMPTZ)""" + == """SELECT * FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE "x" IS NULL AND "dt" BETWEEN CAST('1970-01-01 00:00:00+00:00' AS TIMESTAMPTZ) AND CAST('1970-01-01 23:59:59.999999+00:00' AS TIMESTAMPTZ)""" ) @@ -907,7 +907,7 @@ def test_load_inline_audits(assert_exp_eq): def test_model_inline_audits(sushi_context: Context): model_name = "sushi.waiter_names" - expected_query = 'SELECT * FROM (SELECT * FROM "memory"."sushi"."waiter_names" AS "waiter_names") AS "_q_0" WHERE "id" < 0' + expected_query = 'SELECT * FROM (SELECT * FROM "memory"."sushi"."waiter_names" AS "waiter_names") AS "_0" WHERE "id" < 0' model = sushi_context.get_snapshot(model_name, raise_if_missing=True).node assert isinstance(model, SeedModel) diff --git a/tests/core/test_context.py b/tests/core/test_context.py index 54b8cd891a..959bedfc00 100644 --- a/tests/core/test_context.py +++ b/tests/core/test_context.py @@ -1985,7 +1985,7 @@ def access_adapter(evaluator): assert ( model.pre_statements[0].sql() - == "@IF(@runtime_stage IN ('evaluating', 'creating'), SET VARIABLE stats_model_start = NOW())" + == "@IF(@runtime_stage IN ('evaluating', 'creating'), SET stats_model_start = NOW())" ) assert ( model.post_statements[0].sql() @@ -2337,13 +2337,13 @@ def test_plan_audit_intervals(tmp_path: pathlib.Path, caplog): # Case 1: The timestamp audit should be in the inclusive range ['2025-02-01 00:00:00', '2025-02-01 23:59:59.999999'] assert ( - f"""SELECT COUNT(*) FROM (SELECT "timestamp_id" AS "timestamp_id" FROM (SELECT * FROM "sqlmesh__sqlmesh_audit"."sqlmesh_audit__timestamp_example__{timestamp_snapshot.version}" AS "sqlmesh_audit__timestamp_example__{timestamp_snapshot.version}" WHERE "timestamp_id" BETWEEN CAST('2025-02-01 00:00:00' AS TIMESTAMP) AND CAST('2025-02-01 23:59:59.999999' AS TIMESTAMP)) AS "_q_0" WHERE TRUE GROUP BY "timestamp_id" HAVING COUNT(*) > 1) AS "audit\"""" + f"""SELECT COUNT(*) FROM (SELECT "timestamp_id" AS "timestamp_id" FROM (SELECT * FROM "sqlmesh__sqlmesh_audit"."sqlmesh_audit__timestamp_example__{timestamp_snapshot.version}" AS "sqlmesh_audit__timestamp_example__{timestamp_snapshot.version}" WHERE "timestamp_id" BETWEEN CAST('2025-02-01 00:00:00' AS TIMESTAMP) AND CAST('2025-02-01 23:59:59.999999' AS TIMESTAMP)) AS "_0" WHERE TRUE GROUP BY "timestamp_id" HAVING COUNT(*) > 1) AS "audit\"""" in caplog.text ) # Case 2: The date audit should be in the inclusive range ['2025-02-01', '2025-02-01'] assert ( - f"""SELECT COUNT(*) FROM (SELECT "date_id" AS "date_id" FROM (SELECT * FROM "sqlmesh__sqlmesh_audit"."sqlmesh_audit__date_example__{date_snapshot.version}" AS "sqlmesh_audit__date_example__{date_snapshot.version}" WHERE "date_id" BETWEEN CAST('2025-02-01' AS DATE) AND CAST('2025-02-01' AS DATE)) AS "_q_0" WHERE TRUE GROUP BY "date_id" HAVING COUNT(*) > 1) AS "audit\"""" + f"""SELECT COUNT(*) FROM (SELECT "date_id" AS "date_id" FROM (SELECT * FROM "sqlmesh__sqlmesh_audit"."sqlmesh_audit__date_example__{date_snapshot.version}" AS "sqlmesh_audit__date_example__{date_snapshot.version}" WHERE "date_id" BETWEEN CAST('2025-02-01' AS DATE) AND CAST('2025-02-01' AS DATE)) AS "_0" WHERE TRUE GROUP BY "date_id" HAVING COUNT(*) > 1) AS "audit\"""" in caplog.text ) diff --git a/tests/core/test_model.py b/tests/core/test_model.py index f9ef97ecc0..cfcb843739 100644 --- a/tests/core/test_model.py +++ b/tests/core/test_model.py @@ -464,10 +464,10 @@ def test_model_qualification(tmp_path: Path): ctx.upsert_model(load_sql_based_model(expressions)) ctx.plan_builder("dev") - assert ( - """Column '"a"' could not be resolved for model '"db"."table"', the column may not exist or is ambiguous.""" - in mock_logger.call_args[0][0] - ) + warning_msg = mock_logger.call_args[0][0] + assert "ambiguousorinvalidcolumn:" in warning_msg + assert "could not be resolved" in warning_msg + assert "db.table" in warning_msg @use_terminal_console @@ -3650,7 +3650,7 @@ def test_model_ctas_query(): assert ( load_sql_based_model(expressions, dialect="bigquery").ctas_query().sql() - == 'WITH RECURSIVE "a" AS (SELECT * FROM (SELECT * FROM (SELECT * FROM "x" AS "x" WHERE FALSE) AS "_q_0" WHERE FALSE) AS "_q_1" WHERE FALSE), "b" AS (SELECT * FROM "a" AS "a" WHERE FALSE UNION ALL SELECT * FROM "a" AS "a" WHERE FALSE) SELECT * FROM "b" AS "b" WHERE FALSE LIMIT 0' + == 'WITH RECURSIVE "a" AS (SELECT * FROM (SELECT * FROM (SELECT * FROM "x" AS "x" WHERE FALSE) AS "_0" WHERE FALSE) AS "_1" WHERE FALSE), "b" AS (SELECT * FROM "a" AS "a" WHERE FALSE UNION ALL SELECT * FROM "a" AS "a" WHERE FALSE) SELECT * FROM "b" AS "b" WHERE FALSE LIMIT 0' ) expressions = d.parse( @@ -3671,7 +3671,7 @@ def test_model_ctas_query(): assert ( load_sql_based_model(expressions, dialect="bigquery").ctas_query().sql() - == 'WITH RECURSIVE "a" AS (WITH "nested_a" AS (SELECT * FROM (SELECT * FROM (SELECT * FROM "x" AS "x" WHERE FALSE) AS "_q_0" WHERE FALSE) AS "_q_1" WHERE FALSE) SELECT * FROM "nested_a" AS "nested_a" WHERE FALSE), "b" AS (SELECT * FROM "a" AS "a" WHERE FALSE UNION ALL SELECT * FROM "a" AS "a" WHERE FALSE) SELECT * FROM "b" AS "b" WHERE FALSE LIMIT 0' + == 'WITH RECURSIVE "a" AS (WITH "nested_a" AS (SELECT * FROM (SELECT * FROM (SELECT * FROM "x" AS "x" WHERE FALSE) AS "_0" WHERE FALSE) AS "_1" WHERE FALSE) SELECT * FROM "nested_a" AS "nested_a" WHERE FALSE), "b" AS (SELECT * FROM "a" AS "a" WHERE FALSE UNION ALL SELECT * FROM "a" AS "a" WHERE FALSE) SELECT * FROM "b" AS "b" WHERE FALSE LIMIT 0' ) @@ -4995,7 +4995,7 @@ def test_model_session_properties(sushi_context): ) ) assert model.session_properties == { - "query_label": parse_one("[('key1', 'value1'), ('key2', 'value2')]") + "query_label": parse_one("[('key1', 'value1'), ('key2', 'value2')]", dialect="bigquery") } model = load_sql_based_model( diff --git a/tests/dbt/test_model.py b/tests/dbt/test_model.py index 6d100e6aa5..a954f98f41 100644 --- a/tests/dbt/test_model.py +++ b/tests/dbt/test_model.py @@ -626,11 +626,11 @@ def test_load_microbatch_with_ref( context = Context(paths=project_dir) assert ( context.render(microbatch_snapshot_fqn, start="2025-01-01", end="2025-01-10").sql() - == 'SELECT "cola" AS "cola", "ds_source" AS "ds" FROM (SELECT * FROM "local"."my_source"."my_table" AS "my_table" WHERE "ds_source" >= \'2025-01-01 00:00:00+00:00\' AND "ds_source" < \'2025-01-11 00:00:00+00:00\') AS "_q_0"' + == 'SELECT "cola" AS "cola", "ds_source" AS "ds" FROM (SELECT * FROM "local"."my_source"."my_table" AS "my_table" WHERE "ds_source" >= \'2025-01-01 00:00:00+00:00\' AND "ds_source" < \'2025-01-11 00:00:00+00:00\') AS "_0"' ) assert ( context.render(microbatch_two_snapshot_fqn, start="2025-01-01", end="2025-01-10").sql() - == 'SELECT "_q_0"."cola" AS "cola", "_q_0"."ds" AS "ds" FROM (SELECT "microbatch"."cola" AS "cola", "microbatch"."ds" AS "ds" FROM "local"."main"."microbatch" AS "microbatch" WHERE "microbatch"."ds" < \'2025-01-11 00:00:00+00:00\' AND "microbatch"."ds" >= \'2025-01-01 00:00:00+00:00\') AS "_q_0"' + == 'SELECT "_0"."cola" AS "cola", "_0"."ds" AS "ds" FROM (SELECT "microbatch"."cola" AS "cola", "microbatch"."ds" AS "ds" FROM "local"."main"."microbatch" AS "microbatch" WHERE "microbatch"."ds" < \'2025-01-11 00:00:00+00:00\' AND "microbatch"."ds" >= \'2025-01-01 00:00:00+00:00\') AS "_0"' ) diff --git a/tests/dbt/test_transformation.py b/tests/dbt/test_transformation.py index 3b4df916d3..fe6073dfad 100644 --- a/tests/dbt/test_transformation.py +++ b/tests/dbt/test_transformation.py @@ -2213,7 +2213,7 @@ def test_clickhouse_properties(mocker: MockerFixture): ] assert [e.sql("clickhouse") for e in model_to_sqlmesh.partitioned_by] == [ - 'toMonday("ds")', + "dateTrunc('WEEK', \"ds\")", '"partition_col"', ] assert model_to_sqlmesh.storage_format == "MergeTree()" diff --git a/tests/lsp/test_reference_model_column_prefix.py b/tests/lsp/test_reference_model_column_prefix.py index 3cd25a080e..082ee9c8e6 100644 --- a/tests/lsp/test_reference_model_column_prefix.py +++ b/tests/lsp/test_reference_model_column_prefix.py @@ -41,7 +41,7 @@ def test_model_reference_with_column_prefix(): model_refs = get_all_references(lsp_context, URI.from_path(sushi_customers_path), position) - assert len(model_refs) >= 7 + assert len(model_refs) >= 6 # Verify that we have the FROM clause reference assert any(ref.range.start.line == from_clause_range.start.line for ref in model_refs), ( @@ -65,8 +65,8 @@ def test_column_prefix_references_are_found(): # Find all occurrences of sushi.orders in the file ranges = find_ranges_from_regex(read_file, r"sushi\.orders") - # Should find exactly 2: FROM clause and WHERE clause with column prefix - assert len(ranges) == 2, f"Expected 2 occurrences of 'sushi.orders', found {len(ranges)}" + # Should find exactly 1 in FROM clause with column prefix + assert len(ranges) == 1, f"Expected 1 occurrence of 'sushi.orders', found {len(ranges)}" # Verify we have the expected lines line_contents = [read_file[r.start.line].strip() for r in ranges] @@ -76,11 +76,6 @@ def test_column_prefix_references_are_found(): "Should find FROM clause with sushi.orders" ) - # Should find customer_id in WHERE clause with column prefix - assert any("WHERE sushi.orders.customer_id" in content for content in line_contents), ( - "Should find WHERE clause with sushi.orders.customer_id" - ) - def test_quoted_uppercase_table_and_column_references(tmp_path: Path): # Initialize example project in temporary directory with case sensitive normalization diff --git a/tests/lsp/test_reference_model_find_all.py b/tests/lsp/test_reference_model_find_all.py index 7c0077d6cd..cd9c0a3a1c 100644 --- a/tests/lsp/test_reference_model_find_all.py +++ b/tests/lsp/test_reference_model_find_all.py @@ -30,8 +30,8 @@ def test_find_references_for_model_usages(): # Click on the model reference position = Position(line=ranges[0].start.line, character=ranges[0].start.character + 6) references = get_model_find_all_references(lsp_context, URI.from_path(customers_path), position) - assert len(references) >= 7, ( - f"Expected at least 7 references to sushi.orders (including column prefix), found {len(references)}" + assert len(references) >= 6, ( + f"Expected at least 6 references to sushi.orders (including column prefix), found {len(references)}" ) # Verify expected files are present @@ -53,7 +53,7 @@ def test_find_references_for_model_usages(): # Note: customers file has multiple references due to column prefix support expected_ranges = { "orders": [(0, 0, 0, 0)], # the start for the model itself - "customers": [(30, 7, 30, 19), (44, 6, 44, 18)], # FROM clause and WHERE clause + "customers": [(30, 7, 30, 19)], # FROM clause "waiter_revenue_by_day": [(19, 5, 19, 17)], "customer_revenue_lifetime": [(38, 7, 38, 19)], "customer_revenue_by_day": [(33, 5, 33, 17)], diff --git a/tests/utils/test_cache.py b/tests/utils/test_cache.py index 0b6d335446..ed19765b8a 100644 --- a/tests/utils/test_cache.py +++ b/tests/utils/test_cache.py @@ -106,7 +106,7 @@ def test_optimized_query_cache_macro_def_change(tmp_path: Path, mocker: MockerFi assert cache.with_optimized_query(model) assert ( model.render_query_or_raise().sql() - == 'SELECT "_q_0"."a" AS "a" FROM (SELECT 1 AS "a") AS "_q_0" WHERE "_q_0"."a" = 1' + == 'SELECT "_0"."a" AS "a" FROM (SELECT 1 AS "a") AS "_0" WHERE "_0"."a" = 1' ) # Change the filter_ definition @@ -129,5 +129,5 @@ def test_optimized_query_cache_macro_def_change(tmp_path: Path, mocker: MockerFi assert cache.with_optimized_query(new_model) assert ( new_model.render_query_or_raise().sql() - == 'SELECT "_q_0"."a" AS "a" FROM (SELECT 1 AS "a") AS "_q_0" WHERE "_q_0"."a" = 2' + == 'SELECT "_0"."a" AS "a" FROM (SELECT 1 AS "a") AS "_0" WHERE "_0"."a" = 2' ) From 7d18676b8193f9fdfb659815bfcb664592509170 Mon Sep 17 00:00:00 2001 From: Trey Spiller Date: Thu, 22 Jan 2026 10:16:37 -0600 Subject: [PATCH 2/4] remove unnecessary support for older sqlglot versions --- sqlmesh/core/engine_adapter/base.py | 5 ++--- sqlmesh/core/metric/rewriter.py | 3 +-- sqlmesh/core/model/definition.py | 3 +-- sqlmesh/core/test/definition.py | 4 ++-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index 50e22c4609..e2dbb51459 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -2861,7 +2861,7 @@ def _order_projections_and_filter( return query query = t.cast(exp.Query, query.copy()) - with_ = query.args.pop("with_", None) or query.args.pop("with", None) + with_ = query.args.pop("with_", None) select_exprs: t.List[exp.Expression] = [ exp.column(c, quoted=True) for c in target_columns_to_types @@ -2877,8 +2877,7 @@ def _order_projections_and_filter( query = query.where(where, copy=False) if with_: - with_key = "with_" if "with_" in query.arg_types else "with" - query.set(with_key, with_) + query.set("with_", with_) return query diff --git a/sqlmesh/core/metric/rewriter.py b/sqlmesh/core/metric/rewriter.py index af68df0cc9..bbdc6c6135 100644 --- a/sqlmesh/core/metric/rewriter.py +++ b/sqlmesh/core/metric/rewriter.py @@ -57,8 +57,7 @@ def _build_sources(self, projections: t.List[exp.Expression]) -> SourceAggsAndJo return sources def _expand(self, select: exp.Select) -> None: - from_clause = t.cast(exp.From, select.args.get("from") or select.args.get("from_")) - base = from_clause.this.find(exp.Table) + base = select.args["from_"].this.find(exp.Table) base_alias = base.alias_or_name base_name = exp.table_name(base) diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index e4068027ac..831b52a44e 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -753,8 +753,7 @@ def ctas_query(self, **render_kwarg: t.Any) -> exp.Query: query = self.render_query_or_raise(**render_kwarg).limit(0) for select_or_set_op in query.find_all(exp.Select, exp.SetOperation): - from_clause = select_or_set_op.args.get("from") or select_or_set_op.args.get("from_") - if isinstance(select_or_set_op, exp.Select) and from_clause: + if isinstance(select_or_set_op, exp.Select) and select_or_set_op.args.get("from_"): select_or_set_op.where(exp.false(), copy=False) if self.managed_columns: diff --git a/sqlmesh/core/test/definition.py b/sqlmesh/core/test/definition.py index 2fe05c9d69..2a838753de 100644 --- a/sqlmesh/core/test/definition.py +++ b/sqlmesh/core/test/definition.py @@ -711,7 +711,7 @@ def runTest(self) -> None: query = self._render_model_query() sql = query.sql(self._test_adapter_dialect, pretty=self.engine_adapter._pretty_sql) - with_clause = query.args.get("with_") or query.args.get("with") + with_clause = query.args.get("with_") if with_clause: self.test_ctes( @@ -905,7 +905,7 @@ def generate_test( if isinstance(model, SqlModel): assert isinstance(test, SqlModelTest) model_query = test._render_model_query() - with_clause = model_query.args.get("with_") or model_query.args.get("with") + with_clause = model_query.args.get("with_") if with_clause and include_ctes: ctes = {} From 7f147860f16efb85808645e24b13c4148a54a75c Mon Sep 17 00:00:00 2001 From: Trey Spiller Date: Thu, 22 Jan 2026 11:07:46 -0600 Subject: [PATCH 3/4] add migration script --- sqlmesh/migrations/v0101_bump_sqlglot_28-6-0.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 sqlmesh/migrations/v0101_bump_sqlglot_28-6-0.py diff --git a/sqlmesh/migrations/v0101_bump_sqlglot_28-6-0.py b/sqlmesh/migrations/v0101_bump_sqlglot_28-6-0.py new file mode 100644 index 0000000000..88f7bf458e --- /dev/null +++ b/sqlmesh/migrations/v0101_bump_sqlglot_28-6-0.py @@ -0,0 +1,9 @@ +"""Bump sqlglot to 28.6.0.""" + + +def migrate_schemas(engine_adapter, schema, **kwargs): # type: ignore + pass + + +def migrate_rows(engine_adapter, schema, **kwargs): # type: ignore + pass From c5f79c10540ac2a587e07d71c79012843a706bef Mon Sep 17 00:00:00 2001 From: Trey Spiller Date: Mon, 26 Jan 2026 17:21:38 -0600 Subject: [PATCH 4/4] use new version's sushi in migration test --- .circleci/test_migration.sh | 7 ++++--- sqlmesh/migrations/v0101_bump_sqlglot_28-6-0.py | 9 --------- 2 files changed, 4 insertions(+), 12 deletions(-) delete mode 100644 sqlmesh/migrations/v0101_bump_sqlglot_28-6-0.py diff --git a/.circleci/test_migration.sh b/.circleci/test_migration.sh index 9b8fe89e6e..bb1776550a 100755 --- a/.circleci/test_migration.sh +++ b/.circleci/test_migration.sh @@ -24,13 +24,14 @@ TEST_DIR="$TMP_DIR/$EXAMPLE_NAME" echo "Running migration test for '$EXAMPLE_NAME' in '$TEST_DIR' for example project '$EXAMPLE_DIR' using options '$SQLMESH_OPTS'" +# Copy the example project from the *current* checkout so it's stable across old/new SQLMesh versions +cp -r "$EXAMPLE_DIR" "$TEST_DIR" + git checkout $LAST_TAG # Install dependencies from the previous release. make install-dev -cp -r $EXAMPLE_DIR $TEST_DIR - # this is only needed temporarily until the released tag for $LAST_TAG includes this config if [ "$EXAMPLE_NAME" == "sushi_dbt" ]; then echo 'migration_test_config = sqlmesh_config(Path(__file__).parent, dbt_target_name="duckdb")' >> $TEST_DIR/config.py @@ -53,4 +54,4 @@ make install-dev pushd $TEST_DIR sqlmesh $SQLMESH_OPTS migrate sqlmesh $SQLMESH_OPTS diff prod -popd \ No newline at end of file +popd diff --git a/sqlmesh/migrations/v0101_bump_sqlglot_28-6-0.py b/sqlmesh/migrations/v0101_bump_sqlglot_28-6-0.py deleted file mode 100644 index 88f7bf458e..0000000000 --- a/sqlmesh/migrations/v0101_bump_sqlglot_28-6-0.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Bump sqlglot to 28.6.0.""" - - -def migrate_schemas(engine_adapter, schema, **kwargs): # type: ignore - pass - - -def migrate_rows(engine_adapter, schema, **kwargs): # type: ignore - pass