From ffe2213f30617d3b7786e0f55b6255ac612891d4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Jan 2026 14:28:44 +0000 Subject: [PATCH 01/11] Add investigation of FK constraints during transform() Document findings from investigation into how the transform() method handles foreign key constraints that REFERENCE the table being transformed (incoming FKs from other tables). Key findings: - Incoming FKs survive transform because SQLite's ALTER TABLE RENAME automatically updates FK references in the schema - Renaming a referenced COLUMN breaks incoming FKs because SQLite cannot update column references - With FK enforcement ON, transform() correctly detects and rolls back operations that would break FK constraints - With FK enforcement OFF, transforms that break FKs succeed but leave the database in an inconsistent state Also documents a minor issue: leftover temp tables when transform fails. --- TRANSFORM_FK_INVESTIGATION.md | 128 ++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 TRANSFORM_FK_INVESTIGATION.md diff --git a/TRANSFORM_FK_INVESTIGATION.md b/TRANSFORM_FK_INVESTIGATION.md new file mode 100644 index 00000000..cac99556 --- /dev/null +++ b/TRANSFORM_FK_INVESTIGATION.md @@ -0,0 +1,128 @@ +# Investigation: Foreign Key Constraints During transform() + +This document summarizes the investigation into how the `transform()` method handles +foreign key constraints that **reference** the table being transformed (incoming FKs). + +## Background + +The `transform()` method in sqlite-utils performs the following steps: +1. Create a new temporary table with the desired schema +2. Copy data from the old table to the new table +3. Drop the old table +4. Rename the new table to the original table name +5. Recreate indexes + +This raises the question: what happens to foreign key constraints in **other tables** +that reference the table being transformed? + +## Key Finding: SQLite's ALTER TABLE RENAME Behavior + +SQLite's `ALTER TABLE ... RENAME TO` command automatically updates all foreign key +references in the schema. This is documented in the SQLite documentation: + +> "The RENAME command renames the table, and also updates all references to the +> table within the schema" - https://www.sqlite.org/lang_altertable.html + +This means: +- When `authors_new_xxx` is renamed to `authors`, all FK constraints that reference + `authors` continue to work correctly +- The FK references are stored by table **name**, and the rename updates them + +## Test Scenarios and Results + +### Scenario 1: Simple transform (no column rename) +``` +Setup: books.author_id REFERENCES authors(id) +Action: db["authors"].transform(types={"name": str}) +Result: FK constraints survive intact (both with FK ON and OFF) +``` + +### Scenario 2: Rename a non-referenced column +``` +Setup: books.author_id REFERENCES authors(id) +Action: db["authors"].transform(rename={"name": "author_name"}) +Result: FK constraints survive intact (both with FK ON and OFF) +``` + +### Scenario 3: Rename the referenced column (FK ON) +``` +Setup: books.author_id REFERENCES authors(id) +Action: db["authors"].transform(rename={"id": "author_pk"}) +Result: FAILS - "foreign key mismatch" error, transaction rolled back +``` +The transform correctly detects the FK violation via `PRAGMA foreign_key_check` +and rolls back the transaction, preserving the original schema. + +### Scenario 4: Rename the referenced column (FK OFF) +``` +Setup: books.author_id REFERENCES authors(id) +Action: db["authors"].transform(rename={"id": "author_pk"}) +Result: Transform succeeds, but FK constraint is now BROKEN +``` +The FK in `books` still references `authors(id)` but that column no longer exists. +Running `PRAGMA foreign_key_check` produces a "foreign key mismatch" error. + +### Scenario 5: Self-referential FK +``` +Setup: employees.manager_id REFERENCES employees(id) +Action: db["employees"].transform(types={"name": str}) +Result: FK constraint survives intact +``` + +### Scenario 6: Multiple tables referencing the transformed table +``` +Setup: books.author_id, articles.writer_id, quotes.speaker_id all REFERENCE authors(id) +Action: db["authors"].transform(types={"name": str}) +Result: All FK constraints survive intact +``` + +## How transform() Ensures FK Safety + +The `transform()` method (db.py lines 1853-1917) implements the following safety measures: + +1. **Saves FK enforcement state**: Checks `PRAGMA foreign_keys` before starting +2. **Disables FK enforcement**: Sets `PRAGMA foreign_keys=0` during the transform +3. **Executes transform SQL**: Within a transaction (`with self.db.conn:`) +4. **Validates FK integrity**: Runs `PRAGMA foreign_key_check` after the transform +5. **Rolls back on failure**: If FK check fails, the transaction is rolled back +6. **Restores FK state**: Re-enables FK enforcement if it was originally on + +## Summary Table + +| Scenario | FK ON | FK OFF | +|-------------------------------------|--------------------|--------------------| +| Simple transform | Works, FKs intact | Works, FKs intact | +| Rename non-referenced column | Works, FKs intact | Works, FKs intact | +| Rename referenced column | FAILS (rollback) | Works, FKs BROKEN! | +| Drop referenced column | FAILS (rollback) | Works, FKs BROKEN! | +| Self-referential FK | Works, FKs intact | Works, FKs intact | +| Multiple tables with FKs | Works, FKs intact | Works, FKs intact | + +## Known Issue: Leftover Temp Table on Failure + +When `transform()` fails (e.g., due to FK check failure), there may be a leftover +temporary table (e.g., `authors_new_xxx`). This appears to be because the error +occurs after some statements have executed but before the transaction fully commits. + +The original table remains intact, so this is a minor cosmetic issue rather than +a data integrity problem. + +## Recommendations + +1. **Always use FK enforcement** (`PRAGMA foreign_keys=ON`) when working with + relational data to ensure transform() catches FK violations early + +2. **Be cautious when renaming columns**: If a column is referenced by FKs from + other tables, you'll need to update those FKs as well. Consider: + - First transforming the referencing tables to update their FK constraints + - Then transforming the referenced table to rename the column + +3. **Use `foreign_key_check`** after bulk operations with FK enforcement off to + verify data integrity + +## Code References + +- `transform()` method: sqlite_utils/db.py:1853-1917 +- `transform_sql()` method: sqlite_utils/db.py:1919-2127 +- FK handling in transform_sql: sqlite_utils/db.py:1957-1993 +- Related tests: tests/test_transform.py:301-500 From 6944ec2a4f3bffef6dcbff3973f7b68b3f3a2e2c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Jan 2026 14:38:59 +0000 Subject: [PATCH 02/11] Expand FK investigation with detection and fixing analysis Additional findings: - Third case that breaks FKs: changing PK away from referenced column (SQLite requires FK targets to be PRIMARY KEY or UNIQUE) - Detection is easy: iterate tables, find incoming FKs, check if referenced columns are being renamed/dropped/losing uniqueness - Automatic fixing for renames is moderate complexity: transform referencing tables first to update their FK constraints - Challenges include circular references and transaction safety - Proposed API: update_incoming_fks=True flag or better error messages --- TRANSFORM_FK_INVESTIGATION.md | 72 +++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/TRANSFORM_FK_INVESTIGATION.md b/TRANSFORM_FK_INVESTIGATION.md index cac99556..b6cf2d47 100644 --- a/TRANSFORM_FK_INVESTIGATION.md +++ b/TRANSFORM_FK_INVESTIGATION.md @@ -95,9 +95,25 @@ The `transform()` method (db.py lines 1853-1917) implements the following safety | Rename non-referenced column | Works, FKs intact | Works, FKs intact | | Rename referenced column | FAILS (rollback) | Works, FKs BROKEN! | | Drop referenced column | FAILS (rollback) | Works, FKs BROKEN! | +| Change PK away from referenced col | FAILS (rollback) | Works, FKs BROKEN! | | Self-referential FK | Works, FKs intact | Works, FKs intact | | Multiple tables with FKs | Works, FKs intact | Works, FKs intact | +## Cases That Break Incoming FKs + +There are **three** scenarios that break incoming FK constraints: + +1. **Rename referenced column**: The FK references `authors(id)` but the column is + renamed to `author_pk` - the FK now references a non-existent column. + +2. **Drop referenced column**: The FK references `authors(id)` but the column is + dropped entirely - the FK now references a non-existent column. + +3. **Remove PK/UNIQUE from referenced column**: SQLite requires that FK targets be + either PRIMARY KEY or UNIQUE. If you change `id` from `INTEGER PRIMARY KEY` to + just `INTEGER`, the FK becomes invalid ("foreign key mismatch") even though the + column still exists. + ## Known Issue: Leftover Temp Table on Failure When `transform()` fails (e.g., due to FK check failure), there may be a leftover @@ -120,6 +136,62 @@ a data integrity problem. 3. **Use `foreign_key_check`** after bulk operations with FK enforcement off to verify data integrity +## Automatic Detection and Fixing + +### Detection (Easy) + +To detect if a transform will break incoming FKs: + +```python +def get_incoming_fks(db, table_name): + """Find all FKs from other tables that reference this table.""" + incoming = [] + for other_table in db.table_names(): + if other_table == table_name: + continue + for fk in db[other_table].foreign_keys: + if fk.other_table == table_name: + incoming.append({ + "from_table": fk.table, + "from_column": fk.column, + "to_column": fk.other_column, + }) + return incoming +``` + +Then check if any `to_column` values are in the `rename` or `drop` sets, or if +they're losing their PK/UNIQUE status. + +### Automatic Fixing for Column Renames (Moderate Complexity) + +For column renames, it's possible to automatically update the referencing tables: + +1. Before transforming table A, find all tables with FKs to A +2. For each table B with `FK(col -> A.old_col)`: + - Transform B with `foreign_keys=` parameter to update to `FK(col -> A.new_col)` +3. Then transform A with the rename + +### Challenges + +- **Circular references**: A -> B -> A requires careful ordering +- **Chain reactions**: Transforming B might affect table C +- **Transaction safety**: All transforms should succeed or all fail + +### Proposed API Enhancement + +```python +# Option 1: Auto-update with a flag +db["authors"].transform( + rename={"id": "author_pk"}, + update_incoming_fks=True # Automatically update FKs in other tables +) + +# Option 2: Better error message +# "Cannot rename 'id': books.author_id references this column. +# First transform 'books' to update its foreign key, or use +# update_incoming_fks=True to do this automatically." +``` + ## Code References - `transform()` method: sqlite_utils/db.py:1853-1917 From 23bc93239ebd7cb2d2a03c8248855131891bf2d7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Jan 2026 14:53:04 +0000 Subject: [PATCH 03/11] Add update_incoming_fks parameter to transform() When renaming columns that are referenced by foreign keys in other tables, the transform() method now accepts update_incoming_fks=True to automatically update those FK constraints. Implementation: - Added _get_incoming_fks_needing_update() helper to find tables with FKs pointing to renamed columns - Modified transform() to collect SQL for updating incoming FKs - Execute main transform first (so new column exists), then update incoming FKs, all within the same transaction - Added _skip_fk_validation flag to allow generating SQL for FKs that reference columns that don't exist yet (will exist after main transform completes) Includes test for basic column rename case with FK enforcement ON. --- sqlite_utils/db.py | 80 ++++++++++++++++++++++++++++++++++++----- tests/test_transform.py | 47 ++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 9 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index aacdc893..8d0ac1a1 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -974,15 +974,17 @@ def sort_key(p): column_items.insert(0, (hash_id, str)) pk = hash_id # Soundness check foreign_keys point to existing tables - for fk in foreign_keys: - if fk.other_table == name and columns.get(fk.other_column): - continue - if fk.other_column != "rowid" and not any( - c for c in self[fk.other_table].columns if c.name == fk.other_column - ): - raise AlterError( - "No such column: {}.{}".format(fk.other_table, fk.other_column) - ) + # (can be skipped for internal operations like update_incoming_fks) + if not getattr(self, "_skip_fk_validation", False): + for fk in foreign_keys: + if fk.other_table == name and columns.get(fk.other_column): + continue + if fk.other_column != "rowid" and not any( + c for c in self[fk.other_table].columns if c.name == fk.other_column + ): + raise AlterError( + "No such column: {}.{}".format(fk.other_table, fk.other_column) + ) column_defs = [] # ensure pk is a tuple @@ -1850,6 +1852,42 @@ def duplicate(self, new_name: str) -> "Table": self.db.execute(sql) return self.db.table(new_name) + def _get_incoming_fks_needing_update(self, rename: dict) -> list: + """ + Find all tables with FK constraints pointing to columns being renamed. + + Returns a list of (table_name, new_fks) tuples where new_fks is the + updated list of foreign keys for that table. + + :param rename: Dictionary mapping old column names to new column names + """ + tables_needing_update = [] + + for other_table_name in self.db.table_names(): + if other_table_name == self.name: + continue + + other_table = self.db[other_table_name] + other_fks = other_table.foreign_keys + + # Check if any FK references a column being renamed + needs_update = False + new_fks = [] + for fk in other_fks: + if fk.other_table == self.name and fk.other_column in rename: + # This FK needs updating + needs_update = True + new_fks.append( + (fk.column, fk.other_table, rename[fk.other_column]) + ) + else: + new_fks.append((fk.column, fk.other_table, fk.other_column)) + + if needs_update: + tables_needing_update.append((other_table_name, new_fks)) + + return tables_needing_update + def transform( self, *, @@ -1864,6 +1902,7 @@ def transform( foreign_keys: Optional[ForeignKeysType] = None, column_order: Optional[List[str]] = None, keep_table: Optional[str] = None, + update_incoming_fks: bool = False, ) -> "Table": """ Apply an advanced alter table, including operations that are not supported by @@ -1884,8 +1923,27 @@ def transform( to use when creating the table :param keep_table: If specified, the existing table will be renamed to this and will not be dropped + :param update_incoming_fks: If True, automatically update foreign key constraints in other + tables that reference columns being renamed in this table """ assert self.exists(), "Cannot transform a table that doesn't exist yet" + + # Collect SQL for updating incoming FKs if needed + incoming_fk_sqls = [] + if update_incoming_fks and rename: + tables_needing_update = self._get_incoming_fks_needing_update(rename) + for other_table_name, new_fks in tables_needing_update: + other_table = self.db[other_table_name] + # Generate transform SQL for the other table with updated FKs + # Skip FK validation since the new column doesn't exist yet + try: + self.db._skip_fk_validation = True + incoming_fk_sqls.extend( + other_table.transform_sql(foreign_keys=new_fks) + ) + finally: + self.db._skip_fk_validation = False + sqls = self.transform_sql( types=types, rename=rename, @@ -1906,8 +1964,12 @@ def transform( if pragma_foreign_keys_was_on: self.db.execute("PRAGMA foreign_keys=0;") with self.db.conn: + # First: transform the main table (so renamed columns exist) for sql in sqls: self.db.execute(sql) + # Then: update incoming FKs in other tables + for sql in incoming_fk_sqls: + self.db.execute(sql) # Run the foreign_key_check before we commit if pragma_foreign_keys_was_on: self.db.execute("PRAGMA foreign_key_check;") diff --git a/tests/test_transform.py b/tests/test_transform.py index e763a6cd..2fdaa6a4 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -661,3 +661,50 @@ def test_transform_with_unique_constraint_implicit_index(fresh_db): "You must manually drop this index prior to running this transformation and manually recreate the new index after running this transformation." in str(excinfo.value) ) + + +def test_transform_update_incoming_fks_on_column_rename(fresh_db): + """ + Test that update_incoming_fks=True updates FK constraints in other tables + when a referenced column is renamed. + """ + fresh_db.execute("PRAGMA foreign_keys=ON") + + # Create authors table with id as PK + fresh_db["authors"].insert({"id": 1, "name": "Alice"}, pk="id") + + # Create books table with FK to authors.id + fresh_db["books"].insert( + {"id": 1, "title": "Book A", "author_id": 1}, + pk="id", + foreign_keys=[("author_id", "authors", "id")], + ) + + # Verify initial FK + assert fresh_db["books"].foreign_keys == [ + ForeignKey(table="books", column="author_id", other_table="authors", other_column="id") + ] + + # Rename authors.id to authors.author_pk with update_incoming_fks=True + fresh_db["authors"].transform( + rename={"id": "author_pk"}, + update_incoming_fks=True, + ) + + # Verify authors column was renamed + assert "author_pk" in fresh_db["authors"].columns_dict + assert "id" not in fresh_db["authors"].columns_dict + + # Verify books FK was updated to point to new column name + assert fresh_db["books"].foreign_keys == [ + ForeignKey(table="books", column="author_id", other_table="authors", other_column="author_pk") + ] + + # Verify data integrity + assert list(fresh_db["authors"].rows) == [{"author_pk": 1, "name": "Alice"}] + assert list(fresh_db["books"].rows) == [{"id": 1, "title": "Book A", "author_id": 1}] + + # Verify FK enforcement still works + assert fresh_db.execute("PRAGMA foreign_keys").fetchone()[0] == 1 + violations = list(fresh_db.execute("PRAGMA foreign_key_check").fetchall()) + assert violations == [] From e22b34ee2453ef1871fc16f765aae7e7978a298e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Jan 2026 14:56:06 +0000 Subject: [PATCH 04/11] Add self-referential FK support and additional tests - Fix transform_sql to update other_column for self-referential FKs when the referenced column is being renamed - Skip FK validation during transform when update_incoming_fks=True since self-referential FKs temporarily reference non-existent columns - Add tests for multiple tables referencing renamed column - Add test for self-referential FK handling --- sqlite_utils/db.py | 39 ++++++++++----- tests/test_transform.py | 102 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 13 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 8d0ac1a1..9c163ab1 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1944,19 +1944,29 @@ def transform( finally: self.db._skip_fk_validation = False - sqls = self.transform_sql( - types=types, - rename=rename, - drop=drop, - pk=pk, - not_null=not_null, - defaults=defaults, - drop_foreign_keys=drop_foreign_keys, - add_foreign_keys=add_foreign_keys, - foreign_keys=foreign_keys, - column_order=column_order, - keep_table=keep_table, - ) + # Skip FK validation for main transform if update_incoming_fks is True + # because self-referential FKs will reference the new column name + # that only exists after the transform completes + if update_incoming_fks and rename: + self.db._skip_fk_validation = True + try: + sqls = self.transform_sql( + types=types, + rename=rename, + drop=drop, + pk=pk, + not_null=not_null, + defaults=defaults, + drop_foreign_keys=drop_foreign_keys, + add_foreign_keys=add_foreign_keys, + foreign_keys=foreign_keys, + column_order=column_order, + keep_table=keep_table, + ) + finally: + if update_incoming_fks and rename: + self.db._skip_fk_validation = False + pragma_foreign_keys_was_on = self.db.execute("PRAGMA foreign_keys").fetchone()[ 0 ] @@ -2034,6 +2044,9 @@ def transform_sql( for table, column, other_table, other_column in self.foreign_keys: # Copy over old foreign keys, unless we are dropping them if (drop_foreign_keys is None) or (column not in drop_foreign_keys): + # For self-referential FKs, also update the referenced column if renamed + if other_table == self.name: + other_column = rename.get(other_column) or other_column create_table_foreign_keys.append( ForeignKey( table, diff --git a/tests/test_transform.py b/tests/test_transform.py index 2fdaa6a4..c0092d75 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -708,3 +708,105 @@ def test_transform_update_incoming_fks_on_column_rename(fresh_db): assert fresh_db.execute("PRAGMA foreign_keys").fetchone()[0] == 1 violations = list(fresh_db.execute("PRAGMA foreign_key_check").fetchall()) assert violations == [] + + +def test_transform_update_incoming_fks_multiple_tables(fresh_db): + """ + Test that update_incoming_fks=True updates FK constraints in multiple tables + when a referenced column is renamed. + """ + fresh_db.execute("PRAGMA foreign_keys=ON") + + # Create authors table with id as PK + fresh_db["authors"].insert({"id": 1, "name": "Alice"}, pk="id") + + # Create multiple tables with FKs to authors.id + fresh_db["books"].insert( + {"id": 1, "title": "Book A", "author_id": 1}, + pk="id", + foreign_keys=[("author_id", "authors", "id")], + ) + fresh_db["articles"].insert( + {"id": 1, "headline": "Article A", "writer_id": 1}, + pk="id", + foreign_keys=[("writer_id", "authors", "id")], + ) + fresh_db["quotes"].insert( + {"id": 1, "text": "Quote A", "speaker_id": 1}, + pk="id", + foreign_keys=[("speaker_id", "authors", "id")], + ) + + # Rename authors.id to authors.author_pk with update_incoming_fks=True + fresh_db["authors"].transform( + rename={"id": "author_pk"}, + update_incoming_fks=True, + ) + + # Verify authors column was renamed + assert "author_pk" in fresh_db["authors"].columns_dict + assert "id" not in fresh_db["authors"].columns_dict + + # Verify all FKs were updated + assert fresh_db["books"].foreign_keys == [ + ForeignKey(table="books", column="author_id", other_table="authors", other_column="author_pk") + ] + assert fresh_db["articles"].foreign_keys == [ + ForeignKey(table="articles", column="writer_id", other_table="authors", other_column="author_pk") + ] + assert fresh_db["quotes"].foreign_keys == [ + ForeignKey(table="quotes", column="speaker_id", other_table="authors", other_column="author_pk") + ] + + # Verify FK enforcement still works + violations = list(fresh_db.execute("PRAGMA foreign_key_check").fetchall()) + assert violations == [] + + +def test_transform_update_incoming_fks_self_referential(fresh_db): + """ + Test that update_incoming_fks=True handles self-referential FK constraints. + """ + fresh_db.execute("PRAGMA foreign_keys=ON") + + # Create employees table with self-referential FK (manager_id -> id) + fresh_db.execute(""" + CREATE TABLE employees ( + id INTEGER PRIMARY KEY, + name TEXT, + manager_id INTEGER REFERENCES employees(id) + ) + """) + fresh_db["employees"].insert_all([ + {"id": 1, "name": "CEO", "manager_id": None}, + {"id": 2, "name": "VP", "manager_id": 1}, + {"id": 3, "name": "Dev", "manager_id": 2}, + ]) + + # Verify initial FK + assert fresh_db["employees"].foreign_keys == [ + ForeignKey(table="employees", column="manager_id", other_table="employees", other_column="id") + ] + + # Rename employees.id to employees.emp_id with update_incoming_fks=True + fresh_db["employees"].transform( + rename={"id": "emp_id"}, + update_incoming_fks=True, + ) + + # Verify column was renamed + assert "emp_id" in fresh_db["employees"].columns_dict + assert "id" not in fresh_db["employees"].columns_dict + + # Verify self-referential FK was updated + assert fresh_db["employees"].foreign_keys == [ + ForeignKey(table="employees", column="manager_id", other_table="employees", other_column="emp_id") + ] + + # Verify data integrity + rows = list(fresh_db.execute("SELECT * FROM employees ORDER BY emp_id").fetchall()) + assert rows == [(1, "CEO", None), (2, "VP", 1), (3, "Dev", 2)] + + # Verify FK enforcement still works + violations = list(fresh_db.execute("PRAGMA foreign_key_check").fetchall()) + assert violations == [] From 798149b816bfc06de2282fc616656b911a0bc497 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Jan 2026 14:58:11 +0000 Subject: [PATCH 05/11] Add --update-incoming-fks CLI flag for transform command When renaming columns that are referenced by foreign keys in other tables, the --update-incoming-fks flag will automatically update those FK constraints. Example: sqlite-utils transform mydb.db authors \ --rename id author_pk \ --update-incoming-fks This will rename the 'id' column to 'author_pk' and also update any foreign key constraints in other tables (e.g., books.author_id) that reference the renamed column. --- sqlite_utils/cli.py | 8 ++++++++ tests/test_cli.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 9b9ee20e..c802fc68 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2545,6 +2545,11 @@ def schema( multiple=True, help="Drop foreign key constraint for this column", ) +@click.option( + "--update-incoming-fks", + is_flag=True, + help="Update foreign keys in other tables that reference renamed columns", +) @click.option("--sql", is_flag=True, help="Output SQL without executing it") @load_extension_option def transform( @@ -2562,6 +2567,7 @@ def transform( default_none, add_foreign_keys, drop_foreign_keys, + update_incoming_fks, sql, load_extension, ): @@ -2615,6 +2621,8 @@ def transform( kwargs["drop_foreign_keys"] = drop_foreign_keys if add_foreign_keys: kwargs["add_foreign_keys"] = add_foreign_keys + if update_incoming_fks: + kwargs["update_incoming_fks"] = True if sql: for line in db.table(table).transform_sql(**kwargs): diff --git a/tests/test_cli.py b/tests/test_cli.py index 40c3595c..fadbe29b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1810,6 +1810,44 @@ def test_transform_add_or_drop_foreign_key(db_path, extra_args, expected_schema) assert schema == expected_schema +def test_transform_update_incoming_fks_cli(db_path): + """Test --update-incoming-fks flag updates foreign keys in other tables""" + db = Database(db_path) + with db.conn: + db["authors"].insert({"id": 1, "name": "Alice"}, pk="id") + db["books"].insert( + {"id": 1, "title": "Book A", "author_id": 1}, + pk="id", + foreign_keys=[("author_id", "authors", "id")], + ) + + # Rename authors.id to authors.author_pk with --update-incoming-fks + result = CliRunner().invoke( + cli.cli, + [ + "transform", + db_path, + "authors", + "--rename", "id", "author_pk", + "--update-incoming-fks", + ], + ) + assert result.exit_code == 0, result.output + + # Verify authors column was renamed + assert "author_pk" in db["authors"].columns_dict + assert "id" not in db["authors"].columns_dict + + # Verify books FK was updated + assert db["books"].schema == ( + 'CREATE TABLE "books" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "title" TEXT,\n' + ' "author_id" INTEGER REFERENCES "authors"("author_pk")\n' + ")" + ) + + _common_other_schema = ( 'CREATE TABLE "species" (\n "id" INTEGER PRIMARY KEY,\n "species" TEXT\n)' ) From 0a567b7c502034eea7047e956d7fa054d6e46c9f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Jan 2026 14:59:42 +0000 Subject: [PATCH 06/11] Add documentation for update_incoming_fks parameter Document the new update_incoming_fks parameter in: - Python API docs (python-api.rst): New section explaining usage - CLI docs (cli.rst): Document --update-incoming-fks flag --- docs/cli.rst | 3 +++ docs/python-api.rst | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index a6081609..b93868c6 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2113,6 +2113,9 @@ Every option for this table (with the exception of ``--pk-none``) can be specifi ``--add-foreign-key column other_table other_column`` Add a foreign key constraint to ``column`` pointing to ``other_table.other_column``. +``--update-incoming-fks`` + When renaming columns, automatically update foreign key constraints in other tables that reference the renamed columns. For example, if ``books.author_id`` references ``authors.id`` and you rename ``authors.id`` to ``authors.author_pk``, this flag will also update the foreign key in ``books`` to reference the new column name. + If you want to see the SQL that will be executed to make the change without actually executing it, add the ``--sql`` flag. For example: .. code-block:: bash diff --git a/docs/python-api.rst b/docs/python-api.rst index 267591ac..2fe2aeb5 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1631,6 +1631,31 @@ This example drops two foreign keys - the one from ``places.country`` to ``count drop_foreign_keys=("country", "continent") ) +.. _python_api_transform_update_incoming_fks: + +Updating foreign keys in other tables +------------------------------------- + +When renaming columns that are referenced by foreign keys in other tables, you can use the ``update_incoming_fks=True`` parameter to automatically update those foreign key constraints. + +For example, if you have a ``books`` table with a foreign key from ``books.author_id`` to ``authors.id``, and you want to rename ``authors.id`` to ``authors.author_pk``: + +.. code-block:: python + + db["authors"].transform( + rename={"id": "author_pk"}, + update_incoming_fks=True, + ) + +This will rename the column in the ``authors`` table and also update the foreign key constraint in the ``books`` table to reference ``authors.author_pk`` instead of ``authors.id``. + +Without ``update_incoming_fks=True``, this operation would fail with a foreign key mismatch error (if foreign key enforcement is enabled) because the ``books`` table would still reference the old column name. + +This parameter also correctly handles: + +- Multiple tables referencing the renamed column +- Self-referential foreign keys (e.g., an ``employees.manager_id`` column referencing ``employees.id``) + .. _python_api_transform_sql: Custom transformations with .transform_sql() From 5ce545b5926f9a25d8fee1cc042925ae371c8cc3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Jan 2026 15:00:37 +0000 Subject: [PATCH 07/11] Run black and cog for formatting and CLI reference update --- docs/cli-reference.rst | 2 ++ sqlite_utils/db.py | 4 +-- tests/test_cli.py | 4 ++- tests/test_transform.py | 68 ++++++++++++++++++++++++++++++++--------- 4 files changed, 59 insertions(+), 19 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 6231dbf2..ff4800fb 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -474,6 +474,8 @@ See :ref:`cli_transform_table`. Add a foreign key constraint from a column to another table with another column --drop-foreign-key TEXT Drop foreign key constraint for this column + --update-incoming-fks Update foreign keys in other tables that + reference renamed columns --sql Output SQL without executing it --load-extension TEXT Path to SQLite extension, with optional :entrypoint diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9c163ab1..13fda14f 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1877,9 +1877,7 @@ def _get_incoming_fks_needing_update(self, rename: dict) -> list: if fk.other_table == self.name and fk.other_column in rename: # This FK needs updating needs_update = True - new_fks.append( - (fk.column, fk.other_table, rename[fk.other_column]) - ) + new_fks.append((fk.column, fk.other_table, rename[fk.other_column])) else: new_fks.append((fk.column, fk.other_table, fk.other_column)) diff --git a/tests/test_cli.py b/tests/test_cli.py index fadbe29b..3107d218 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1828,7 +1828,9 @@ def test_transform_update_incoming_fks_cli(db_path): "transform", db_path, "authors", - "--rename", "id", "author_pk", + "--rename", + "id", + "author_pk", "--update-incoming-fks", ], ) diff --git a/tests/test_transform.py b/tests/test_transform.py index c0092d75..7c39af3f 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -682,7 +682,9 @@ def test_transform_update_incoming_fks_on_column_rename(fresh_db): # Verify initial FK assert fresh_db["books"].foreign_keys == [ - ForeignKey(table="books", column="author_id", other_table="authors", other_column="id") + ForeignKey( + table="books", column="author_id", other_table="authors", other_column="id" + ) ] # Rename authors.id to authors.author_pk with update_incoming_fks=True @@ -697,12 +699,19 @@ def test_transform_update_incoming_fks_on_column_rename(fresh_db): # Verify books FK was updated to point to new column name assert fresh_db["books"].foreign_keys == [ - ForeignKey(table="books", column="author_id", other_table="authors", other_column="author_pk") + ForeignKey( + table="books", + column="author_id", + other_table="authors", + other_column="author_pk", + ) ] # Verify data integrity assert list(fresh_db["authors"].rows) == [{"author_pk": 1, "name": "Alice"}] - assert list(fresh_db["books"].rows) == [{"id": 1, "title": "Book A", "author_id": 1}] + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "Book A", "author_id": 1} + ] # Verify FK enforcement still works assert fresh_db.execute("PRAGMA foreign_keys").fetchone()[0] == 1 @@ -749,13 +758,28 @@ def test_transform_update_incoming_fks_multiple_tables(fresh_db): # Verify all FKs were updated assert fresh_db["books"].foreign_keys == [ - ForeignKey(table="books", column="author_id", other_table="authors", other_column="author_pk") + ForeignKey( + table="books", + column="author_id", + other_table="authors", + other_column="author_pk", + ) ] assert fresh_db["articles"].foreign_keys == [ - ForeignKey(table="articles", column="writer_id", other_table="authors", other_column="author_pk") + ForeignKey( + table="articles", + column="writer_id", + other_table="authors", + other_column="author_pk", + ) ] assert fresh_db["quotes"].foreign_keys == [ - ForeignKey(table="quotes", column="speaker_id", other_table="authors", other_column="author_pk") + ForeignKey( + table="quotes", + column="speaker_id", + other_table="authors", + other_column="author_pk", + ) ] # Verify FK enforcement still works @@ -770,22 +794,31 @@ def test_transform_update_incoming_fks_self_referential(fresh_db): fresh_db.execute("PRAGMA foreign_keys=ON") # Create employees table with self-referential FK (manager_id -> id) - fresh_db.execute(""" + fresh_db.execute( + """ CREATE TABLE employees ( id INTEGER PRIMARY KEY, name TEXT, manager_id INTEGER REFERENCES employees(id) ) - """) - fresh_db["employees"].insert_all([ - {"id": 1, "name": "CEO", "manager_id": None}, - {"id": 2, "name": "VP", "manager_id": 1}, - {"id": 3, "name": "Dev", "manager_id": 2}, - ]) + """ + ) + fresh_db["employees"].insert_all( + [ + {"id": 1, "name": "CEO", "manager_id": None}, + {"id": 2, "name": "VP", "manager_id": 1}, + {"id": 3, "name": "Dev", "manager_id": 2}, + ] + ) # Verify initial FK assert fresh_db["employees"].foreign_keys == [ - ForeignKey(table="employees", column="manager_id", other_table="employees", other_column="id") + ForeignKey( + table="employees", + column="manager_id", + other_table="employees", + other_column="id", + ) ] # Rename employees.id to employees.emp_id with update_incoming_fks=True @@ -800,7 +833,12 @@ def test_transform_update_incoming_fks_self_referential(fresh_db): # Verify self-referential FK was updated assert fresh_db["employees"].foreign_keys == [ - ForeignKey(table="employees", column="manager_id", other_table="employees", other_column="emp_id") + ForeignKey( + table="employees", + column="manager_id", + other_table="employees", + other_column="emp_id", + ) ] # Verify data integrity From c727ceab0763ff02fe10da720f427db5e9693e4e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Jan 2026 15:11:56 +0000 Subject: [PATCH 08/11] Fix mypy type errors in update_incoming_fks implementation - Add type annotation for incoming_fk_sqls: List[str] - Use self.db.table() instead of self.db[] to get Table type - Use setattr/getattr for _skip_fk_validation to avoid attr-defined error --- sqlite_utils/db.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 13fda14f..84b97a27 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1867,7 +1867,7 @@ def _get_incoming_fks_needing_update(self, rename: dict) -> list: if other_table_name == self.name: continue - other_table = self.db[other_table_name] + other_table = self.db.table(other_table_name) other_fks = other_table.foreign_keys # Check if any FK references a column being renamed @@ -1927,26 +1927,26 @@ def transform( assert self.exists(), "Cannot transform a table that doesn't exist yet" # Collect SQL for updating incoming FKs if needed - incoming_fk_sqls = [] + incoming_fk_sqls: List[str] = [] if update_incoming_fks and rename: tables_needing_update = self._get_incoming_fks_needing_update(rename) for other_table_name, new_fks in tables_needing_update: - other_table = self.db[other_table_name] + other_table = self.db.table(other_table_name) # Generate transform SQL for the other table with updated FKs # Skip FK validation since the new column doesn't exist yet try: - self.db._skip_fk_validation = True + setattr(self.db, "_skip_fk_validation", True) incoming_fk_sqls.extend( other_table.transform_sql(foreign_keys=new_fks) ) finally: - self.db._skip_fk_validation = False + setattr(self.db, "_skip_fk_validation", False) # Skip FK validation for main transform if update_incoming_fks is True # because self-referential FKs will reference the new column name # that only exists after the transform completes if update_incoming_fks and rename: - self.db._skip_fk_validation = True + setattr(self.db, "_skip_fk_validation", True) try: sqls = self.transform_sql( types=types, @@ -1963,7 +1963,7 @@ def transform( ) finally: if update_incoming_fks and rename: - self.db._skip_fk_validation = False + setattr(self.db, "_skip_fk_validation", False) pragma_foreign_keys_was_on = self.db.execute("PRAGMA foreign_keys").fetchone()[ 0 From a946e808092e7950fd7d4a0a57c54d02c1965854 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Jan 2026 15:18:24 +0000 Subject: [PATCH 09/11] Remove TRANSFORM_FK_INVESTIGATION.md --- TRANSFORM_FK_INVESTIGATION.md | 200 ---------------------------------- 1 file changed, 200 deletions(-) delete mode 100644 TRANSFORM_FK_INVESTIGATION.md diff --git a/TRANSFORM_FK_INVESTIGATION.md b/TRANSFORM_FK_INVESTIGATION.md deleted file mode 100644 index b6cf2d47..00000000 --- a/TRANSFORM_FK_INVESTIGATION.md +++ /dev/null @@ -1,200 +0,0 @@ -# Investigation: Foreign Key Constraints During transform() - -This document summarizes the investigation into how the `transform()` method handles -foreign key constraints that **reference** the table being transformed (incoming FKs). - -## Background - -The `transform()` method in sqlite-utils performs the following steps: -1. Create a new temporary table with the desired schema -2. Copy data from the old table to the new table -3. Drop the old table -4. Rename the new table to the original table name -5. Recreate indexes - -This raises the question: what happens to foreign key constraints in **other tables** -that reference the table being transformed? - -## Key Finding: SQLite's ALTER TABLE RENAME Behavior - -SQLite's `ALTER TABLE ... RENAME TO` command automatically updates all foreign key -references in the schema. This is documented in the SQLite documentation: - -> "The RENAME command renames the table, and also updates all references to the -> table within the schema" - https://www.sqlite.org/lang_altertable.html - -This means: -- When `authors_new_xxx` is renamed to `authors`, all FK constraints that reference - `authors` continue to work correctly -- The FK references are stored by table **name**, and the rename updates them - -## Test Scenarios and Results - -### Scenario 1: Simple transform (no column rename) -``` -Setup: books.author_id REFERENCES authors(id) -Action: db["authors"].transform(types={"name": str}) -Result: FK constraints survive intact (both with FK ON and OFF) -``` - -### Scenario 2: Rename a non-referenced column -``` -Setup: books.author_id REFERENCES authors(id) -Action: db["authors"].transform(rename={"name": "author_name"}) -Result: FK constraints survive intact (both with FK ON and OFF) -``` - -### Scenario 3: Rename the referenced column (FK ON) -``` -Setup: books.author_id REFERENCES authors(id) -Action: db["authors"].transform(rename={"id": "author_pk"}) -Result: FAILS - "foreign key mismatch" error, transaction rolled back -``` -The transform correctly detects the FK violation via `PRAGMA foreign_key_check` -and rolls back the transaction, preserving the original schema. - -### Scenario 4: Rename the referenced column (FK OFF) -``` -Setup: books.author_id REFERENCES authors(id) -Action: db["authors"].transform(rename={"id": "author_pk"}) -Result: Transform succeeds, but FK constraint is now BROKEN -``` -The FK in `books` still references `authors(id)` but that column no longer exists. -Running `PRAGMA foreign_key_check` produces a "foreign key mismatch" error. - -### Scenario 5: Self-referential FK -``` -Setup: employees.manager_id REFERENCES employees(id) -Action: db["employees"].transform(types={"name": str}) -Result: FK constraint survives intact -``` - -### Scenario 6: Multiple tables referencing the transformed table -``` -Setup: books.author_id, articles.writer_id, quotes.speaker_id all REFERENCE authors(id) -Action: db["authors"].transform(types={"name": str}) -Result: All FK constraints survive intact -``` - -## How transform() Ensures FK Safety - -The `transform()` method (db.py lines 1853-1917) implements the following safety measures: - -1. **Saves FK enforcement state**: Checks `PRAGMA foreign_keys` before starting -2. **Disables FK enforcement**: Sets `PRAGMA foreign_keys=0` during the transform -3. **Executes transform SQL**: Within a transaction (`with self.db.conn:`) -4. **Validates FK integrity**: Runs `PRAGMA foreign_key_check` after the transform -5. **Rolls back on failure**: If FK check fails, the transaction is rolled back -6. **Restores FK state**: Re-enables FK enforcement if it was originally on - -## Summary Table - -| Scenario | FK ON | FK OFF | -|-------------------------------------|--------------------|--------------------| -| Simple transform | Works, FKs intact | Works, FKs intact | -| Rename non-referenced column | Works, FKs intact | Works, FKs intact | -| Rename referenced column | FAILS (rollback) | Works, FKs BROKEN! | -| Drop referenced column | FAILS (rollback) | Works, FKs BROKEN! | -| Change PK away from referenced col | FAILS (rollback) | Works, FKs BROKEN! | -| Self-referential FK | Works, FKs intact | Works, FKs intact | -| Multiple tables with FKs | Works, FKs intact | Works, FKs intact | - -## Cases That Break Incoming FKs - -There are **three** scenarios that break incoming FK constraints: - -1. **Rename referenced column**: The FK references `authors(id)` but the column is - renamed to `author_pk` - the FK now references a non-existent column. - -2. **Drop referenced column**: The FK references `authors(id)` but the column is - dropped entirely - the FK now references a non-existent column. - -3. **Remove PK/UNIQUE from referenced column**: SQLite requires that FK targets be - either PRIMARY KEY or UNIQUE. If you change `id` from `INTEGER PRIMARY KEY` to - just `INTEGER`, the FK becomes invalid ("foreign key mismatch") even though the - column still exists. - -## Known Issue: Leftover Temp Table on Failure - -When `transform()` fails (e.g., due to FK check failure), there may be a leftover -temporary table (e.g., `authors_new_xxx`). This appears to be because the error -occurs after some statements have executed but before the transaction fully commits. - -The original table remains intact, so this is a minor cosmetic issue rather than -a data integrity problem. - -## Recommendations - -1. **Always use FK enforcement** (`PRAGMA foreign_keys=ON`) when working with - relational data to ensure transform() catches FK violations early - -2. **Be cautious when renaming columns**: If a column is referenced by FKs from - other tables, you'll need to update those FKs as well. Consider: - - First transforming the referencing tables to update their FK constraints - - Then transforming the referenced table to rename the column - -3. **Use `foreign_key_check`** after bulk operations with FK enforcement off to - verify data integrity - -## Automatic Detection and Fixing - -### Detection (Easy) - -To detect if a transform will break incoming FKs: - -```python -def get_incoming_fks(db, table_name): - """Find all FKs from other tables that reference this table.""" - incoming = [] - for other_table in db.table_names(): - if other_table == table_name: - continue - for fk in db[other_table].foreign_keys: - if fk.other_table == table_name: - incoming.append({ - "from_table": fk.table, - "from_column": fk.column, - "to_column": fk.other_column, - }) - return incoming -``` - -Then check if any `to_column` values are in the `rename` or `drop` sets, or if -they're losing their PK/UNIQUE status. - -### Automatic Fixing for Column Renames (Moderate Complexity) - -For column renames, it's possible to automatically update the referencing tables: - -1. Before transforming table A, find all tables with FKs to A -2. For each table B with `FK(col -> A.old_col)`: - - Transform B with `foreign_keys=` parameter to update to `FK(col -> A.new_col)` -3. Then transform A with the rename - -### Challenges - -- **Circular references**: A -> B -> A requires careful ordering -- **Chain reactions**: Transforming B might affect table C -- **Transaction safety**: All transforms should succeed or all fail - -### Proposed API Enhancement - -```python -# Option 1: Auto-update with a flag -db["authors"].transform( - rename={"id": "author_pk"}, - update_incoming_fks=True # Automatically update FKs in other tables -) - -# Option 2: Better error message -# "Cannot rename 'id': books.author_id references this column. -# First transform 'books' to update its foreign key, or use -# update_incoming_fks=True to do this automatically." -``` - -## Code References - -- `transform()` method: sqlite_utils/db.py:1853-1917 -- `transform_sql()` method: sqlite_utils/db.py:1919-2127 -- FK handling in transform_sql: sqlite_utils/db.py:1957-1993 -- Related tests: tests/test_transform.py:301-500 From cb33bec699ea262b4f68c408dbb9829e51c1950f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Jan 2026 15:23:44 +0000 Subject: [PATCH 10/11] Reformat with black 26.1.0 --- sqlite_utils/cli.py | 19 ++++-------- sqlite_utils/db.py | 60 +++++++++++------------------------- tests/conftest.py | 6 ++-- tests/test_analyze_tables.py | 9 ++---- tests/test_cli.py | 21 ++++--------- tests/test_cli_convert.py | 6 ++-- tests/test_create.py | 1 - tests/test_default_value.py | 1 - tests/test_duplicate.py | 6 ++-- tests/test_extract.py | 9 ++---- tests/test_introspect.py | 12 +++----- tests/test_transform.py | 12 +++----- 12 files changed, 48 insertions(+), 114 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c802fc68..38d5c7f7 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -42,7 +42,6 @@ TypeTracker, ) - CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) @@ -2919,8 +2918,7 @@ def _analyze(db, tables, columns, save, common_limit=10, no_most=False, no_least ) details = ( ( - textwrap.dedent( - """ + textwrap.dedent(""" {table}.{column}: ({i}/{total}) Total rows: {total_rows} @@ -2928,8 +2926,7 @@ def _analyze(db, tables, columns, save, common_limit=10, no_most=False, no_least Blank rows: {num_blank} Distinct values: {num_distinct}{most_common_rendered}{least_common_rendered} - """ - ) + """) .strip() .format( i=i + 1, @@ -2976,8 +2973,7 @@ def uninstall(packages, yes): def _generate_convert_help(): - help = textwrap.dedent( - """ + help = textwrap.dedent(""" Convert columns using Python code you supply. For example: \b @@ -2990,8 +2986,7 @@ def _generate_convert_help(): Use "-" for CODE to read Python code from standard input. The following common operations are available as recipe functions: - """ - ).strip() + """).strip() recipe_names = [ n for n in dir(recipes) @@ -3005,15 +3000,13 @@ def _generate_convert_help(): name, str(inspect.signature(fn)), textwrap.dedent(fn.__doc__.rstrip()) ) help += "\n\n" - help += textwrap.dedent( - """ + help += textwrap.dedent(""" You can use these recipes like so: \b sqlite-utils convert my.db mytable mycolumn \\ 'r.jsonsplit(value, delimiter=":")' - """ - ).strip() + """).strip() return help diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 84b97a27..77a6f809 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2348,12 +2348,10 @@ def create_index( "{}_{}".format(index_name, suffix) if suffix else index_name ) sql = ( - textwrap.dedent( - """ + textwrap.dedent(""" CREATE {unique}INDEX {if_not_exists}{index_name} ON {table_name} ({columns}); - """ - ) + """) .strip() .format( index_name=quote_identifier(created_index_name), @@ -2548,8 +2546,7 @@ def enable_counts(self) -> None: See :ref:`python_api_cached_table_counts` for details. """ sql = ( - textwrap.dedent( - """ + textwrap.dedent(""" {create_counts_table} CREATE TRIGGER IF NOT EXISTS {trigger_insert} AFTER INSERT ON {table} BEGIN @@ -2574,8 +2571,7 @@ def enable_counts(self) -> None: ); END; INSERT OR REPLACE INTO _counts VALUES ({table_quoted}, (select count(*) from {table})); - """ - ) + """) .strip() .format( create_counts_table=_COUNTS_TABLE_CREATE_SQL.format( @@ -2627,14 +2623,12 @@ def enable_fts( :param replace: Should any existing FTS index for this table be replaced by the new one? """ create_fts_sql = ( - textwrap.dedent( - """ + textwrap.dedent(""" CREATE VIRTUAL TABLE {table_fts} USING {fts_version} ( {columns},{tokenize} content={table} ) - """ - ) + """) .strip() .format( table=quote_identifier(self.name), @@ -2672,8 +2666,7 @@ def enable_fts( table = quote_identifier(self.name) table_fts = quote_identifier(self.name + "_fts") triggers = ( - textwrap.dedent( - """ + textwrap.dedent(""" CREATE TRIGGER {table_ai} AFTER INSERT ON {table} BEGIN INSERT INTO {table_fts} (rowid, {columns}) VALUES (new.rowid, {new_cols}); END; @@ -2684,8 +2677,7 @@ def enable_fts( INSERT INTO {table_fts} ({table_fts}, rowid, {columns}) VALUES('delete', old.rowid, {old_cols}); INSERT INTO {table_fts} (rowid, {columns}) VALUES (new.rowid, {new_cols}); END; - """ - ) + """) .strip() .format( table=table, @@ -2710,12 +2702,10 @@ def populate_fts(self, columns: Iterable[str]) -> "Table": """ columns_quoted = ", ".join(quote_identifier(c) for c in columns) sql = ( - textwrap.dedent( - """ + textwrap.dedent(""" INSERT INTO {table_fts} (rowid, {columns}) SELECT rowid, {columns} FROM {table}; - """ - ) + """) .strip() .format( table=quote_identifier(self.name), @@ -2732,17 +2722,11 @@ def disable_fts(self) -> "Table": if fts_table: self.db[fts_table].drop() # Now delete the triggers that related to that table - sql = ( - textwrap.dedent( - """ + sql = textwrap.dedent(""" SELECT name FROM sqlite_master WHERE type = 'trigger' AND (sql LIKE '% INSERT INTO [{}]%' OR sql LIKE '% INSERT INTO "{}"%') - """ - ) - .strip() - .format(fts_table, fts_table) - ) + """).strip().format(fts_table, fts_table) trigger_names = [] for row in self.db.execute(sql).fetchall(): trigger_names.append(row[0]) @@ -2768,8 +2752,7 @@ def rebuild_fts(self) -> "Table": def detect_fts(self) -> Optional[str]: "Detect if table has a corresponding FTS virtual table and return it" - sql = textwrap.dedent( - """ + sql = textwrap.dedent(""" SELECT name FROM sqlite_master WHERE rootpage = 0 AND ( @@ -2780,8 +2763,7 @@ def detect_fts(self) -> Optional[str]: AND sql LIKE '%VIRTUAL TABLE%USING FTS%' ) ) - """ - ).strip() + """).strip() args = { "like": '%VIRTUAL TABLE%USING FTS%content="{}"%'.format(self.name), "like2": '%VIRTUAL TABLE%USING FTS%content="{}"%'.format(self.name), @@ -2797,13 +2779,9 @@ def optimize(self) -> "Table": "Run the ``optimize`` operation against the associated full-text search index table." fts_table = self.detect_fts() if fts_table is not None: - self.db.execute( - """ + self.db.execute(""" INSERT INTO {table} ({table}) VALUES ("optimize"); - """.strip().format( - table=quote_identifier(fts_table) - ) - ) + """.strip().format(table=quote_identifier(fts_table))) return self def search_sql( @@ -2841,8 +2819,7 @@ def search_sql( ) fts_table_quoted = quote_identifier(fts_table) virtual_table_using = self.db.table(fts_table).virtual_table_using - sql = textwrap.dedent( - """ + sql = textwrap.dedent(""" with {original} as ( select rowid, @@ -2859,8 +2836,7 @@ def search_sql( order by {order_by} {limit_offset} - """ - ).strip() + """).strip() if virtual_table_using == "FTS5": rank_implementation = "{}.rank".format(fts_table_quoted) else: diff --git a/tests/conftest.py b/tests/conftest.py index 3990d76e..6fd35b85 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,14 +42,12 @@ def fresh_db(): @pytest.fixture def existing_db(): database = Database(memory=True) - database.executescript( - """ + database.executescript(""" CREATE TABLE foo (text TEXT); INSERT INTO foo (text) values ("one"); INSERT INTO foo (text) values ("two"); INSERT INTO foo (text) values ("three"); - """ - ) + """) return database diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index 4618eff1..a2ce585d 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -143,10 +143,7 @@ def db_to_analyze_path(db_to_analyze, tmpdir): def test_analyze_table(db_to_analyze_path): result = CliRunner().invoke(cli.cli, ["analyze-tables", db_to_analyze_path]) - assert ( - result.output.strip() - == ( - """ + assert result.output.strip() == (""" stuff.id: (1/3) Total rows: 8 @@ -179,9 +176,7 @@ def test_analyze_table(db_to_analyze_path): Most common: 5: 5 - 3: 4""" - ).strip() - ) + 3: 4""").strip() def test_analyze_table_save(db_to_analyze_path): diff --git a/tests/test_cli.py b/tests/test_cli.py index 3107d218..e11508ad 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -967,12 +967,9 @@ def test_query_json_with_json_cols(db_path): result = CliRunner().invoke( cli.cli, [db_path, "select id, name, friends from dogs"] ) - assert ( - r""" + assert r""" [{"id": 1, "name": "Cleo", "friends": "[{\"name\": \"Pancakes\"}, {\"name\": \"Bailey\"}]"}] - """.strip() - == result.output.strip() - ) + """.strip() == result.output.strip() # With --json-cols: result = CliRunner().invoke( cli.cli, [db_path, "select id, name, friends from dogs", "--json-cols"] @@ -2038,12 +2035,10 @@ def test_search_quote(tmpdir): def test_indexes(tmpdir): db_path = str(tmpdir / "test.db") db = Database(db_path) - db.conn.executescript( - """ + db.conn.executescript(""" create table Gosh (c1 text, c2 text, c3 text); create index Gosh_idx on Gosh(c2, c3 desc); - """ - ) + """) result = CliRunner().invoke( cli.cli, ["indexes", str(db_path)], @@ -2134,16 +2129,12 @@ def test_triggers(tmpdir, extra_args, expected): pk="id", ) db["counter"].insert({"count": 1}) - db.conn.execute( - textwrap.dedent( - """ + db.conn.execute(textwrap.dedent(""" CREATE TRIGGER blah AFTER INSERT ON articles BEGIN UPDATE counter SET count = count + 1; END - """ - ) - ) + """)) args = ["triggers", db_path] if extra_args: args.extend(extra_args) diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 387b3181..443e72c6 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -371,16 +371,14 @@ def test_convert_multi_complex_column_types(fresh_db_and_path): ], pk="id", ) - code = textwrap.dedent( - """ + code = textwrap.dedent(""" if value == 1: return {"is_str": "", "is_float": 1.2, "is_int": None} elif value == 2: return {"is_float": 1, "is_int": 12} elif value == 3: return {"is_bytes": b"blah"} - """ - ) + """) result = CliRunner().invoke( cli.cli, [ diff --git a/tests/test_create.py b/tests/test_create.py index ea09c459..b1a6ad1f 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -20,7 +20,6 @@ import pytest import uuid - try: import pandas as pd # type: ignore except ImportError: diff --git a/tests/test_default_value.py b/tests/test_default_value.py index 9ffdb144..c594c9f5 100644 --- a/tests/test_default_value.py +++ b/tests/test_default_value.py @@ -1,6 +1,5 @@ import pytest - EXAMPLES = [ ("TEXT DEFAULT 'foo'", "'foo'", "'foo'"), ("TEXT DEFAULT 'foo)'", "'foo)'", "'foo)'"), diff --git a/tests/test_duplicate.py b/tests/test_duplicate.py index 552f697c..28961d2e 100644 --- a/tests/test_duplicate.py +++ b/tests/test_duplicate.py @@ -5,14 +5,12 @@ def test_duplicate(fresh_db): # Create table using native Sqlite statement: - fresh_db.execute( - """CREATE TABLE "table1" ( + fresh_db.execute("""CREATE TABLE "table1" ( "text_col" TEXT, "real_col" REAL, "int_col" INTEGER, "bool_col" INTEGER, - "datetime_col" TEXT)""" - ) + "datetime_col" TEXT)""") # Insert one row of mock data: dt = datetime.datetime.now() data = { diff --git a/tests/test_extract.py b/tests/test_extract.py index 435c1aea..d24c5974 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -126,9 +126,7 @@ def test_extract_rowid_table(fresh_db): ' "common_name_latin_name_id" INTEGER REFERENCES "common_name_latin_name"("id")\n' ")" ) - assert ( - fresh_db.execute( - """ + assert fresh_db.execute(""" select tree.name, common_name_latin_name.common_name, @@ -136,10 +134,7 @@ def test_extract_rowid_table(fresh_db): from tree join common_name_latin_name on tree.common_name_latin_name_id = common_name_latin_name.id - """ - ).fetchall() - == [("Tree 1", "Palm", "Arecaceae")] - ) + """).fetchall() == [("Tree 1", "Palm", "Arecaceae")] def test_reuse_lookup_table(fresh_db): diff --git a/tests/test_introspect.py b/tests/test_introspect.py index ab61c158..4ff3f772 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -109,13 +109,11 @@ def test_table_repr(fresh_db): def test_indexes(fresh_db): - fresh_db.executescript( - """ + fresh_db.executescript(""" create table Gosh (c1 text, c2 text, c3 text); create index Gosh_c1 on Gosh(c1); create index Gosh_c2c3 on Gosh(c2, c3); - """ - ) + """) assert [ Index( seq=0, @@ -130,13 +128,11 @@ def test_indexes(fresh_db): def test_xindexes(fresh_db): - fresh_db.executescript( - """ + fresh_db.executescript(""" create table Gosh (c1 text, c2 text, c3 text); create index Gosh_c1 on Gosh(c1); create index Gosh_c2c3 on Gosh(c2, c3 desc); - """ - ) + """) assert fresh_db["Gosh"].xindexes == [ XIndex( name="Gosh_c2c3", diff --git a/tests/test_transform.py b/tests/test_transform.py index 7c39af3f..e7bb4143 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -638,15 +638,13 @@ def test_transform_with_indexes_errors(fresh_db, transform_params): def test_transform_with_unique_constraint_implicit_index(fresh_db): dogs = fresh_db["dogs"] # Create a table with a UNIQUE constraint on 'name', which creates an implicit index - fresh_db.execute( - """ + fresh_db.execute(""" CREATE TABLE dogs ( id INTEGER PRIMARY KEY, name TEXT UNIQUE, age INTEGER ); - """ - ) + """) dogs.insert({"id": 1, "name": "Cleo", "age": 5}) # Attempt to transform the table without modifying 'name' @@ -794,15 +792,13 @@ def test_transform_update_incoming_fks_self_referential(fresh_db): fresh_db.execute("PRAGMA foreign_keys=ON") # Create employees table with self-referential FK (manager_id -> id) - fresh_db.execute( - """ + fresh_db.execute(""" CREATE TABLE employees ( id INTEGER PRIMARY KEY, name TEXT, manager_id INTEGER REFERENCES employees(id) ) - """ - ) + """) fresh_db["employees"].insert_all( [ {"id": 1, "name": "CEO", "manager_id": None}, From 8bddd3ccfe3dc8a240910ba08460f1d14b924097 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Jan 2026 15:26:15 +0000 Subject: [PATCH 11/11] Bump minimum black version to 26.1.0 Black 26.1.0 introduced new formatting rules that affect this codebase. Pin to >=26.1.0 to ensure consistent formatting across all environments. Black 26.1.0 requires Python >=3.10 which matches this project's minimum. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a50a3a8e..55ccda9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ [dependency-groups] dev = [ - "black>=24.1.1", + "black>=26.1.0", "cogapp", "hypothesis", "pytest",