Skip to content

Commit 2b52b5e

Browse files
committed
Preserve AUTOINCREMENT through transforms
1 parent fcfccea commit 2b52b5e

5 files changed

Lines changed: 139 additions & 1 deletion

File tree

‎docs/changelog.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
Unreleased
1010
----------
1111

12+
- ``table.transform()`` now preserves ``AUTOINCREMENT`` primary keys and their sequence high-water marks. Previously a transform removed ``AUTOINCREMENT`` and could reuse deleted row IDs. (:issue:`602`)
1213
- New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`)
1314
- New ``sqlite_utils.ANY`` marker type for creating and introspecting SQLite ``ANY`` columns. The Python API and CLI can create, add and transform these columns, and ``table.transform()`` and ``table.extract()`` now preserve ``ANY`` columns and their values in ``STRICT`` tables. (:issue:`790`)
1415
- ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`)

‎sqlite_utils/create_table_parser.py‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
"""Helpers for parsing CHECK constraints from SQLite CREATE TABLE SQL.
1+
"""Helpers for parsing constraints from SQLite CREATE TABLE SQL.
22
33
SQLite does not expose CHECK constraints through a pragma, so preserving them
44
across a table rebuild requires reading ``sqlite_schema.sql``. This module is
@@ -564,6 +564,34 @@ def parse_checks(create_sql: str) -> list[Check]:
564564
return checks
565565

566566

567+
def parse_autoincrement(create_sql: str) -> str | None:
568+
"""Return the AUTOINCREMENT column from a valid CREATE TABLE statement."""
569+
body_info = _table_body(create_sql)
570+
if body_info is None:
571+
return None
572+
body, _ = body_info
573+
for item, _, _ in _split_spans(body, _lex(body)):
574+
item_tokens = _meaningful(_lex(item))
575+
if not item_tokens:
576+
continue
577+
head = item_tokens[0]
578+
if (
579+
head.kind == "word" and head.text.upper() in _TABLE_CONSTRAINT_KEYWORDS
580+
) or head.is_keyword("CONSTRAINT"):
581+
continue
582+
column = _unquote(head.text)
583+
index = 1
584+
while index < len(item_tokens):
585+
token = item_tokens[index]
586+
if token.text == "(":
587+
index = _matching_paren(item_tokens, index) + 1
588+
continue
589+
if token.is_keyword("AUTOINCREMENT"):
590+
return column
591+
index += 1
592+
return None
593+
594+
567595
def parse_column_comments(create_sql: str) -> dict[str, ColumnComments]:
568596
"""Return comments immediately before and after each column definition."""
569597
body_info = _table_body(create_sql)

‎sqlite_utils/db.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
ColumnComments,
3434
ParseError,
3535
check_references_identifier,
36+
parse_autoincrement,
3637
parse_checks,
3738
parse_column_comments,
3839
rewrite_check_expression,
@@ -1422,6 +1423,7 @@ def create_table_sql(
14221423
strict: bool = False,
14231424
_checks: Iterable[Check] | None = None,
14241425
_column_comments: Mapping[str, ColumnComments] | None = None,
1426+
_autoincrement: str | None = None,
14251427
) -> str:
14261428
"""
14271429
Returns the SQL ``CREATE TABLE`` statement for creating the specified table.
@@ -1525,10 +1527,22 @@ def sort_key(p):
15251527
column_items.insert(0, (pk, int))
15261528
elif pk:
15271529
pk = [resolve_casing(p, [c[0] for c in column_items]) for p in pk]
1530+
if _autoincrement is not None:
1531+
_autoincrement = resolve_casing(
1532+
_autoincrement, [c[0] for c in column_items]
1533+
)
1534+
if _autoincrement != single_pk:
1535+
raise ValueError("AUTOINCREMENT requires a single-column primary key")
15281536
for column_name, column_type in column_items:
15291537
column_extras = []
15301538
if column_name == single_pk:
15311539
column_extras.append("PRIMARY KEY")
1540+
if column_name == _autoincrement:
1541+
if COLUMN_TYPE_MAPPING[column_type] != "INTEGER":
1542+
raise ValueError(
1543+
"AUTOINCREMENT requires an INTEGER PRIMARY KEY column"
1544+
)
1545+
column_extras.append("AUTOINCREMENT")
15321546
if column_name in not_null:
15331547
column_extras.append("NOT NULL")
15341548
if column_name in defaults and defaults[column_name] is not None:
@@ -2748,6 +2762,7 @@ def transform_sql(
27482762
try:
27492763
existing_checks = self.checks
27502764
existing_column_comments = parse_column_comments(self.schema)
2765+
existing_autoincrement = parse_autoincrement(self.schema)
27512766
except ParseError as ex:
27522767
raise TransformError(
27532768
f"Could not parse table schema for table {self.name!r}: {ex}"
@@ -2870,6 +2885,11 @@ def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey:
28702885
new_column_pairs.append((new_name, type_))
28712886
copy_from_to[name] = new_name
28722887

2888+
if existing_autoincrement:
2889+
existing_autoincrement = resolve_casing(
2890+
existing_autoincrement, existing_columns
2891+
)
2892+
28732893
if pk is DEFAULT:
28742894
pks_renamed = tuple(
28752895
rename.get(pk_name) or pk_name
@@ -2880,6 +2900,28 @@ def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey:
28802900
else:
28812901
pk = pks_renamed
28822902

2903+
create_table_autoincrement = None
2904+
if existing_autoincrement and existing_autoincrement not in drop:
2905+
renamed_autoincrement = (
2906+
rename.get(existing_autoincrement) or existing_autoincrement
2907+
)
2908+
single_pk = pk[0] if isinstance(pk, (list, tuple)) and len(pk) == 1 else pk
2909+
new_column_types = dict(new_column_pairs)
2910+
if (
2911+
single_pk == renamed_autoincrement
2912+
and COLUMN_TYPE_MAPPING.get(new_column_types.get(renamed_autoincrement))
2913+
== "INTEGER"
2914+
):
2915+
create_table_autoincrement = renamed_autoincrement
2916+
2917+
autoincrement_sequence = None
2918+
if create_table_autoincrement:
2919+
sequence_row = self.db.execute(
2920+
"SELECT seq FROM sqlite_sequence WHERE name = ?", [self.name]
2921+
).fetchone()
2922+
if sequence_row is not None:
2923+
autoincrement_sequence = sequence_row[0]
2924+
28832925
# not_null may be a set or dict, need to convert to a set
28842926
create_table_not_null = {
28852927
rename.get(c.name) or c.name
@@ -2931,6 +2973,7 @@ def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey:
29312973
strict=self.strict if strict is None else strict,
29322974
_checks=create_table_checks,
29332975
_column_comments=create_table_column_comments,
2976+
_autoincrement=create_table_autoincrement,
29342977
).strip()
29352978
)
29362979

@@ -3053,6 +3096,23 @@ def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey:
30533096
"ON" if legacy_alter_table_was_on else "OFF"
30543097
)
30553098
)
3099+
if autoincrement_sequence is not None:
3100+
table_name_literal = self.db.quote(self.name)
3101+
sqls.extend(
3102+
(
3103+
"UPDATE sqlite_sequence SET seq = MAX(seq, {sequence}) "
3104+
"WHERE name = {table_name};".format(
3105+
sequence=autoincrement_sequence,
3106+
table_name=table_name_literal,
3107+
),
3108+
"INSERT INTO sqlite_sequence (name, seq) "
3109+
"SELECT {table_name}, {sequence} WHERE NOT EXISTS "
3110+
"(SELECT 1 FROM sqlite_sequence WHERE name = {table_name});".format(
3111+
sequence=autoincrement_sequence,
3112+
table_name=table_name_literal,
3113+
),
3114+
)
3115+
)
30563116
# Re-add existing indexes
30573117
sqls.extend(index_create_sqls)
30583118
return sqls

‎tests/test_create_table_parser.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
Check,
99
ColumnComments,
1010
ParseError,
11+
parse_autoincrement,
1112
parse_checks,
1213
parse_column_comments,
1314
)
@@ -117,6 +118,36 @@ def test_virtual_table_has_no_checks():
117118
)
118119

119120

121+
@pytest.mark.parametrize(
122+
"sql,expected",
123+
[
124+
(
125+
"CREATE TABLE t(id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT)",
126+
"id",
127+
),
128+
(
129+
'CREATE TABLE t("quoted id" INTEGER PRIMARY KEY AUTOINCREMENT)',
130+
"quoted id",
131+
),
132+
(
133+
'CREATE TABLE t("autoincrement" INTEGER PRIMARY KEY, value TEXT)',
134+
None,
135+
),
136+
(
137+
"CREATE TABLE t(id INTEGER PRIMARY KEY /* AUTOINCREMENT */, value TEXT)",
138+
None,
139+
),
140+
(
141+
"CREATE TABLE t(id INTEGER PRIMARY KEY, value TEXT CHECK(value != 'AUTOINCREMENT'))",
142+
None,
143+
),
144+
],
145+
)
146+
def test_parse_autoincrement(sql, expected):
147+
sqlite3.connect(":memory:").execute(sql)
148+
assert parse_autoincrement(sql) == expected
149+
150+
120151
comment_or_space = st.sampled_from(
121152
[
122153
" ",

‎tests/test_transform.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1053,6 +1053,24 @@ def test_transform_with_unique_constraint_implicit_index(fresh_db):
10531053
)
10541054

10551055

1056+
def test_transform_preserves_autoincrement_and_sequence(fresh_db):
1057+
fresh_db.execute(
1058+
"CREATE TABLE entries (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT)"
1059+
)
1060+
entries = fresh_db.table("entries")
1061+
entries.insert_all(({"value": "one"}, {"value": "two"}))
1062+
entries.delete(2)
1063+
1064+
entries.transform(rename={"value": "label"})
1065+
1066+
assert "PRIMARY KEY AUTOINCREMENT" in entries.schema
1067+
entries.insert({"label": "three"})
1068+
assert list(entries.rows) == [
1069+
{"id": 1, "label": "one"},
1070+
{"id": 3, "label": "three"},
1071+
]
1072+
1073+
10561074
def test_transform_preserves_view(fresh_db):
10571075
# https://github.com/simonw/sqlite-utils/issues/831
10581076
dogs = fresh_db.table("dogs")

0 commit comments

Comments
 (0)