Skip to content

Commit 0b71df5

Browse files
committed
Preserve column COLLATE through transforms
table.transform() rebuilt each column from PRAGMA table_xinfo, which does not report collation, so a column declared "name TEXT COLLATE NOCASE" became plain "name TEXT" and comparisons on it turned case-sensitive. Parse column collations from the existing schema with a new parse_column_collations() helper and pass them to create_table_sql() via a private _column_collations argument. Collations follow renamed columns and are dropped with dropped columns. Refs #954
1 parent 9c66baf commit 0b71df5

6 files changed

Lines changed: 132 additions & 4 deletions

File tree

‎docs/python-api.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2015,6 +2015,8 @@ A column-level check is removed if its owning column is dropped. Dropping a colu
20152015
20162016
Comments immediately before or after a column definition are preserved too. They move with that column if it is renamed or reordered, and are removed if the column is dropped. A comment between two column definitions is treated as belonging to the following column.
20172017
2018+
Column collations such as ``name TEXT COLLATE NOCASE`` are preserved as well. They move with the column if it is renamed, and are removed if the column is dropped.
2019+
20182020
.. _python_api_transform_views:
20192021
20202022
Tables referenced by views

‎sqlite_utils/create_table_parser.py‎

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -661,10 +661,7 @@ def _unique_columns(
661661
return tuple(columns), close + 1
662662

663663

664-
def _column_uniques(
665-
item: str, tokens: list[_Token], column: str, base_offset: int
666-
) -> list[Unique]:
667-
uniques: list[Unique] = []
664+
def _column_collation(tokens: list[_Token]) -> str:
668665
collation = ""
669666
collation_index = 1
670667
while collation_index < len(tokens):
@@ -679,6 +676,14 @@ def _column_uniques(
679676
collation_index += 2
680677
continue
681678
collation_index += 1
679+
return collation
680+
681+
682+
def _column_uniques(
683+
item: str, tokens: list[_Token], column: str, base_offset: int
684+
) -> list[Unique]:
685+
uniques: list[Unique] = []
686+
collation = _column_collation(tokens)
682687
pending_name = ""
683688
pending_start: int | None = None
684689
index = 1
@@ -785,6 +790,28 @@ def parse_uniques(create_sql: str) -> list[Unique]:
785790
return uniques
786791

787792

793+
def parse_column_collations(create_sql: str) -> dict[str, str]:
794+
"""Return the COLLATE sequence declared on each column definition."""
795+
body_info = _table_body(create_sql)
796+
if body_info is None:
797+
return {}
798+
body, _ = body_info
799+
collations: dict[str, str] = {}
800+
for item, _, _ in _split_spans(body, _lex(body)):
801+
item_tokens = _meaningful(_lex(item))
802+
if not item_tokens:
803+
continue
804+
head = item_tokens[0]
805+
if (
806+
head.kind == "word" and head.text.upper() in _TABLE_CONSTRAINT_KEYWORDS
807+
) or head.is_keyword("CONSTRAINT"):
808+
continue
809+
collation = _column_collation(item_tokens)
810+
if collation:
811+
collations[_unquote(head.text)] = collation
812+
return collations
813+
814+
788815
def parse_column_comments(create_sql: str) -> dict[str, ColumnComments]:
789816
"""Return comments immediately before and after each column definition."""
790817
body_info = _table_body(create_sql)

‎sqlite_utils/db.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
check_references_identifier,
3636
parse_autoincrement,
3737
parse_checks,
38+
parse_column_collations,
3839
parse_column_comments,
3940
parse_uniques,
4041
rewrite_check_expression,
@@ -1444,6 +1445,7 @@ def create_table_sql(
14441445
_column_comments: Mapping[str, ColumnComments] | None = None,
14451446
_autoincrement: str | None = None,
14461447
_uniques: Iterable[Unique] | None = None,
1448+
_column_collations: Mapping[str, str] | None = None,
14471449
) -> str:
14481450
"""
14491451
Returns the SQL ``CREATE TABLE`` statement for creating the specified table.
@@ -1493,6 +1495,10 @@ def create_table_sql(
14931495
resolve_casing(name, columns): comments
14941496
for name, comments in (_column_comments or {}).items()
14951497
}
1498+
column_collations = {
1499+
resolve_casing(name, columns): collation
1500+
for name, collation in (_column_collations or {}).items()
1501+
}
14961502
checks = list(_checks or ())
14971503
checks_by_column: dict[str, list[Check]] = {}
14981504
table_checks: list[Check] = []
@@ -1609,6 +1615,10 @@ def sort_key(p):
16091615
raise ValueError("AUTOINCREMENT requires a single-column primary key")
16101616
for column_name, column_type in column_items:
16111617
column_extras = []
1618+
if column_name in column_collations:
1619+
column_extras.append(
1620+
f"COLLATE {quote_identifier(column_collations[column_name])}"
1621+
)
16121622
if column_name == single_pk:
16131623
column_extras.append("PRIMARY KEY")
16141624
if column_name == _autoincrement:
@@ -2843,6 +2853,7 @@ def transform_sql(
28432853
try:
28442854
existing_checks = self.checks
28452855
existing_column_comments = parse_column_comments(self.schema)
2856+
existing_column_collations = parse_column_collations(self.schema)
28462857
existing_autoincrement = parse_autoincrement(self.schema)
28472858
existing_uniques = parse_uniques(self.schema)
28482859
except ParseError as ex:
@@ -2908,6 +2919,12 @@ def transform_sql(
29082919
if owner not in drop:
29092920
create_table_column_comments[rename.get(owner) or owner] = comments
29102921

2922+
create_table_column_collations: dict[str, str] = {}
2923+
for column, collation in existing_column_collations.items():
2924+
owner = resolve_casing(column, existing_columns)
2925+
if owner not in drop:
2926+
create_table_column_collations[rename.get(owner) or owner] = collation
2927+
29112928
create_table_foreign_keys: list[ForeignKeyIndicator] = []
29122929

29132930
if foreign_keys is not None:
@@ -3088,6 +3105,7 @@ def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey:
30883105
_column_comments=create_table_column_comments,
30893106
_autoincrement=create_table_autoincrement,
30903107
_uniques=create_table_uniques,
3108+
_column_collations=create_table_column_collations,
30913109
).strip()
30923110
)
30933111

‎tests/test_cli.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2008,6 +2008,18 @@ def test_transform_strict_option_with_invalid_data(db_path):
20082008
assert not any(name.startswith("dogs_new_") for name in db.table_names())
20092009

20102010

2011+
def test_transform_preserves_column_collation(db_path):
2012+
db = Database(db_path)
2013+
db.execute("CREATE TABLE dogs (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)")
2014+
2015+
result = CliRunner().invoke(
2016+
cli.cli, ["transform", db_path, "dogs", "--rename", "name", "dog_name"]
2017+
)
2018+
2019+
assert result.exit_code == 0, result.output
2020+
assert '"dog_name" TEXT COLLATE "NOCASE"' in db.table("dogs").schema
2021+
2022+
20112023
def test_transform_column_to_any(db_path):
20122024
db = Database(db_path)
20132025
if not db.supports_strict:

‎tests/test_create_table_parser.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
UniqueColumn,
1313
parse_autoincrement,
1414
parse_checks,
15+
parse_column_collations,
1516
parse_column_comments,
1617
parse_uniques,
1718
)
@@ -151,6 +152,36 @@ def test_parse_autoincrement(sql, expected):
151152
assert parse_autoincrement(sql) == expected
152153

153154

155+
@pytest.mark.parametrize(
156+
"sql,expected",
157+
[
158+
(
159+
"CREATE TABLE t(id INTEGER, name TEXT COLLATE NOCASE)",
160+
{"name": "NOCASE"},
161+
),
162+
(
163+
'CREATE TABLE t("quoted name" TEXT COLLATE "RTRIM" NOT NULL, other TEXT)',
164+
{"quoted name": "RTRIM"},
165+
),
166+
(
167+
"CREATE TABLE t(code VARCHAR(10) PRIMARY KEY COLLATE BINARY)",
168+
{"code": "BINARY"},
169+
),
170+
(
171+
"CREATE TABLE t(name TEXT CHECK (name = 'x' COLLATE NOCASE))",
172+
{},
173+
),
174+
(
175+
"CREATE TABLE t(name TEXT /* COLLATE NOCASE */, UNIQUE (name COLLATE NOCASE))",
176+
{},
177+
),
178+
],
179+
)
180+
def test_parse_column_collations(sql, expected):
181+
sqlite3.connect(":memory:").execute(sql)
182+
assert parse_column_collations(sql) == expected
183+
184+
154185
def test_parse_column_and_table_uniques():
155186
sql = """
156187
CREATE TABLE memberships (

‎tests/test_transform.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1090,6 +1090,44 @@ def test_transform_preserves_column_unique_collation(fresh_db):
10901090
people.insert({"id": 2, "full_name": "cleo"})
10911091

10921092

1093+
def test_transform_preserves_column_collation(fresh_db):
1094+
fresh_db.execute("""
1095+
CREATE TABLE people (
1096+
id INTEGER PRIMARY KEY,
1097+
name TEXT COLLATE NOCASE NOT NULL,
1098+
code TEXT COLLATE RTRIM,
1099+
age INTEGER
1100+
)
1101+
""")
1102+
people = fresh_db.table("people")
1103+
people.insert({"id": 1, "name": "Cleo", "code": "a", "age": 5})
1104+
1105+
people.transform(types={"age": str})
1106+
1107+
assert '"name" TEXT COLLATE "NOCASE" NOT NULL' in people.schema
1108+
assert '"code" TEXT COLLATE "RTRIM"' in people.schema
1109+
assert people.columns_dict["age"] is str
1110+
assert fresh_db.execute(
1111+
"select count(*) from people where name = 'cleo'"
1112+
).fetchone() == (1,)
1113+
1114+
1115+
def test_transform_collation_follows_renamed_column_and_drop(fresh_db):
1116+
fresh_db.execute("""
1117+
CREATE TABLE people (
1118+
id INTEGER PRIMARY KEY,
1119+
name TEXT COLLATE NOCASE,
1120+
nickname TEXT COLLATE NOCASE
1121+
)
1122+
""")
1123+
people = fresh_db.table("people")
1124+
1125+
people.transform(rename={"name": "full_name"}, drop={"nickname"})
1126+
1127+
assert '"full_name" TEXT COLLATE "NOCASE"' in people.schema
1128+
assert "nickname" not in people.schema
1129+
1130+
10931131
def test_transform_drops_entire_composite_unique_constraint(fresh_db):
10941132
fresh_db.execute("""
10951133
CREATE TABLE memberships (

0 commit comments

Comments
 (0)