Skip to content

Commit e4935e0

Browse files
authored
transform: coerce empty strings to NULL when converting TEXT columns to numeric types (#805)
* transform: coerce empty strings to NULL when converting TEXT columns to numeric types When a TEXT column is transformed to INTEGER, FLOAT, or REAL and a row contains an empty string, the empty string is now converted to NULL during the INSERT...SELECT copy, matching the expected behavior described in #488. Fixes #488
1 parent 75ba588 commit e4935e0

5 files changed

Lines changed: 57 additions & 5 deletions

File tree

‎docs/changelog.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ Unreleased
2525
- ``sqlite-utils convert --dry-run`` now works for table and column names containing closing square brackets. (:issue:`829`)
2626
- ``table.indexes`` and ``table.xindexes`` now work for table, index and column names containing double quotes. This also fixes ``table.transform()`` for tables with those identifiers. Thanks, `nyxst4ck <https://github.com/nyxst4ck>`__. (:issue:`824`, `#825 <https://github.com/simonw/sqlite-utils/pull/825>`__)
2727
- Improved type annotations throughout the package and added Pyright regression checks to CI. (:issue:`833`)
28+
- Changing a ``TEXT`` column to ``INTEGER``, ``FLOAT`` or ``REAL`` using ``table.transform()`` or ``sqlite-utils transform`` now converts exact empty strings to ``NULL``. Previously they remained empty strings in the numeric column. Thanks, `ikatyal2110 <https://github.com/ikatyal2110>`__. (:issue:`488`, `#805 <https://github.com/simonw/sqlite-utils/pull/805>`__)
2829

2930
.. _v3_39_1:
3031

‎docs/cli.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2236,7 +2236,7 @@ The ``transform`` command allows you to apply complex transformations to a table
22362236
Every option for this table (with the exception of ``--pk-none``) can be specified multiple times. The options are as follows:
22372237

22382238
``--type column-name new-type``
2239-
Change the type of the specified column. Valid types are ``integer``, ``text``, ``float``, ``real``, ``blob`` and ``any``.
2239+
Change the type of the specified column. Valid types are ``integer``, ``text``, ``float``, ``real``, ``blob`` and ``any``. Changing a ``TEXT`` column to ``INTEGER``, ``FLOAT`` or ``REAL`` converts exact empty-string values to ``NULL``.
22402240

22412241
``--drop column-name``
22422242
Drop the specified column.

‎docs/python-api.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1826,6 +1826,8 @@ To alter the type of a column, use the ``types=`` argument:
18261826
# Convert the 'age' column to an integer, and 'weight' to a float
18271827
table.transform(types={"age": int, "weight": float})
18281828
1829+
When a ``TEXT`` column is changed to ``INTEGER``, ``FLOAT`` or ``REAL``, exact empty-string values are stored as ``NULL``. Other values, including whitespace-only strings, are copied normally.
1830+
18291831
See :ref:`python_api_add_column` for a list of available types.
18301832

18311833
.. _python_api_transform_strict:

‎sqlite_utils/db.py‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3093,6 +3093,19 @@ def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey:
30933093
).strip()
30943094
)
30953095

3096+
# Columns being changed from TEXT to a numeric type: coerce empty strings to NULL
3097+
_numeric_sql_types = {"INTEGER", "REAL", "FLOAT", "NUMERIC"}
3098+
text_to_numeric_cols = {
3099+
col_name
3100+
for col_name, new_type in types.items()
3101+
if existing_columns.get(col_name) == str
3102+
and COLUMN_TYPE_MAPPING.get(
3103+
new_type,
3104+
new_type.upper() if isinstance(new_type, str) else "",
3105+
)
3106+
in _numeric_sql_types
3107+
}
3108+
30963109
# Copy across data, respecting any renamed columns
30973110
new_cols = []
30983111
old_cols = []
@@ -3103,10 +3116,16 @@ def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey:
31033116
if "rowid" not in new_cols:
31043117
new_cols.insert(0, "rowid")
31053118
old_cols.insert(0, "rowid")
3119+
3120+
def _copy_expr(col):
3121+
if col in text_to_numeric_cols:
3122+
return "NULLIF({}, '')".format(quote_identifier(col))
3123+
return quote_identifier(col)
3124+
31063125
copy_sql = "INSERT INTO {} ({new_cols})\n SELECT {old_cols} FROM {};".format(
31073126
quote_identifier(new_table_name),
31083127
quote_identifier(self.name),
3109-
old_cols=", ".join(quote_identifier(col) for col in old_cols),
3128+
old_cols=", ".join(_copy_expr(col) for col in old_cols),
31103129
new_cols=", ".join(quote_identifier(col) for col in new_cols),
31113130
)
31123131
sqls.append(copy_sql)

‎tests/test_transform.py‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
{"types": {"age": int}},
2828
[
2929
'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" INTEGER\n);',
30-
'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";',
30+
'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";',
3131
'DROP TABLE "dogs";',
3232
"PRAGMA legacy_alter_table=ON;",
3333
'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
@@ -63,7 +63,7 @@
6363
{"types": {"age": int}, "rename": {"age": "dog_age"}},
6464
[
6565
'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "dog_age" INTEGER\n);',
66-
'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";',
66+
'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";',
6767
'DROP TABLE "dogs";',
6868
"PRAGMA legacy_alter_table=ON;",
6969
'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
@@ -168,7 +168,7 @@ def tracer(sql, params):
168168
{"types": {"age": int}},
169169
[
170170
'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" INTEGER\n);',
171-
'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";',
171+
'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";',
172172
'DROP TABLE "dogs";',
173173
"PRAGMA legacy_alter_table=ON;",
174174
'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
@@ -1125,6 +1125,36 @@ def test_transform_preserves_autoincrement_and_sequence(fresh_db):
11251125
]
11261126

11271127

1128+
@pytest.mark.parametrize(
1129+
"new_type,expected_value,expected_type",
1130+
[
1131+
(int, 42, int),
1132+
(float, 42.0, float),
1133+
("integer", 42, int),
1134+
("float", 42.0, float),
1135+
("REAL", 42.0, float),
1136+
],
1137+
)
1138+
def test_transform_empty_string_to_null_for_numeric_types(
1139+
fresh_db, new_type, expected_value, expected_type
1140+
):
1141+
fresh_db["test"].insert_all(
1142+
[
1143+
{"id": 1, "value": "42"},
1144+
{"id": 2, "value": ""},
1145+
{"id": 3, "value": None},
1146+
{"id": 4, "value": " "},
1147+
]
1148+
)
1149+
fresh_db["test"].transform(types={"value": new_type})
1150+
rows = {r["id"]: r["value"] for r in fresh_db["test"].rows}
1151+
assert rows[1] == expected_value
1152+
assert type(rows[1]) is expected_type
1153+
assert rows[2] is None
1154+
assert rows[3] is None
1155+
assert rows[4] == " "
1156+
1157+
11281158
def test_transform_preserves_view(fresh_db):
11291159
# https://github.com/simonw/sqlite-utils/issues/831
11301160
dogs = fresh_db.table("dogs")

0 commit comments

Comments
 (0)