Skip to content

Commit fde5e0e

Browse files
committed
Fix upsert() raising NOT NULL for omitted column on existing row, fixes #878
1 parent 6bc1d33 commit fde5e0e

2 files changed

Lines changed: 139 additions & 1 deletion

File tree

‎sqlite_utils/db.py‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4460,8 +4460,59 @@ def build_insert_queries_and_params(
44604460
# All columns are in the PK – nothing to update.
44614461
do_clause = "DO NOTHING"
44624462

4463+
# Detect omitted columns with NOT NULL constraints and no defaults.
4464+
# SQLite validates NOT NULL constraints during the INSERT evaluation
4465+
# before ON CONFLICT triggers. Supplying a correlated scalar subquery
4466+
# pulls the existing value so the constraint passes on existing rows,
4467+
# while still failing with IntegrityError if the row does not exist.
4468+
omitted_not_null_cols: list[str] = []
4469+
if self.exists():
4470+
lower_all_columns = {c.lower() for c in all_columns}
4471+
for col in self.columns:
4472+
if col.notnull and col.default_value is None and not col.is_pk:
4473+
if col.name.lower() not in lower_all_columns:
4474+
omitted_not_null_cols.append(col.name)
4475+
if not_null:
4476+
lower_all_columns = {c.lower() for c in all_columns}
4477+
lower_omitted = {c.lower() for c in omitted_not_null_cols}
4478+
for nn in not_null:
4479+
if (
4480+
nn.lower() not in lower_all_columns
4481+
and nn.lower() not in lower_omitted
4482+
):
4483+
omitted_not_null_cols.append(nn)
4484+
4485+
if omitted_not_null_cols:
4486+
all_insert_cols = list(all_columns) + omitted_not_null_cols
4487+
insert_columns_sql = ", ".join(
4488+
quote_identifier(c) for c in all_insert_cols
4489+
)
4490+
where_pk_sql = " AND ".join(
4491+
f"{quote_identifier(pk)} = ?" for pk in pk_cols
4492+
)
4493+
subqueries = [
4494+
f"(SELECT {quote_identifier(c)} FROM {quote_identifier(self.name)} WHERE {where_pk_sql})"
4495+
for c in omitted_not_null_cols
4496+
]
4497+
row_placeholder_parts = [
4498+
conversions.get(c, "?") for c in all_columns
4499+
] + subqueries
4500+
single_row_placeholder = f"({', '.join(row_placeholder_parts)})"
4501+
row_placeholders_sql = ", ".join(single_row_placeholder for _ in values)
4502+
4503+
pk_indexes = [all_columns.index(c) for c in pk_cols]
4504+
all_flat_params: list[Any] = []
4505+
for record_values in values:
4506+
all_flat_params.extend(record_values)
4507+
row_pk_params = [record_values[i] for i in pk_indexes]
4508+
for _ in omitted_not_null_cols:
4509+
all_flat_params.extend(row_pk_params)
4510+
flat_params = all_flat_params
4511+
else:
4512+
insert_columns_sql = columns_sql
4513+
44634514
sql = (
4464-
f"INSERT INTO {quote_identifier(self.name)} ({columns_sql}) "
4515+
f"INSERT INTO {quote_identifier(self.name)} ({insert_columns_sql}) "
44654516
f"VALUES {row_placeholders_sql} "
44664517
f"ON CONFLICT({conflict_sql}) {do_clause}"
44674518
)

‎tests/test_upsert.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,3 +185,90 @@ def test_upsert_compound_primary_key(fresh_db):
185185
# .upsert_all() with a single item should set .last_pk
186186
table.upsert_all([{"species": "cat", "id": 1, "age": 5}], pk=("species", "id"))
187187
assert ("cat", 1) == table.last_pk
188+
189+
190+
@pytest.mark.parametrize("use_old_upsert", (False, True))
191+
def test_upsert_omitted_not_null_column_on_existing_row(use_old_upsert):
192+
# https://github.com/simonw/sqlite-utils/issues/878
193+
db = Database(memory=True, use_old_upsert=use_old_upsert)
194+
table = db.table("dogs")
195+
table.insert(
196+
{"id": 1, "name": "Cleo", "color": "brown"},
197+
pk="id",
198+
not_null={"name"},
199+
)
200+
201+
# Upserting with omitted NOT NULL column on an existing row must succeed
202+
table.upsert({"id": 1, "color": "black"})
203+
assert table.get(1) == {"id": 1, "name": "Cleo", "color": "black"}
204+
205+
206+
@pytest.mark.parametrize("use_old_upsert", (False, True))
207+
@pytest.mark.parametrize("batch_size", (1, 2))
208+
def test_upsert_all_omitted_not_null_compound_pk(use_old_upsert, batch_size):
209+
db = Database(memory=True, use_old_upsert=use_old_upsert)
210+
table = db.table("pets")
211+
table.insert_all(
212+
[
213+
{"species": "dog", "id": 1, "name": "Cleo", "breed": "Lab", "age": 4},
214+
{"species": "cat", "id": 1, "name": "Nixie", "breed": "Siamese", "age": 5},
215+
],
216+
pk=("species", "id"),
217+
not_null={"name", "breed"},
218+
)
219+
220+
table.upsert_all(
221+
[
222+
{"species": "dog", "id": 1, "age": 5},
223+
{"species": "cat", "id": 1, "age": 6},
224+
],
225+
batch_size=batch_size,
226+
)
227+
228+
assert table.get(("dog", 1)) == {
229+
"species": "dog",
230+
"id": 1,
231+
"name": "Cleo",
232+
"breed": "Lab",
233+
"age": 5,
234+
}
235+
assert table.get(("cat", 1)) == {
236+
"species": "cat",
237+
"id": 1,
238+
"name": "Nixie",
239+
"breed": "Siamese",
240+
"age": 6,
241+
}
242+
243+
244+
@pytest.mark.parametrize("use_old_upsert", (False, True))
245+
def test_upsert_omitted_not_null_with_conversions_and_explicit_not_null(use_old_upsert):
246+
db = Database(memory=True, use_old_upsert=use_old_upsert)
247+
table = db.table("dogs")
248+
table.insert(
249+
{"id": 1, "name": "Cleo", "color": "brown"},
250+
pk="id",
251+
)
252+
# Upsert with explicit not_null argument and conversion
253+
table.upsert(
254+
{"id": 1, "color": "black"},
255+
not_null=["name"],
256+
conversions={"color": "upper(?)"},
257+
)
258+
assert table.get(1) == {"id": 1, "name": "Cleo", "color": "BLACK"}
259+
260+
261+
@pytest.mark.parametrize("use_old_upsert", (False, True))
262+
def test_upsert_omitted_not_null_with_check_constraint(use_old_upsert):
263+
db = Database(memory=True, use_old_upsert=use_old_upsert)
264+
db.execute("""
265+
CREATE TABLE dogs (
266+
id INTEGER PRIMARY KEY,
267+
name TEXT NOT NULL CHECK(length(name) >= 3),
268+
color TEXT
269+
)
270+
""")
271+
table = db.table("dogs")
272+
table.insert({"id": 1, "name": "Cleo", "color": "brown"})
273+
table.upsert({"id": 1, "color": "black"})
274+
assert table.get(1) == {"id": 1, "name": "Cleo", "color": "black"}

0 commit comments

Comments
 (0)