Skip to content

Commit 6bc1d33

Browse files
committed
Fix resource leaks caused by buffered readers
Spotted running 'just test -Wall' on the LLM project.
1 parent 85b1be1 commit 6bc1d33

2 files changed

Lines changed: 75 additions & 1 deletion

File tree

‎sqlite_utils/utils.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -380,10 +380,14 @@ class Format(enum.Enum):
380380
"rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO"
381381
)
382382
if not first_bytes:
383+
buffered.close()
383384
return (), Format.CSV
384385
if first_bytes.startswith((b"[", b"{")):
386+
# JSON is read eagerly, so the detection wrapper can close now,
387+
# including when parsing raises an error.
385388
# TODO: Detect newline-JSON
386-
return rows_from_file(buffered, format=Format.JSON)
389+
with buffered:
390+
return rows_from_file(buffered, format=Format.JSON)
387391
else:
388392
dialect = csv.Sniffer().sniff(
389393
first_bytes.decode(encoding or "utf-8-sig", "ignore")

‎tests/test_rows_from_file.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import io
2+
import json
13
from io import BytesIO, StringIO
24

35
import pytest
@@ -61,3 +63,71 @@ def test_rows_from_file_error_on_string_io():
6163
assert ex.value.args == (
6264
"rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO",
6365
)
66+
67+
68+
@pytest.fixture
69+
def buffered_readers(monkeypatch):
70+
# Keep wrappers alive so these checks cannot pass due to garbage collection.
71+
readers = []
72+
original = io.BufferedReader
73+
74+
def buffered_reader(*args, **kwargs):
75+
reader = original(*args, **kwargs)
76+
readers.append(reader)
77+
return reader
78+
79+
monkeypatch.setattr(io, "BufferedReader", buffered_reader)
80+
yield readers
81+
for reader in readers:
82+
reader.close()
83+
84+
85+
@pytest.mark.parametrize(
86+
"content, expected_format, expected_rows",
87+
[
88+
(b'[{"id": 1}]', Format.JSON, [{"id": 1}]),
89+
(b'{"id": 1}', Format.JSON, [{"id": 1}]),
90+
(b"[]", Format.JSON, []),
91+
(b"", Format.CSV, []),
92+
(b" \n\t", Format.CSV, []),
93+
],
94+
)
95+
def test_detect_format_closes_eager_reader(
96+
tmp_path, buffered_readers, content, expected_format, expected_rows
97+
):
98+
path = tmp_path / "input"
99+
path.write_bytes(content)
100+
with path.open("rb") as fp:
101+
rows, detected = rows_from_file(fp)
102+
assert detected == expected_format
103+
assert list(rows) == expected_rows
104+
assert len(buffered_readers) == 1
105+
assert buffered_readers[0].closed
106+
107+
108+
@pytest.mark.parametrize("content", [b"[", b'{"id":'])
109+
def test_detect_format_closes_reader_on_invalid_json(
110+
tmp_path, buffered_readers, content
111+
):
112+
path = tmp_path / "input.json"
113+
path.write_bytes(content)
114+
with path.open("rb") as fp:
115+
with pytest.raises(json.JSONDecodeError):
116+
rows_from_file(fp)
117+
assert len(buffered_readers) == 1
118+
assert buffered_readers[0].closed
119+
120+
121+
@pytest.mark.parametrize(
122+
"delimiter, expected_format", [(b",", Format.CSV), (b"\t", Format.TSV)]
123+
)
124+
def test_detect_format_keeps_streaming_reader_open(
125+
buffered_readers, delimiter, expected_format
126+
):
127+
content = b"id" + delimiter + b"name\n" + (b"1" + delimiter + b"Cleo\n") * 2000
128+
rows, detected = rows_from_file(BytesIO(content))
129+
assert detected == expected_format
130+
assert len(buffered_readers) == 1
131+
assert not buffered_readers[0].closed
132+
assert len(list(rows)) == 2000
133+
assert buffered_readers[0].closed

0 commit comments

Comments
 (0)