Skip to content

Commit c270922

Browse files
committed
Preserve carriage returns in quoted CSV/TSV fields (#895)
When reading CSV or TSV files through rows_from_file() or the CLI insert command, io.TextIOWrapper defaulted to universal-newline mode (newline=None), causing carriage returns (\r\n or lone \r) embedded inside quoted fields to be silently translated to \n before csv.reader could process them. Setting newline='' in TextIOWrapper preserves carriage returns in quoted fields according to the Python standard library csv module specification. Fixes #895
1 parent 6bc1d33 commit c270922

4 files changed

Lines changed: 49 additions & 3 deletions

File tree

‎sqlite_utils/cli.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1226,11 +1226,14 @@ def _insert_docs(docs, tracker=None):
12261226
# The --sniff option needs us to buffer the file to peek ahead
12271227
sniff_buffer = None
12281228
decoded_buffer = None
1229+
newline = "" if (csv or tsv) else None
12291230
if sniff:
12301231
sniff_buffer = io.BufferedReader(file, buffer_size=4096)
1231-
decoded_buffer = io.TextIOWrapper(sniff_buffer, encoding=encoding)
1232+
decoded_buffer = io.TextIOWrapper(
1233+
sniff_buffer, encoding=encoding, newline=newline
1234+
)
12321235
else:
1233-
decoded_buffer = io.TextIOWrapper(file, encoding=encoding)
1236+
decoded_buffer = io.TextIOWrapper(file, encoding=encoding, newline=newline)
12341237

12351238
tracker = None
12361239
with file_progress(decoded_buffer, silent=silent) as decoded:

‎sqlite_utils/utils.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ class Format(enum.Enum):
346346
return (json.loads(line) for line in fp if line.strip()), Format.NL
347347
elif format == Format.CSV:
348348
use_encoding: str = encoding or "utf-8-sig"
349-
decoded_fp = io.TextIOWrapper(fp, encoding=use_encoding)
349+
decoded_fp = io.TextIOWrapper(fp, encoding=use_encoding, newline="")
350350
if dialect is not None:
351351
reader = csv.DictReader(decoded_fp, dialect=dialect)
352352
else:

‎tests/test_cli_insert.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,21 @@ def test_insert_csv_tsv(content, options, db_path, tmpdir):
235235
assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db.table("data").rows)
236236

237237

238+
@pytest.mark.parametrize("fmt_opt", (["--csv"], ["--tsv"]))
239+
def test_insert_csv_tsv_preserves_carriage_returns_in_quoted_fields(fmt_opt, db_path):
240+
delimiter = "\t" if "--tsv" in fmt_opt else ","
241+
content = f'id{delimiter}v\r\n1{delimiter}"line1\r\nline2"\r\n'
242+
result = CliRunner().invoke(
243+
cli.cli,
244+
["insert", db_path, "data", "-"] + fmt_opt + ["--no-detect-types"],
245+
catch_exceptions=False,
246+
input=content.encode("utf-8"),
247+
)
248+
assert result.exit_code == 0
249+
db = Database(db_path)
250+
assert list(db.table("data").rows) == [{"id": "1", "v": "line1\r\nline2"}]
251+
252+
238253
@pytest.mark.parametrize("empty_null", (True, False))
239254
def test_insert_csv_empty_null(db_path, empty_null):
240255
options = ["--csv", "--no-detect-types"]

‎tests/test_rows_from_file.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,3 +131,31 @@ def test_detect_format_keeps_streaming_reader_open(
131131
assert not buffered_readers[0].closed
132132
assert len(list(rows)) == 2000
133133
assert buffered_readers[0].closed
134+
135+
136+
@pytest.mark.parametrize(
137+
"file_bytes,expected_format,expected_rows",
138+
[
139+
(
140+
b'id,v\r\n1,"line1\r\nline2"\r\n',
141+
Format.CSV,
142+
[{"id": "1", "v": "line1\r\nline2"}],
143+
),
144+
(
145+
b'id\tv\r\n1\t"tab1\r\ntab2"\r\n',
146+
Format.TSV,
147+
[{"id": "1", "v": "tab1\r\ntab2"}],
148+
),
149+
(
150+
b'id,v\r\n1,"isolated\rreturn"\r\n',
151+
Format.CSV,
152+
[{"id": "1", "v": "isolated\rreturn"}],
153+
),
154+
],
155+
)
156+
def test_rows_from_file_preserves_carriage_returns_in_quoted_fields(
157+
file_bytes, expected_format, expected_rows
158+
):
159+
rows, detected_format = rows_from_file(BytesIO(file_bytes), format=expected_format)
160+
assert detected_format == expected_format
161+
assert list(rows) == expected_rows

0 commit comments

Comments
 (0)