Repository navigation
_NumcodecsArrayArrayCodec._encode_sync does not reshape the data on encode #4553
Copy link
Copy link
Labels
bugPotential issues with the zarr-python libraryPotential issues with the zarr-python library
Description
Zarr version
3.4.2.dev22+g957a5f68
Numcodecs version
v0.17.0
Python Version
3.14
Operating System
Linux
Installation
using uv pip in a virtual environment
Description
I was trying to run some custom filter codec on my data before compressing the data with jpegxr, when Claude complained that zarr-python had a bug:
A zarr-python bug I hit along the way: zarr's wrapper for numcodecs filters reshapes the data on decode but not on encode. Filters like FixedScaleOffset and Delta output a flat array, so the image serializer receives a 1 × N line instead of a 2D tile.
- With JPEG-XR the data is lost: it read back as zeros.
- With JPEG-XL it stays lossless but compresses badly.
Steps to reproduce
❯ cat zarr_numcodecs_filter_shape_bug.py
"""Reproduce: zarr-python numcodecs filters pass a flattened chunk to the next codec.
`_NumcodecsArrayArrayCodec._encode_sync` returns the numcodecs output as is.
numcodecs filters (Delta, FixedScaleOffset, ...) return a flattened 1D array,
so the next codec in the chain gets a 1D chunk instead of the chunk shape.
(`_decode_sync` reshapes to the chunk shape, so only the encode side is wrong.)
1. With a TransposeCodec after the filter, writing fails.
2. With a 2D image serializer (imagecodecs JPEG-XR), the encoder receives a
1 x N "image": depending on the data, writing fails or data is silently lost.
"""
import numpy as np
import zarr
from zarr.codecs import TransposeCodec
from zarr.codecs.numcodecs import Delta, FixedScaleOffset
print("zarr", zarr.__version__)
data = np.arange(4 * 6, dtype="uint16").reshape(4, 6)
# 1. numcodecs filter followed by a transpose: the transpose receives a 1D chunk.
try:
a = zarr.create_array(
{},
shape=data.shape,
dtype=data.dtype,
chunks=data.shape,
filters=[Delta(dtype="<u2"), TransposeCodec(order=(1, 0))],
)
a[:] = data
print("1. Delta + TransposeCodec: OK, round trip equal:", np.array_equal(a[:], data))
except Exception as e:
print(f"1. Delta + TransposeCodec: FAILED: {type(e).__name__}: {e}")
# 2. numcodecs filter followed by a 2D image serializer: the encoder gets a
# 1 x N image, so writing fails or the data is lost.
try:
import imagecodecs.zarr
from imagecodecs.zarr import Jpegxr
except ImportError:
print("2. skipped (needs imagecodecs)")
else:
imagecodecs.zarr.register_codecs()
image = np.random.default_rng(0).integers(100, 3000, (64, 64)).astype("uint16") * 4
filters = {
"Delta": Delta(dtype="<u2"),
"FixedScaleOffset": FixedScaleOffset(offset=0, scale=0.25, dtype="<u2", astype="<u2"),
}
for name, codec in filters.items():
try:
a = zarr.create_array(
{},
shape=image.shape,
dtype=image.dtype,
chunks=image.shape,
filters=[codec],
serializer=Jpegxr(level=1.0), # lossless
compressors=None,
)
a[:] = image
print(
f"2. {name} + JPEG-XR (lossless): round trip equal:",
np.array_equal(a[:], image),
"| first values:", image.ravel()[:4], "->", a[:].ravel()[:4],
)
except Exception as e:
print(f"2. {name} + JPEG-XR (lossless): FAILED: {type(e).__name__}: {e}")Additional output
# Original zarr-python.
$ python zarr_numcodecs_filter_shape_bug.py
zarr 3.4.1
1. Delta + TransposeCodec: FAILED: ValueError: axes don't match array
2. Delta + JPEG-XR (lossless): FAILED: JpegxrError: PKImageEncode_WritePixels returned WMP_errFail
2. FixedScaleOffset + JPEG-XR (lossless): FAILED: JpegxrError: PKImageEncode_WritePixels returned WMP_errFail
# patched zarr-python.
$ python zarr_numcodecs_filter_shape_bug.py
zarr 3.4.2.dev22+g957a5f68
1. Delta + TransposeCodec: OK, round trip equal: True
2. Delta + JPEG-XR (lossless): round trip equal: True | first values: [10264 7788 6328 3528] -> [10264 7788 6328 3528]
2. FixedScaleOffset + JPEG-XR (lossless): round trip equal: True | first values: [10264 7788 6328 3528] -> [10264 7788 6328 3528]
Claude's zarr-python patch:
diff --git a/src/zarr/codecs/numcodecs/_codecs.py b/src/zarr/codecs/numcodecs/_codecs.py
index c9807431..12b7cd4a 100644
--- a/src/zarr/codecs/numcodecs/_codecs.py
+++ b/src/zarr/codecs/numcodecs/_codecs.py
@@ -164,7 +164,11 @@ class _NumcodecsArrayArrayCodec(_NumcodecsCodec, ArrayArrayCodec):
# would be encoded in transposed element order (gh-3558)
chunk_ndarray = np.ascontiguousarray(chunk_data.as_ndarray_like())
out = self._codec.encode(chunk_ndarray)
- return chunk_spec.prototype.nd_buffer.from_ndarray_like(out)
+ # numcodecs filters return a flattened array; restore the shape this
+ # codec declares, so that the next codec (e.g. an image serializer)
+ # receives a chunk of the shape given by `resolve_metadata`
+ out_shape = self.resolve_metadata(chunk_spec).shape
+ return chunk_spec.prototype.nd_buffer.from_ndarray_like(out.reshape(out_shape))
async def _encode_single(self, chunk_data: NDBuffer, chunk_spec: ArraySpec) -> NDBuffer:
return await asyncio.to_thread(self._encode_sync, chunk_data, chunk_spec)
diff --git a/tests/test_codecs/test_numcodecs.py b/tests/test_codecs/test_numcodecs.py
index d78b73f3..902b5294 100644
--- a/tests/test_codecs/test_numcodecs.py
+++ b/tests/test_codecs/test_numcodecs.py
@@ -11,6 +11,9 @@ from numcodecs import GZip
from zarr import config, create_array, open_array
from zarr.abc.numcodec import _is_numcodec, _is_numcodec_cls
from zarr.codecs import numcodecs as _numcodecs
+from zarr.core.array_spec import ArrayConfig, ArraySpec
+from zarr.core.buffer import default_buffer_prototype
+from zarr.core.dtype import get_data_type_from_native_dtype
from zarr.errors import UnknownCodecError
from zarr.registry import get_codec_class, get_numcodec
@@ -192,6 +195,44 @@ def test_generic_filter_f_contiguous(
np.testing.assert_array_equal(data, a[:, :])
+@pytest.mark.parametrize(
+ ("codec", "dtype"),
+ [
+ (_numcodecs.Delta(dtype="uint16"), "uint16"),
+ (_numcodecs.FixedScaleOffset(offset=0, scale=0.25, dtype="<u2", astype="<u2"), "uint16"),
+ (_numcodecs.AsType(encode_dtype="float64", decode_dtype="float32"), "float32"),
+ (_numcodecs.BitRound(keepbits=10), "float32"),
+ (_numcodecs.Quantize(digits=3, dtype="float32"), "float32"),
+ (_numcodecs.PackBits(), "bool"),
+ ],
+ ids=["delta", "fixedscaleoffset", "astype", "bitround", "quantize", "packbits"],
+)
+def test_generic_filter_encoded_shape(
+ codec: _numcodecs._NumcodecsArrayArrayCodec, dtype: str
+) -> None:
+ # numcodecs filters return flattened arrays; the encoded chunk must have the
+ # shape declared by `resolve_metadata`, or the next codec (e.g. an image
+ # serializer) receives a 1D chunk
+ if dtype == "bool":
+ data = np.tril(np.ones((16, 32), dtype=bool))
+ else:
+ data = (np.arange(16 * 32).reshape(16, 32) * 4).astype(dtype)
+ zdtype = get_data_type_from_native_dtype(data.dtype)
+ spec = ArraySpec(
+ shape=data.shape,
+ dtype=zdtype,
+ fill_value=zdtype.cast_scalar(0),
+ config=ArrayConfig(order="C", write_empty_chunks=True),
+ prototype=default_buffer_prototype(),
+ )
+ nd_buf = default_buffer_prototype().nd_buffer.from_numpy_array(data)
+
+ encoded = codec._encode_sync(nd_buf, spec)
+ assert encoded.shape == codec.resolve_metadata(spec).shape
+ decoded = codec._decode_sync(encoded, spec)
+ assert decoded.shape == data.shape
+
+
def test_generic_filter_bitround() -> None:
data = np.linspace(0, 1, 256, dtype="float32").reshape((16, 16))Reactions are currently unavailable
Metadata
Metadata
Assignees
Labels
bugPotential issues with the zarr-python libraryPotential issues with the zarr-python library
Activity
chore: add release notes fragment for zarr-developersgh-4553
chore: add release notes fragment for zarr-developersgh-4553
d-v-b commentedon Oct 29, 2026
thanks for this report! we will get the fix in for the next release