Skip to content

_NumcodecsArrayArrayCodec._encode_sync does not reshape the data on encode #4553

Description

@ghuls

Zarr version

3.4.2.dev22+g957a5f68

Numcodecs version

v0.17.0

Python Version

3.14

Operating System

Linux

Installation

using uv pip in a virtual environment

Description

I was trying to run some custom filter codec on my data before compressing the data with jpegxr, when Claude complained that zarr-python had a bug:

A zarr-python bug I hit along the way: zarr's wrapper for numcodecs filters reshapes the data on decode but not on encode. Filters like FixedScaleOffset and Delta output a flat array, so the image serializer receives a 1 × N line instead of a 2D tile.

  • With JPEG-XR the data is lost: it read back as zeros.
  • With JPEG-XL it stays lossless but compresses badly.

Steps to reproduce

❯ cat zarr_numcodecs_filter_shape_bug.py
"""Reproduce: zarr-python numcodecs filters pass a flattened chunk to the next codec.

`_NumcodecsArrayArrayCodec._encode_sync` returns the numcodecs output as is.
numcodecs filters (Delta, FixedScaleOffset, ...) return a flattened 1D array,
so the next codec in the chain gets a 1D chunk instead of the chunk shape.
(`_decode_sync` reshapes to the chunk shape, so only the encode side is wrong.)

1. With a TransposeCodec after the filter, writing fails.
2. With a 2D image serializer (imagecodecs JPEG-XR), the encoder receives a
   1 x N "image": depending on the data, writing fails or data is silently lost.
"""

import numpy as np
import zarr
from zarr.codecs import TransposeCodec
from zarr.codecs.numcodecs import Delta, FixedScaleOffset

print("zarr", zarr.__version__)
data = np.arange(4 * 6, dtype="uint16").reshape(4, 6)

# 1. numcodecs filter followed by a transpose: the transpose receives a 1D chunk.
try:
    a = zarr.create_array(
        {},
        shape=data.shape,
        dtype=data.dtype,
        chunks=data.shape,
        filters=[Delta(dtype="<u2"), TransposeCodec(order=(1, 0))],
    )
    a[:] = data
    print("1. Delta + TransposeCodec: OK, round trip equal:", np.array_equal(a[:], data))
except Exception as e:
    print(f"1. Delta + TransposeCodec: FAILED: {type(e).__name__}: {e}")

# 2. numcodecs filter followed by a 2D image serializer: the encoder gets a
#    1 x N image, so writing fails or the data is lost.
try:
    import imagecodecs.zarr
    from imagecodecs.zarr import Jpegxr
except ImportError:
    print("2. skipped (needs imagecodecs)")
else:
    imagecodecs.zarr.register_codecs()
    image = np.random.default_rng(0).integers(100, 3000, (64, 64)).astype("uint16") * 4
    filters = {
        "Delta": Delta(dtype="<u2"),
        "FixedScaleOffset": FixedScaleOffset(offset=0, scale=0.25, dtype="<u2", astype="<u2"),
    }
    for name, codec in filters.items():
        try:
            a = zarr.create_array(
                {},
                shape=image.shape,
                dtype=image.dtype,
                chunks=image.shape,
                filters=[codec],
                serializer=Jpegxr(level=1.0),  # lossless
                compressors=None,
            )
            a[:] = image
            print(
                f"2. {name} + JPEG-XR (lossless): round trip equal:",
                np.array_equal(a[:], image),
                "| first values:", image.ravel()[:4], "->", a[:].ravel()[:4],
            )
        except Exception as e:
            print(f"2. {name} + JPEG-XR (lossless): FAILED: {type(e).__name__}: {e}")

Additional output

# Original zarr-python.
$ python zarr_numcodecs_filter_shape_bug.py
zarr 3.4.1
1. Delta + TransposeCodec: FAILED: ValueError: axes don't match array
2. Delta + JPEG-XR (lossless): FAILED: JpegxrError: PKImageEncode_WritePixels returned WMP_errFail
2. FixedScaleOffset + JPEG-XR (lossless): FAILED: JpegxrError: PKImageEncode_WritePixels returned WMP_errFail

# patched zarr-python.
$ python zarr_numcodecs_filter_shape_bug.py
zarr 3.4.2.dev22+g957a5f68
1. Delta + TransposeCodec: OK, round trip equal: True
2. Delta + JPEG-XR (lossless): round trip equal: True | first values: [10264  7788  6328  3528] -> [10264  7788  6328  3528]
2. FixedScaleOffset + JPEG-XR (lossless): round trip equal: True | first values: [10264  7788  6328  3528] -> [10264  7788  6328  3528]

Claude's zarr-python patch:

diff --git a/src/zarr/codecs/numcodecs/_codecs.py b/src/zarr/codecs/numcodecs/_codecs.py
index c9807431..12b7cd4a 100644
--- a/src/zarr/codecs/numcodecs/_codecs.py
+++ b/src/zarr/codecs/numcodecs/_codecs.py
@@ -164,7 +164,11 @@ class _NumcodecsArrayArrayCodec(_NumcodecsCodec, ArrayArrayCodec):
         # would be encoded in transposed element order (gh-3558)
         chunk_ndarray = np.ascontiguousarray(chunk_data.as_ndarray_like())
         out = self._codec.encode(chunk_ndarray)
-        return chunk_spec.prototype.nd_buffer.from_ndarray_like(out)
+        # numcodecs filters return a flattened array; restore the shape this
+        # codec declares, so that the next codec (e.g. an image serializer)
+        # receives a chunk of the shape given by `resolve_metadata`
+        out_shape = self.resolve_metadata(chunk_spec).shape
+        return chunk_spec.prototype.nd_buffer.from_ndarray_like(out.reshape(out_shape))

     async def _encode_single(self, chunk_data: NDBuffer, chunk_spec: ArraySpec) -> NDBuffer:
         return await asyncio.to_thread(self._encode_sync, chunk_data, chunk_spec)
diff --git a/tests/test_codecs/test_numcodecs.py b/tests/test_codecs/test_numcodecs.py
index d78b73f3..902b5294 100644
--- a/tests/test_codecs/test_numcodecs.py
+++ b/tests/test_codecs/test_numcodecs.py
@@ -11,6 +11,9 @@ from numcodecs import GZip
 from zarr import config, create_array, open_array
 from zarr.abc.numcodec import _is_numcodec, _is_numcodec_cls
 from zarr.codecs import numcodecs as _numcodecs
+from zarr.core.array_spec import ArrayConfig, ArraySpec
+from zarr.core.buffer import default_buffer_prototype
+from zarr.core.dtype import get_data_type_from_native_dtype
 from zarr.errors import UnknownCodecError
 from zarr.registry import get_codec_class, get_numcodec

@@ -192,6 +195,44 @@ def test_generic_filter_f_contiguous(
     np.testing.assert_array_equal(data, a[:, :])


+@pytest.mark.parametrize(
+    ("codec", "dtype"),
+    [
+        (_numcodecs.Delta(dtype="uint16"), "uint16"),
+        (_numcodecs.FixedScaleOffset(offset=0, scale=0.25, dtype="<u2", astype="<u2"), "uint16"),
+        (_numcodecs.AsType(encode_dtype="float64", decode_dtype="float32"), "float32"),
+        (_numcodecs.BitRound(keepbits=10), "float32"),
+        (_numcodecs.Quantize(digits=3, dtype="float32"), "float32"),
+        (_numcodecs.PackBits(), "bool"),
+    ],
+    ids=["delta", "fixedscaleoffset", "astype", "bitround", "quantize", "packbits"],
+)
+def test_generic_filter_encoded_shape(
+    codec: _numcodecs._NumcodecsArrayArrayCodec, dtype: str
+) -> None:
+    # numcodecs filters return flattened arrays; the encoded chunk must have the
+    # shape declared by `resolve_metadata`, or the next codec (e.g. an image
+    # serializer) receives a 1D chunk
+    if dtype == "bool":
+        data = np.tril(np.ones((16, 32), dtype=bool))
+    else:
+        data = (np.arange(16 * 32).reshape(16, 32) * 4).astype(dtype)
+    zdtype = get_data_type_from_native_dtype(data.dtype)
+    spec = ArraySpec(
+        shape=data.shape,
+        dtype=zdtype,
+        fill_value=zdtype.cast_scalar(0),
+        config=ArrayConfig(order="C", write_empty_chunks=True),
+        prototype=default_buffer_prototype(),
+    )
+    nd_buf = default_buffer_prototype().nd_buffer.from_numpy_array(data)
+
+    encoded = codec._encode_sync(nd_buf, spec)
+    assert encoded.shape == codec.resolve_metadata(spec).shape
+    decoded = codec._decode_sync(encoded, spec)
+    assert decoded.shape == data.shape
+
+
 def test_generic_filter_bitround() -> None:
     data = np.linspace(0, 1, 256, dtype="float32").reshape((16, 16))

Activity

  1. added
    bugPotential issues with the zarr-python library
    on Oct 22, 2026
  2. d-v-b commented on Oct 29, 2026

    @d-v-b
    Contributor

    thanks for this report! we will get the fix in for the next release

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    bugPotential issues with the zarr-python library

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions