Skip to content

Commit

Permalink
[sonix pygrain dataset] Support FlatMap transformations.
Browse files Browse the repository at this point in the history
PiperOrigin-RevId: 705783333
  • Loading branch information
Grain Team authored and copybara-github committed Dec 13, 2024
1 parent e5c4fd4 commit 4ee3678
Showing 1 changed file with 6 additions and 0 deletions.
6 changes: 6 additions & 0 deletions grain/_src/python/dataset/transformations/flatmap.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# limitations under the License.
"""Flatmap transformation for MapDataset."""
import functools
import sys
from typing import Any, Callable, Sequence, TypeVar

from grain._src.core import transforms
Expand All @@ -37,6 +38,11 @@ def __init__(
self._transform = transform

def __len__(self) -> int:
# If the parent dataset is on infinite repeat, its length is
# sys.maxsize and would result in overflows if further increased.
# In this case, we just keep the length as sys.maxsize.
if len(self._parent) >= sys.maxsize:
return sys.maxsize
return self._transform.max_fan_out * len(self._parent)

def __str__(self) -> str:
Expand Down

0 comments on commit 4ee3678

Please sign in to comment.