Skip to content

Commit

Permalink
Even better implementation
Browse files Browse the repository at this point in the history
Signed-off-by: elronbandel <[email protected]>
  • Loading branch information
elronbandel committed Feb 12, 2025
1 parent 44462d3 commit d682d27
Showing 1 changed file with 5 additions and 8 deletions.
13 changes: 5 additions & 8 deletions src/unitxt/loaders.py
Original file line number Diff line number Diff line change
Expand Up @@ -769,13 +769,6 @@ class MultipleSourceLoader(LazyLoader):

sources: List[Loader]

def prepare(self):
super().prepare()
self._fusion = FixedFusion(
subsets=self.sources, max_instances_per_subset=self.get_limit(),
include_splits=self.get_splits(),
)

def add_data_classification(self, multi_stream: MultiStream) -> MultiStream:
if self.data_classification_policy is None:
return multi_stream
Expand All @@ -788,7 +781,11 @@ def get_splits(self):
return list(set(splits))

def split_generator(self, split: str) -> Generator[Any, None, None]:
yield from self._fusion()[split]
yield from FixedFusion(
subsets=self.sources,
max_instances_per_subset=self.get_limit(),
include_splits=[split],
)()[split]


class LoadFromDictionary(Loader):
Expand Down

0 comments on commit d682d27

Please sign in to comment.