From 4c5f9074cb070d5871440fc067b1ea9f3863c5cc Mon Sep 17 00:00:00 2001 From: niujunhao Date: Mon, 24 Nov 2025 17:44:35 +0800 Subject: [PATCH] fix iter num_parallel_workers in hf streaming load. --- mindformers/dataset/dataloader/hf_dataloader.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mindformers/dataset/dataloader/hf_dataloader.py b/mindformers/dataset/dataloader/hf_dataloader.py index 94bfd6837..0839b4da4 100644 --- a/mindformers/dataset/dataloader/hf_dataloader.py +++ b/mindformers/dataset/dataloader/hf_dataloader.py @@ -354,6 +354,10 @@ class HFDataLoader: """Wrap source dataset with Mindspore Dataset.""" if getattr(config, 'streaming', False): hf_dataset = HFIterableDataset(config, dataset, num_shards, shard_id) + if num_parallel_workers > 1: + num_parallel_workers = 1 + logger.warning( + "Streaming mode only supports 'num_parallel_workers=1'. Automatically resetting the value.") else: hf_dataset = HFDataset(config, dataset) dataset = GeneratorDataset( -- Gitee