请问训练阶段没有出错，但是一进入到验证阶段时就报错TypeError: 'DataContainer' object is not iterable是什么原因
我使用的config文件是官方的config，没有做修改。
执行的命令是nohup ./tools/dist_train.sh ./configs/deeplabv3/deeplabv3_r50-d8_512x1024_40k_cityscapes.py > test.log 2>&1 &
详细报错如下：
File "./tools/train.py", line 242, in
main()
File "./tools/train.py", line 231, in main
train_segmentor(
File "/home/amax/YCH/divided-conv-mmsegmentation/mmseg/apis/train.py", line 194, in train_segmentor
runner.run(data_loaders, cfg.workflow)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/mmcv/runner/iter_based_runner.py", line 138, in run
iter_runner(iter_loaders[i], **kwargs)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/mmcv/runner/iter_based_runner.py", line 68, in train
self.call_hook('after_train_iter')
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/mmcv/runner/base_runner.py", line 309, in call_hook
getattr(hook, fn_name)(self)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/mmcv/runner/hooks/evaluation.py", line 262, in after_train_iter
self._do_evaluate(runner)
File "/home/amax/YCH/divided-conv-mmsegmentation/mmseg/core/evaluation/eval_hooks.py", line 117, in _do_evaluate
results = multi_gpu_test(
File "/home/amax/YCH/divided-conv-mmsegmentation/mmseg/apis/test.py", line 208, in multi_gpu_test
result = model(return_loss=False, rescale=True, **data)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
return forward_call(*input, **kwargs)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 1008, in forward
output = self._run_ddp_forward(*inputs, **kwargs)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 969, in _run_ddp_forward
return module_to_run(*inputs[0], **kwargs[0])
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in call_impl
return forward_call(*input, **kwargs)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/mmcv/runner/fp16_utils.py", line 116, in new_func
return old_func(*args, **kwargs)
File "/home/amax/YCH/divided-conv-mmsegmentation/mmseg/models/segmentors/base.py", line 110, in forward
return self.forward_test(img, img_metas, **kwargs)
File "/home/amax/YCH/divided-conv-mmsegmentation/mmseg/models/segmentors/base.py", line 84, in forward_test
ori_shapes = [['ori_shape'] for _ in img_meta]
TypeError: 'DataContainer' object is not iterable
Traceback (most recent call last):
File "./tools/train.py", line 242, in
Traceback (most recent call last):
File "./tools/train.py", line 242, in
main()
File "./tools/train.py", line 231, in main
train_segmentor(
File "/home/amax/YCH/divided-conv-mmsegmentation/mmseg/apis/train.py", line 194, in train_segmentor
runner.run(data_loaders, cfg.workflow)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/mmcv/runner/iter_based_runner.py", line 138, in run
main()
File "./tools/train.py", line 231, in main
train_segmentor(
File "/home/amax/YCH/divided-conv-mmsegmentation/mmseg/apis/train.py", line 194, in train_segmentor
runner.run(data_loaders, cfg.workflow)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/mmcv/runner/iter_based_runner.py", line 138, in run
iter_runner(iter_loaders[i], **kwargs)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/mmcv/runner/iter_based_runner.py", line 68, in train
self.call_hook('after_train_iter')
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/mmcv/runner/base_runner.py", line 309, in call_hook
getattr(hook, fn_name)(self)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/mmcv/runner/hooks/evaluation.py", line 262, in after_train_iter
iter_runner(iter_loaders[i], **kwargs)
self._do_evaluate(runner)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/mmcv/runner/iter_based_runner.py", line 68, in train
File "/home/amax/YCH/divided-conv-mmsegmentation/mmseg/core/evaluation/eval_hooks.py", line 117, in _do_evaluate
self.call_hook('after_train_iter')results = multi_gpu_test(

File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/mmcv/runner/base_runner.py", line 309, in call_hook
File "/home/amax/YCH/divided-conv-mmsegmentation/mmseg/apis/test.py", line 208, in multi_gpu_test
result = model(return_loss=False, rescale=True, **data)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
getattr(hook, fn_name)(self)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/mmcv/runner/hooks/evaluation.py", line 262, in after_train_iter
self._do_evaluate(runner)
File "/home/amax/YCH/divided-conv-mmsegmentation/mmseg/core/evaluation/eval_hooks.py", line 117, in _do_evaluate
results = multi_gpu_test(
File "/home/amax/YCH/divided-conv-mmsegmentation/mmseg/apis/test.py", line 208, in multi_gpu_test
return forward_call(*input, **kwargs)
result = model(return_loss=False, rescale=True, **data) File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 1008, in forward

File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
output = self._run_ddp_forward(*inputs, **kwargs)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 969, in _run_ddp_forward
return forward_call(*input, **kwargs)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 1008, in forward
return module_to_run(*inputs[0], **kwargs[0])
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
output = self._run_ddp_forward(*inputs, **kwargs)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 969, in run_ddp_forward
return forward_call(*input, **kwargs)
return module_to_run(*inputs[0], **kwargs[0])
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/mmcv/runner/fp16_utils.py", line 116, in new_func
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in call_impl
return old_func(*args, **kwargs)
File "/home/amax/YCH/divided-conv-mmsegmentation/mmseg/models/segmentors/base.py", line 110, in forward
return self.forward_test(img, img_metas, **kwargs)
File "/home/amax/YCH/divided-conv-mmsegmentation/mmseg/models/segmentors/base.py", line 84, in forward_test
ori_shapes = [['ori_shape'] for _ in img_meta]
TypeError: 'DataContainer' object is not iterable
return forward_call(*input, **kwargs)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/mmcv/runner/fp16_utils.py", line 116, in new_func
return old_func(*args, **kwargs)
File "/home/amax/YCH/divided-conv-mmsegmentation/mmseg/models/segmentors/base.py", line 110, in forward
return self.forward_test(img, img_metas, **kwargs)
File "/home/amax/YCH/divided-conv-mmsegmentation/mmseg/models/segmentors/base.py", line 84, in forward_test
ori_shapes = [['ori_shape'] for _ in img_meta]
TypeError: 'DataContainer' object is not iterable
ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 2546018) of binary: /opt/anaconda3/envs/openmmlab_seg/bin/python
Traceback (most recent call last):
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/distributed/launch.py", line 193, in
main()
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/distributed/launch.py", line 189, in main
launch(args)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/distributed/launch.py", line 174, in launch
run(args)
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run
elastic_launch(
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in call
return launch_agent(self._config, self._entrypoint, list(args))
File "/opt/anaconda3/envs/openmmlab_seg/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError: