You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
run python run.py configs/eval_sailor.py -w outputs/sailor --hf-num-gpus 1 --max-num-workers 64
error:
Traceback (most recent call last):
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_dataset.py", line 3544, in _map_single
writer.write(example)
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_writer.py", line 500, in write
self.write_examples_on_file()
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_writer.py", line 455, in write_examples_on_file
row[0][col].to_pylist()[0] if isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) else row[0][col]
~~~~~~^^^^^
KeyError: 'options_2'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/run.py", line 4, in
main()
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/opencompass/cli/main.py", line 299, in main
tasks = partitioner(cfg)
^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/opencompass/partitioners/base.py", line 83, in call
tasks = self.partition(**model_and_dataset_args,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/opencompass/partitioners/num_worker.py", line 69, in partition
dataset_size = self.get_size(dataset)
^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/opencompass/partitioners/num_worker.py", line 141, in get_size
dataset = build_dataset_from_cfg(dataset)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/opencompass/utils/build.py", line 13, in build_dataset_from_cfg
return LOAD_DATASET.build(dataset_cfg)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/mmengine/registry/registry.py", line 570, in build
return self.build_func(cfg, *args, **kwargs, registry=self)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/mmengine/registry/build_functions.py", line 121, in build_from_cfg
obj = obj_cls(**args) # type: ignore
^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/opencompass/datasets/base.py", line 12, in init
self.dataset = self.load(**kwargs)
^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/opencompass/datasets/m3exam.py", line 45, in load
return dataset.map(preprocess)
^^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_dataset.py", line 602, in wrapper
out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_dataset.py", line 567, in wrapper
out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_dataset.py", line 3167, in map
for rank, done, content in Dataset._map_single(**dataset_kwargs):
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_dataset.py", line 3593, in _map_single
writer.finalize()
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_writer.py", line 599, in finalize
self.write_examples_on_file()
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_writer.py", line 455, in write_examples_on_file
row[0][col].to_pylist()[0] if isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) else row[0][col]
~~~~~~^^^^^
KeyError: 'options_2'
The text was updated successfully, but these errors were encountered:
run python run.py configs/eval_sailor.py -w outputs/sailor --hf-num-gpus 1 --max-num-workers 64
error:
Traceback (most recent call last):
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_dataset.py", line 3544, in _map_single
writer.write(example)
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_writer.py", line 500, in write
self.write_examples_on_file()
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_writer.py", line 455, in write_examples_on_file
row[0][col].to_pylist()[0] if isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) else row[0][col]
~~~~~~^^^^^
KeyError: 'options_2'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/run.py", line 4, in
main()
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/opencompass/cli/main.py", line 299, in main
tasks = partitioner(cfg)
^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/opencompass/partitioners/base.py", line 83, in call
tasks = self.partition(**model_and_dataset_args,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/opencompass/partitioners/num_worker.py", line 69, in partition
dataset_size = self.get_size(dataset)
^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/opencompass/partitioners/num_worker.py", line 141, in get_size
dataset = build_dataset_from_cfg(dataset)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/opencompass/utils/build.py", line 13, in build_dataset_from_cfg
return LOAD_DATASET.build(dataset_cfg)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/mmengine/registry/registry.py", line 570, in build
return self.build_func(cfg, *args, **kwargs, registry=self)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/mmengine/registry/build_functions.py", line 121, in build_from_cfg
obj = obj_cls(**args) # type: ignore
^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/opencompass/datasets/base.py", line 12, in init
self.dataset = self.load(**kwargs)
^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/lizhengyang/sailor-llm/opencompass/opencompass/datasets/m3exam.py", line 45, in load
return dataset.map(preprocess)
^^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_dataset.py", line 602, in wrapper
out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_dataset.py", line 567, in wrapper
out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_dataset.py", line 3167, in map
for rank, done, content in Dataset._map_single(**dataset_kwargs):
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_dataset.py", line 3593, in _map_single
writer.finalize()
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_writer.py", line 599, in finalize
self.write_examples_on_file()
File "/ofs/llm-for-dptd/conda-envs/dev/lib/python3.12/site-packages/datasets/arrow_writer.py", line 455, in write_examples_on_file
row[0][col].to_pylist()[0] if isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) else row[0][col]
~~~~~~^^^^^
KeyError: 'options_2'
The text was updated successfully, but these errors were encountered: