Skip to content

Commit 50e7217

Browse files
authored
hardcode configs to not make lighteval crash if lcb repo unavailable (#677)
1 parent f96c54a commit 50e7217

File tree

1 file changed

+23
-2
lines changed
  • src/lighteval/tasks/extended/lcb

1 file changed

+23
-2
lines changed

src/lighteval/tasks/extended/lcb/main.py

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434

3535
import numpy as np
3636
from aenum import extend_enum
37-
from datasets import get_dataset_config_names
3837

3938
from lighteval.metrics.metrics import MetricCategory, Metrics, MetricUseCase, SampleLevelMetric
4039
from lighteval.tasks.extended.lcb.codegen_metrics import (
@@ -115,7 +114,29 @@ def codegen_metric(predictions: list[str], formatted_doc: Doc, **kwargs) -> floa
115114

116115
extend_enum(Metrics, "lcb_codegen_metric", lcb_codegen_metric)
117116

118-
configs = get_dataset_config_names("livecodebench/code_generation_lite", trust_remote_code=True)
117+
configs = [
118+
"release_v1",
119+
"release_v2",
120+
"release_v3",
121+
"release_v4",
122+
"release_v5",
123+
"release_latest",
124+
"v1",
125+
"v2",
126+
"v3",
127+
"v4",
128+
"v5",
129+
"v1_v2",
130+
"v1_v3",
131+
"v1_v4",
132+
"v1_v5",
133+
"v2_v3",
134+
"v2_v4",
135+
"v2_v5",
136+
"v3_v4",
137+
"v3_v5",
138+
"v4_v5",
139+
]
119140

120141
tasks = []
121142

0 commit comments

Comments
 (0)