From becfd04c50845255b6cb854f970b62d97c018482 Mon Sep 17 00:00:00 2001 From: Daniel Zautner Date: Wed, 25 Mar 2026 13:20:56 +0200 Subject: [PATCH] Add --load-multilingual flag and fix --custom-tasks for inspect backend The inspect backend hardcoded load_multilingual=False and custom_tasks=None, making it impossible to use multilingual tasks or custom task files. Wire both CLI parameters through to the Registry. --- src/lighteval/main_inspect.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/lighteval/main_inspect.py b/src/lighteval/main_inspect.py index 4aa5cbf87..0175e5e12 100644 --- a/src/lighteval/main_inspect.py +++ b/src/lighteval/main_inspect.py @@ -432,10 +432,17 @@ def eval( # noqa C901 rich_help_panel=HELP_PANEL_NAME_4, ), ] = False, + load_multilingual: Annotated[ + bool, + Option( + help="Whether to load multilingual tasks from the multilingual tasks directory.", + rich_help_panel=HELP_PANEL_NAME_2, + ), + ] = False, ): from lighteval.tasks.registry import Registry - registry = Registry(tasks=tasks, custom_tasks=None, load_multilingual=False) + registry = Registry(tasks=tasks, custom_tasks=custom_tasks, load_multilingual=load_multilingual) task_configs = registry.task_to_configs inspect_ai_tasks = []